blob: 4672553f1cc7c9b6f6dea3668d5f206958e5f5aa [file] [log] [blame]
Guido van Rossume7b146f2000-02-04 15:28:42 +00001"""A lexical analyzer class for simple shell-like syntaxes."""
2
Tim Peters70c43782001-01-17 08:48:39 +00003# Module and documentation by Eric S. Raymond, 21 Dec 1998
Guido van Rossumeb4e11a2000-05-01 20:08:46 +00004# Input stacking and error message cleanup added by ESR, March 2000
Tim Peters70c43782001-01-17 08:48:39 +00005# push_source() and pop_source() made explicit by ESR, January 2001.
Gustavo Niemeyer68d8cef2003-04-17 21:31:33 +00006# Posix compliance, split(), string arguments, and
7# iterator interface by Gustavo Niemeyer, April 2003.
Guido van Rossum9c30c241998-12-22 05:19:29 +00008
Éric Araujo9bce3112011-07-27 18:29:31 +02009import os
10import re
Guido van Rossum73898c71999-05-03 18:14:16 +000011import sys
Raymond Hettinger756b3f32004-01-29 06:37:52 +000012from collections import deque
Guido van Rossum9c30c241998-12-22 05:19:29 +000013
Guido van Rossum68937b42007-05-18 00:51:22 +000014from io import StringIO
Gustavo Niemeyer68d8cef2003-04-17 21:31:33 +000015
Éric Araujo9bce3112011-07-27 18:29:31 +020016__all__ = ["shlex", "split", "quote"]
Skip Montanaro0de65802001-02-15 22:15:14 +000017
Guido van Rossum9c30c241998-12-22 05:19:29 +000018class shlex:
Tim Peters70c43782001-01-17 08:48:39 +000019 "A lexical analyzer class for simple shell-like syntaxes."
Fred Drake24315232003-04-17 22:01:17 +000020 def __init__(self, instream=None, infile=None, posix=False):
Guido van Rossum3172c5d2007-10-16 18:12:55 +000021 if isinstance(instream, str):
Gustavo Niemeyer68d8cef2003-04-17 21:31:33 +000022 instream = StringIO(instream)
Raymond Hettingerf13eb552002-06-02 00:40:05 +000023 if instream is not None:
Guido van Rossum9c30c241998-12-22 05:19:29 +000024 self.instream = instream
Guido van Rossumeb4e11a2000-05-01 20:08:46 +000025 self.infile = infile
Guido van Rossum9c30c241998-12-22 05:19:29 +000026 else:
27 self.instream = sys.stdin
Guido van Rossumeb4e11a2000-05-01 20:08:46 +000028 self.infile = None
Gustavo Niemeyer68d8cef2003-04-17 21:31:33 +000029 self.posix = posix
30 if posix:
31 self.eof = None
32 else:
33 self.eof = ''
Guido van Rossum9c30c241998-12-22 05:19:29 +000034 self.commenters = '#'
Fred Drakedbbf76b2000-07-09 16:44:26 +000035 self.wordchars = ('abcdfeghijklmnopqrstuvwxyz'
36 'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_')
Gustavo Niemeyer68d8cef2003-04-17 21:31:33 +000037 if self.posix:
Antoine Pitroud72402e2010-10-27 18:52:48 +000038 self.wordchars += ('ßàáâãäåæçèéêëìíîïðñòóôõöøùúûüýþÿ'
39 'ÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖØÙÚÛÜÝÞ')
Guido van Rossum9c30c241998-12-22 05:19:29 +000040 self.whitespace = ' \t\r\n'
Fred Drake24315232003-04-17 22:01:17 +000041 self.whitespace_split = False
Guido van Rossum9c30c241998-12-22 05:19:29 +000042 self.quotes = '\'"'
Gustavo Niemeyer68d8cef2003-04-17 21:31:33 +000043 self.escape = '\\'
44 self.escapedquotes = '"'
Guido van Rossum9c30c241998-12-22 05:19:29 +000045 self.state = ' '
Raymond Hettinger756b3f32004-01-29 06:37:52 +000046 self.pushback = deque()
Guido van Rossum9c30c241998-12-22 05:19:29 +000047 self.lineno = 1
48 self.debug = 0
49 self.token = ''
Raymond Hettinger756b3f32004-01-29 06:37:52 +000050 self.filestack = deque()
Guido van Rossumeb4e11a2000-05-01 20:08:46 +000051 self.source = None
52 if self.debug:
Guido van Rossumbe19ed72007-02-09 05:37:30 +000053 print('shlex: reading from %s, line %d' \
54 % (self.instream, self.lineno))
Guido van Rossum9c30c241998-12-22 05:19:29 +000055
56 def push_token(self, tok):
57 "Push a token onto the stack popped by the get_token method"
Guido van Rossumeb4e11a2000-05-01 20:08:46 +000058 if self.debug >= 1:
Guido van Rossumbe19ed72007-02-09 05:37:30 +000059 print("shlex: pushing token " + repr(tok))
Raymond Hettinger756b3f32004-01-29 06:37:52 +000060 self.pushback.appendleft(tok)
Guido van Rossum9c30c241998-12-22 05:19:29 +000061
Eric S. Raymondbddbaf72001-01-16 15:19:13 +000062 def push_source(self, newstream, newfile=None):
63 "Push an input source onto the lexer's input source stack."
Guido van Rossum3172c5d2007-10-16 18:12:55 +000064 if isinstance(newstream, str):
Gustavo Niemeyer68d8cef2003-04-17 21:31:33 +000065 newstream = StringIO(newstream)
Raymond Hettinger756b3f32004-01-29 06:37:52 +000066 self.filestack.appendleft((self.infile, self.instream, self.lineno))
Eric S. Raymondbddbaf72001-01-16 15:19:13 +000067 self.infile = newfile
68 self.instream = newstream
69 self.lineno = 1
70 if self.debug:
Raymond Hettingerf13eb552002-06-02 00:40:05 +000071 if newfile is not None:
Guido van Rossumbe19ed72007-02-09 05:37:30 +000072 print('shlex: pushing to file %s' % (self.infile,))
Eric S. Raymondbddbaf72001-01-16 15:19:13 +000073 else:
Guido van Rossumbe19ed72007-02-09 05:37:30 +000074 print('shlex: pushing to stream %s' % (self.instream,))
Eric S. Raymondbddbaf72001-01-16 15:19:13 +000075
76 def pop_source(self):
77 "Pop the input source stack."
78 self.instream.close()
Raymond Hettinger756b3f32004-01-29 06:37:52 +000079 (self.infile, self.instream, self.lineno) = self.filestack.popleft()
Eric S. Raymondbddbaf72001-01-16 15:19:13 +000080 if self.debug:
Guido van Rossumbe19ed72007-02-09 05:37:30 +000081 print('shlex: popping to %s, line %d' \
82 % (self.instream, self.lineno))
Eric S. Raymondbddbaf72001-01-16 15:19:13 +000083 self.state = ' '
84
Guido van Rossum9c30c241998-12-22 05:19:29 +000085 def get_token(self):
Guido van Rossumeb4e11a2000-05-01 20:08:46 +000086 "Get a token from the input stream (or from stack if it's nonempty)"
Guido van Rossum9c30c241998-12-22 05:19:29 +000087 if self.pushback:
Raymond Hettinger756b3f32004-01-29 06:37:52 +000088 tok = self.pushback.popleft()
Guido van Rossumeb4e11a2000-05-01 20:08:46 +000089 if self.debug >= 1:
Guido van Rossumbe19ed72007-02-09 05:37:30 +000090 print("shlex: popping token " + repr(tok))
Guido van Rossum9c30c241998-12-22 05:19:29 +000091 return tok
Fred Drakedbbf76b2000-07-09 16:44:26 +000092 # No pushback. Get a token.
Guido van Rossumeb4e11a2000-05-01 20:08:46 +000093 raw = self.read_token()
94 # Handle inclusions
Gustavo Niemeyer68d8cef2003-04-17 21:31:33 +000095 if self.source is not None:
96 while raw == self.source:
97 spec = self.sourcehook(self.read_token())
98 if spec:
99 (newfile, newstream) = spec
100 self.push_source(newstream, newfile)
101 raw = self.get_token()
Guido van Rossumeb4e11a2000-05-01 20:08:46 +0000102 # Maybe we got EOF instead?
Gustavo Niemeyer68d8cef2003-04-17 21:31:33 +0000103 while raw == self.eof:
Fred Drake24315232003-04-17 22:01:17 +0000104 if not self.filestack:
Gustavo Niemeyer68d8cef2003-04-17 21:31:33 +0000105 return self.eof
Guido van Rossumeb4e11a2000-05-01 20:08:46 +0000106 else:
Eric S. Raymondbddbaf72001-01-16 15:19:13 +0000107 self.pop_source()
Guido van Rossumeb4e11a2000-05-01 20:08:46 +0000108 raw = self.get_token()
Gustavo Niemeyer68d8cef2003-04-17 21:31:33 +0000109 # Neither inclusion nor EOF
Guido van Rossumeb4e11a2000-05-01 20:08:46 +0000110 if self.debug >= 1:
Gustavo Niemeyer68d8cef2003-04-17 21:31:33 +0000111 if raw != self.eof:
Guido van Rossumbe19ed72007-02-09 05:37:30 +0000112 print("shlex: token=" + repr(raw))
Guido van Rossumeb4e11a2000-05-01 20:08:46 +0000113 else:
Guido van Rossumbe19ed72007-02-09 05:37:30 +0000114 print("shlex: token=EOF")
Guido van Rossumeb4e11a2000-05-01 20:08:46 +0000115 return raw
116
117 def read_token(self):
Fred Drake24315232003-04-17 22:01:17 +0000118 quoted = False
Gustavo Niemeyer68d8cef2003-04-17 21:31:33 +0000119 escapedstate = ' '
Neal Norwitz10cf2182003-04-17 23:09:08 +0000120 while True:
Andrew M. Kuchling49d27c82000-12-23 14:20:24 +0000121 nextchar = self.instream.read(1)
Guido van Rossum9c30c241998-12-22 05:19:29 +0000122 if nextchar == '\n':
Petri Lehtinen0362b542013-02-23 23:03:15 +0100123 self.lineno = self.lineno + 1
Guido van Rossum9c30c241998-12-22 05:19:29 +0000124 if self.debug >= 3:
Guido van Rossumbe19ed72007-02-09 05:37:30 +0000125 print("shlex: in state", repr(self.state), \
126 "I see character:", repr(nextchar))
Fred Drakedbbf76b2000-07-09 16:44:26 +0000127 if self.state is None:
Eric S. Raymondbddbaf72001-01-16 15:19:13 +0000128 self.token = '' # past end of file
Guido van Rossumeb4e11a2000-05-01 20:08:46 +0000129 break
Guido van Rossum9c30c241998-12-22 05:19:29 +0000130 elif self.state == ' ':
131 if not nextchar:
Eric S. Raymondbddbaf72001-01-16 15:19:13 +0000132 self.state = None # end of file
Guido van Rossum9c30c241998-12-22 05:19:29 +0000133 break
134 elif nextchar in self.whitespace:
135 if self.debug >= 2:
Guido van Rossumbe19ed72007-02-09 05:37:30 +0000136 print("shlex: I see whitespace in whitespace state")
Gustavo Niemeyer68d8cef2003-04-17 21:31:33 +0000137 if self.token or (self.posix and quoted):
Fred Drakedbbf76b2000-07-09 16:44:26 +0000138 break # emit current token
Guido van Rossum9c30c241998-12-22 05:19:29 +0000139 else:
140 continue
141 elif nextchar in self.commenters:
142 self.instream.readline()
143 self.lineno = self.lineno + 1
Gustavo Niemeyer68d8cef2003-04-17 21:31:33 +0000144 elif self.posix and nextchar in self.escape:
145 escapedstate = 'a'
146 self.state = nextchar
Guido van Rossum9c30c241998-12-22 05:19:29 +0000147 elif nextchar in self.wordchars:
148 self.token = nextchar
149 self.state = 'a'
150 elif nextchar in self.quotes:
Gustavo Niemeyer68d8cef2003-04-17 21:31:33 +0000151 if not self.posix:
152 self.token = nextchar
Guido van Rossum9c30c241998-12-22 05:19:29 +0000153 self.state = nextchar
Gustavo Niemeyer68d8cef2003-04-17 21:31:33 +0000154 elif self.whitespace_split:
155 self.token = nextchar
156 self.state = 'a'
Guido van Rossum9c30c241998-12-22 05:19:29 +0000157 else:
158 self.token = nextchar
Gustavo Niemeyer68d8cef2003-04-17 21:31:33 +0000159 if self.token or (self.posix and quoted):
Fred Drakedbbf76b2000-07-09 16:44:26 +0000160 break # emit current token
Guido van Rossum9c30c241998-12-22 05:19:29 +0000161 else:
162 continue
163 elif self.state in self.quotes:
Fred Drake24315232003-04-17 22:01:17 +0000164 quoted = True
Gustavo Niemeyer68d8cef2003-04-17 21:31:33 +0000165 if not nextchar: # end of file
Andrew M. Kuchling9d56cd12001-01-09 03:01:15 +0000166 if self.debug >= 2:
Guido van Rossumbe19ed72007-02-09 05:37:30 +0000167 print("shlex: I see EOF in quotes state")
Andrew M. Kuchling9d56cd12001-01-09 03:01:15 +0000168 # XXX what error should be raised here?
Collin Winterce36ad82007-08-30 01:19:48 +0000169 raise ValueError("No closing quotation")
Gustavo Niemeyer68d8cef2003-04-17 21:31:33 +0000170 if nextchar == self.state:
171 if not self.posix:
172 self.token = self.token + nextchar
173 self.state = ' '
174 break
175 else:
176 self.state = 'a'
177 elif self.posix and nextchar in self.escape and \
178 self.state in self.escapedquotes:
179 escapedstate = self.state
180 self.state = nextchar
181 else:
182 self.token = self.token + nextchar
183 elif self.state in self.escape:
184 if not nextchar: # end of file
185 if self.debug >= 2:
Guido van Rossumbe19ed72007-02-09 05:37:30 +0000186 print("shlex: I see EOF in escape state")
Gustavo Niemeyer68d8cef2003-04-17 21:31:33 +0000187 # XXX what error should be raised here?
Collin Winterce36ad82007-08-30 01:19:48 +0000188 raise ValueError("No escaped character")
Gustavo Niemeyer68d8cef2003-04-17 21:31:33 +0000189 # In posix shells, only the quote itself or the escape
190 # character may be escaped within quotes.
191 if escapedstate in self.quotes and \
192 nextchar != self.state and nextchar != escapedstate:
193 self.token = self.token + self.state
194 self.token = self.token + nextchar
195 self.state = escapedstate
Guido van Rossum9c30c241998-12-22 05:19:29 +0000196 elif self.state == 'a':
197 if not nextchar:
Tim Peters70c43782001-01-17 08:48:39 +0000198 self.state = None # end of file
Guido van Rossum9c30c241998-12-22 05:19:29 +0000199 break
200 elif nextchar in self.whitespace:
201 if self.debug >= 2:
Guido van Rossumbe19ed72007-02-09 05:37:30 +0000202 print("shlex: I see whitespace in word state")
Guido van Rossum9c30c241998-12-22 05:19:29 +0000203 self.state = ' '
Gustavo Niemeyer68d8cef2003-04-17 21:31:33 +0000204 if self.token or (self.posix and quoted):
Fred Drakedbbf76b2000-07-09 16:44:26 +0000205 break # emit current token
Guido van Rossum9c30c241998-12-22 05:19:29 +0000206 else:
207 continue
208 elif nextchar in self.commenters:
209 self.instream.readline()
210 self.lineno = self.lineno + 1
Gustavo Niemeyer68d8cef2003-04-17 21:31:33 +0000211 if self.posix:
212 self.state = ' '
213 if self.token or (self.posix and quoted):
214 break # emit current token
215 else:
216 continue
217 elif self.posix and nextchar in self.quotes:
218 self.state = nextchar
219 elif self.posix and nextchar in self.escape:
220 escapedstate = 'a'
221 self.state = nextchar
222 elif nextchar in self.wordchars or nextchar in self.quotes \
223 or self.whitespace_split:
Guido van Rossum9c30c241998-12-22 05:19:29 +0000224 self.token = self.token + nextchar
225 else:
Raymond Hettinger756b3f32004-01-29 06:37:52 +0000226 self.pushback.appendleft(nextchar)
Guido van Rossum9c30c241998-12-22 05:19:29 +0000227 if self.debug >= 2:
Guido van Rossumbe19ed72007-02-09 05:37:30 +0000228 print("shlex: I see punctuation in word state")
Guido van Rossumf247d751999-03-22 15:28:08 +0000229 self.state = ' '
Guido van Rossum9c30c241998-12-22 05:19:29 +0000230 if self.token:
Fred Drakedbbf76b2000-07-09 16:44:26 +0000231 break # emit current token
Guido van Rossum9c30c241998-12-22 05:19:29 +0000232 else:
233 continue
Guido van Rossum9c30c241998-12-22 05:19:29 +0000234 result = self.token
235 self.token = ''
Gustavo Niemeyer68d8cef2003-04-17 21:31:33 +0000236 if self.posix and not quoted and result == '':
237 result = None
Guido van Rossumeb4e11a2000-05-01 20:08:46 +0000238 if self.debug > 1:
239 if result:
Guido van Rossumbe19ed72007-02-09 05:37:30 +0000240 print("shlex: raw token=" + repr(result))
Guido van Rossumeb4e11a2000-05-01 20:08:46 +0000241 else:
Guido van Rossumbe19ed72007-02-09 05:37:30 +0000242 print("shlex: raw token=EOF")
Guido van Rossum9c30c241998-12-22 05:19:29 +0000243 return result
244
Guido van Rossumeb4e11a2000-05-01 20:08:46 +0000245 def sourcehook(self, newfile):
246 "Hook called on a filename to be sourced."
247 if newfile[0] == '"':
248 newfile = newfile[1:-1]
Fred Drake52dc76c2000-07-03 09:56:23 +0000249 # This implements cpp-like semantics for relative-path inclusion.
Guido van Rossum3172c5d2007-10-16 18:12:55 +0000250 if isinstance(self.infile, str) and not os.path.isabs(newfile):
Fred Drake52dc76c2000-07-03 09:56:23 +0000251 newfile = os.path.join(os.path.dirname(self.infile), newfile)
Guido van Rossumeb4e11a2000-05-01 20:08:46 +0000252 return (newfile, open(newfile, "r"))
253
Guido van Rossum4b83ecb2000-05-01 20:14:12 +0000254 def error_leader(self, infile=None, lineno=None):
255 "Emit a C-compiler-like, Emacs-friendly error-message leader."
Raymond Hettingerf13eb552002-06-02 00:40:05 +0000256 if infile is None:
Guido van Rossum4b83ecb2000-05-01 20:14:12 +0000257 infile = self.infile
Raymond Hettingerf13eb552002-06-02 00:40:05 +0000258 if lineno is None:
Guido van Rossum4b83ecb2000-05-01 20:14:12 +0000259 lineno = self.lineno
260 return "\"%s\", line %d: " % (infile, lineno)
261
Gustavo Niemeyer68d8cef2003-04-17 21:31:33 +0000262 def __iter__(self):
263 return self
264
Georg Brandla18af4e2007-04-21 15:47:16 +0000265 def __next__(self):
Gustavo Niemeyer68d8cef2003-04-17 21:31:33 +0000266 token = self.get_token()
267 if token == self.eof:
268 raise StopIteration
269 return token
270
Guido van Rossume7ba4952007-06-06 23:52:48 +0000271def split(s, comments=False, posix=True):
272 lex = shlex(s, posix=posix)
Gustavo Niemeyer48f3dcc2003-04-20 01:57:03 +0000273 lex.whitespace_split = True
274 if not comments:
275 lex.commenters = ''
Gustavo Niemeyer68d8cef2003-04-17 21:31:33 +0000276 return list(lex)
Guido van Rossum9c30c241998-12-22 05:19:29 +0000277
Éric Araujo9bce3112011-07-27 18:29:31 +0200278
Ezio Melotti67321cc2011-08-16 19:03:41 +0300279_find_unsafe = re.compile(r'[^\w@%+=:,./-]', re.ASCII).search
Éric Araujo9bce3112011-07-27 18:29:31 +0200280
281def quote(s):
282 """Return a shell-escaped version of the string *s*."""
283 if not s:
284 return "''"
285 if _find_unsafe(s) is None:
286 return s
287
288 # use single quotes, and put single quotes into double quotes
289 # the string $'b is then quoted as '$'"'"'b'
290 return "'" + s.replace("'", "'\"'\"'") + "'"
291
292
R David Murray838f2c42014-10-17 20:28:47 -0400293def _print_tokens(lexer):
Guido van Rossum9c30c241998-12-22 05:19:29 +0000294 while 1:
295 tt = lexer.get_token()
R David Murray838f2c42014-10-17 20:28:47 -0400296 if not tt:
Guido van Rossum9c30c241998-12-22 05:19:29 +0000297 break
R David Murray838f2c42014-10-17 20:28:47 -0400298 print("Token: " + repr(tt))
299
300if __name__ == '__main__':
301 if len(sys.argv) == 1:
302 _print_tokens(shlex())
303 else:
304 fn = sys.argv[1]
305 with open(fn) as f:
306 _print_tokens(shlex(f, fn))