blob: 19932ca8e64f1cd8da079795f1b2d950f8a7bf94 [file] [log] [blame]
Guido van Rossum7d5b99d1997-11-21 17:12:59 +00001"""Helper class to quickly write a loop over all standard input files.
2
3Typical use is:
4
5 import fileinput
6 for line in fileinput.input():
7 process(line)
8
9This iterates over the lines of all files listed in sys.argv[1:],
10defaulting to sys.stdin if the list is empty. If a filename is '-' it
11is also replaced by sys.stdin. To specify an alternative list of
12filenames, pass it as the argument to input(). A single file name is
13also allowed.
14
15Functions filename(), lineno() return the filename and cumulative line
16number of the line that has just been read; filelineno() returns its
17line number in the current file; isfirstline() returns true iff the
18line just read is the first line of its file; isstdin() returns true
19iff the line was read from sys.stdin. Function nextfile() closes the
20current file so that the next iteration will read the first line from
21the next file (if any); lines not read from the file will not count
22towards the cumulative line count; the filename is not changed until
23after the first line of the next file has been read. Function close()
24closes the sequence.
25
26Before any lines have been read, filename() returns None and both line
27numbers are zero; nextfile() has no effect. After all lines have been
28read, filename() and the line number functions return the values
29pertaining to the last line read; nextfile() has no effect.
30
Georg Brandlc029f872006-02-19 14:12:34 +000031All files are opened in text mode by default, you can override this by
32setting the mode parameter to input() or FileInput.__init__().
33If an I/O error occurs during opening or reading a file, the IOError
34exception is raised.
Guido van Rossum7d5b99d1997-11-21 17:12:59 +000035
36If sys.stdin is used more than once, the second and further use will
37return no lines, except perhaps for interactive use, or if it has been
38explicitly reset (e.g. using sys.stdin.seek(0)).
39
40Empty files are opened and immediately closed; the only time their
41presence in the list of filenames is noticeable at all is when the
42last file opened is empty.
43
44It is possible that the last line of a file doesn't end in a newline
45character; otherwise lines are returned including the trailing
46newline.
47
48Class FileInput is the implementation; its methods filename(),
49lineno(), fileline(), isfirstline(), isstdin(), nextfile() and close()
50correspond to the functions in the module. In addition it has a
51readline() method which returns the next input line, and a
52__getitem__() method which implements the sequence behavior. The
53sequence must be accessed in strictly sequential order; sequence
54access and readline() cannot be mixed.
55
56Optional in-place filtering: if the keyword argument inplace=1 is
57passed to input() or to the FileInput constructor, the file is moved
58to a backup file and standard output is directed to the input file.
59This makes it possible to write a filter that rewrites its input file
60in place. If the keyword argument backup=".<some extension>" is also
61given, it specifies the extension for the backup file, and the backup
62file remains around; by default, the extension is ".bak" and it is
63deleted when the output file is closed. In-place filtering is
64disabled when standard input is read. XXX The current implementation
65does not work for MS-DOS 8+3 filesystems.
66
Guido van Rossum47955242001-01-05 14:44:39 +000067Performance: this module is unfortunately one of the slower ways of
68processing large numbers of input lines. Nevertheless, a significant
69speed-up has been obtained by using readlines(bufsize) instead of
70readline(). A new keyword argument, bufsize=N, is present on the
71input() function and the FileInput() class to override the default
72buffer size.
73
Guido van Rossum7d5b99d1997-11-21 17:12:59 +000074XXX Possible additions:
75
76- optional getopt argument processing
Guido van Rossum7d5b99d1997-11-21 17:12:59 +000077- isatty()
78- read(), read(size), even readlines()
79
80"""
81
Walter Dörwald294bbf32002-06-06 09:48:13 +000082import sys, os
Guido van Rossum7d5b99d1997-11-21 17:12:59 +000083
Skip Montanaroeccd02a2001-01-20 23:34:12 +000084__all__ = ["input","close","nextfile","filename","lineno","filelineno",
85 "isfirstline","isstdin","FileInput"]
86
Guido van Rossum7d5b99d1997-11-21 17:12:59 +000087_state = None
88
Guido van Rossum47955242001-01-05 14:44:39 +000089DEFAULT_BUFSIZE = 8*1024
90
Tim Peters200a5802006-02-19 21:26:07 +000091def input(files=None, inplace=0, backup="", bufsize=0,
Georg Brandlc98eeed2006-02-19 14:57:47 +000092 mode="r", openhook=None):
93 """input([files[, inplace[, backup[, mode[, openhook]]]]])
Raymond Hettingerd1fa3db2002-05-15 02:56:03 +000094
95 Create an instance of the FileInput class. The instance will be used
96 as global state for the functions of this module, and is also returned
97 to use during iteration. The parameters to this function will be passed
Tim Peters8ac14952002-05-23 15:15:30 +000098 along to the constructor of the FileInput class.
Raymond Hettingerd1fa3db2002-05-15 02:56:03 +000099 """
Guido van Rossum7d5b99d1997-11-21 17:12:59 +0000100 global _state
101 if _state and _state._file:
Guido van Rossum45e2fbc1998-03-26 21:13:24 +0000102 raise RuntimeError, "input() already active"
Georg Brandlc98eeed2006-02-19 14:57:47 +0000103 _state = FileInput(files, inplace, backup, bufsize, mode, openhook)
Guido van Rossum7d5b99d1997-11-21 17:12:59 +0000104 return _state
105
106def close():
Raymond Hettingerd1fa3db2002-05-15 02:56:03 +0000107 """Close the sequence."""
Guido van Rossum7d5b99d1997-11-21 17:12:59 +0000108 global _state
109 state = _state
110 _state = None
111 if state:
Guido van Rossum45e2fbc1998-03-26 21:13:24 +0000112 state.close()
Guido van Rossum7d5b99d1997-11-21 17:12:59 +0000113
114def nextfile():
Raymond Hettingerd1fa3db2002-05-15 02:56:03 +0000115 """
116 Close the current file so that the next iteration will read the first
117 line from the next file (if any); lines not read from the file will
118 not count towards the cumulative line count. The filename is not
119 changed until after the first line of the next file has been read.
120 Before the first line has been read, this function has no effect;
121 it cannot be used to skip the first file. After the last line of the
Tim Peters8ac14952002-05-23 15:15:30 +0000122 last file has been read, this function has no effect.
Raymond Hettingerd1fa3db2002-05-15 02:56:03 +0000123 """
Guido van Rossum7d5b99d1997-11-21 17:12:59 +0000124 if not _state:
Guido van Rossum45e2fbc1998-03-26 21:13:24 +0000125 raise RuntimeError, "no active input()"
Guido van Rossum7d5b99d1997-11-21 17:12:59 +0000126 return _state.nextfile()
127
128def filename():
Raymond Hettingerd1fa3db2002-05-15 02:56:03 +0000129 """
130 Return the name of the file currently being read.
Tim Peters8ac14952002-05-23 15:15:30 +0000131 Before the first line has been read, returns None.
Raymond Hettingerd1fa3db2002-05-15 02:56:03 +0000132 """
Guido van Rossum7d5b99d1997-11-21 17:12:59 +0000133 if not _state:
Guido van Rossum45e2fbc1998-03-26 21:13:24 +0000134 raise RuntimeError, "no active input()"
Guido van Rossum7d5b99d1997-11-21 17:12:59 +0000135 return _state.filename()
136
137def lineno():
Raymond Hettingerd1fa3db2002-05-15 02:56:03 +0000138 """
139 Return the cumulative line number of the line that has just been read.
140 Before the first line has been read, returns 0. After the last line
Tim Peters8ac14952002-05-23 15:15:30 +0000141 of the last file has been read, returns the line number of that line.
Raymond Hettingerd1fa3db2002-05-15 02:56:03 +0000142 """
Guido van Rossum7d5b99d1997-11-21 17:12:59 +0000143 if not _state:
Guido van Rossum45e2fbc1998-03-26 21:13:24 +0000144 raise RuntimeError, "no active input()"
Guido van Rossum7d5b99d1997-11-21 17:12:59 +0000145 return _state.lineno()
146
147def filelineno():
Raymond Hettingerd1fa3db2002-05-15 02:56:03 +0000148 """
149 Return the line number in the current file. Before the first line
150 has been read, returns 0. After the last line of the last file has
Tim Peters8ac14952002-05-23 15:15:30 +0000151 been read, returns the line number of that line within the file.
Raymond Hettingerd1fa3db2002-05-15 02:56:03 +0000152 """
Guido van Rossum7d5b99d1997-11-21 17:12:59 +0000153 if not _state:
Guido van Rossum45e2fbc1998-03-26 21:13:24 +0000154 raise RuntimeError, "no active input()"
Guido van Rossum7d5b99d1997-11-21 17:12:59 +0000155 return _state.filelineno()
156
Georg Brandl67e9fb92006-02-19 13:56:17 +0000157def fileno():
158 """
159 Return the file number of the current file. When no file is currently
160 opened, returns -1.
161 """
162 if not _state:
163 raise RuntimeError, "no active input()"
164 return _state.fileno()
165
Guido van Rossum7d5b99d1997-11-21 17:12:59 +0000166def isfirstline():
Raymond Hettingerd1fa3db2002-05-15 02:56:03 +0000167 """
168 Returns true the line just read is the first line of its file,
Tim Peters8ac14952002-05-23 15:15:30 +0000169 otherwise returns false.
Raymond Hettingerd1fa3db2002-05-15 02:56:03 +0000170 """
Guido van Rossum7d5b99d1997-11-21 17:12:59 +0000171 if not _state:
Guido van Rossum45e2fbc1998-03-26 21:13:24 +0000172 raise RuntimeError, "no active input()"
Guido van Rossum7d5b99d1997-11-21 17:12:59 +0000173 return _state.isfirstline()
174
175def isstdin():
Raymond Hettingerd1fa3db2002-05-15 02:56:03 +0000176 """
177 Returns true if the last line was read from sys.stdin,
Tim Peters8ac14952002-05-23 15:15:30 +0000178 otherwise returns false.
Raymond Hettingerd1fa3db2002-05-15 02:56:03 +0000179 """
Guido van Rossum7d5b99d1997-11-21 17:12:59 +0000180 if not _state:
Guido van Rossum45e2fbc1998-03-26 21:13:24 +0000181 raise RuntimeError, "no active input()"
Guido van Rossum7d5b99d1997-11-21 17:12:59 +0000182 return _state.isstdin()
183
184class FileInput:
Georg Brandlc98eeed2006-02-19 14:57:47 +0000185 """class FileInput([files[, inplace[, backup[, mode[, openhook]]]]])
Tim Peters8ac14952002-05-23 15:15:30 +0000186
Raymond Hettingerd1fa3db2002-05-15 02:56:03 +0000187 Class FileInput is the implementation of the module; its methods
Georg Brandl67e9fb92006-02-19 13:56:17 +0000188 filename(), lineno(), fileline(), isfirstline(), isstdin(), fileno(),
189 nextfile() and close() correspond to the functions of the same name
190 in the module.
Raymond Hettingerd1fa3db2002-05-15 02:56:03 +0000191 In addition it has a readline() method which returns the next
192 input line, and a __getitem__() method which implements the
193 sequence behavior. The sequence must be accessed in strictly
Tim Peters8ac14952002-05-23 15:15:30 +0000194 sequential order; random access and readline() cannot be mixed.
Raymond Hettingerd1fa3db2002-05-15 02:56:03 +0000195 """
Guido van Rossum7d5b99d1997-11-21 17:12:59 +0000196
Tim Peters200a5802006-02-19 21:26:07 +0000197 def __init__(self, files=None, inplace=0, backup="", bufsize=0,
Georg Brandlc98eeed2006-02-19 14:57:47 +0000198 mode="r", openhook=None):
Georg Brandle4662172006-02-19 09:51:27 +0000199 if isinstance(files, basestring):
Guido van Rossum45e2fbc1998-03-26 21:13:24 +0000200 files = (files,)
201 else:
Guido van Rossum2516b392000-04-10 17:16:12 +0000202 if files is None:
203 files = sys.argv[1:]
Guido van Rossum45e2fbc1998-03-26 21:13:24 +0000204 if not files:
Guido van Rossum2516b392000-04-10 17:16:12 +0000205 files = ('-',)
206 else:
207 files = tuple(files)
Guido van Rossum45e2fbc1998-03-26 21:13:24 +0000208 self._files = files
209 self._inplace = inplace
210 self._backup = backup
Guido van Rossum47955242001-01-05 14:44:39 +0000211 self._bufsize = bufsize or DEFAULT_BUFSIZE
Guido van Rossum45e2fbc1998-03-26 21:13:24 +0000212 self._savestdout = None
213 self._output = None
214 self._filename = None
215 self._lineno = 0
216 self._filelineno = 0
217 self._file = None
Guido van Rossum8ca162f2002-04-07 06:36:23 +0000218 self._isstdin = False
Guido van Rossum0aec9fb1998-07-20 15:49:28 +0000219 self._backupfilename = None
Guido van Rossum47955242001-01-05 14:44:39 +0000220 self._buffer = []
221 self._bufindex = 0
Georg Brandlc029f872006-02-19 14:12:34 +0000222 # restrict mode argument to reading modes
223 if mode not in ('r', 'rU', 'U', 'rb'):
224 raise ValueError("FileInput opening mode must be one of "
225 "'r', 'rU', 'U' and 'rb'")
226 self._mode = mode
Georg Brandlc98eeed2006-02-19 14:57:47 +0000227 if inplace and openhook:
228 raise ValueError("FileInput cannot use an opening hook in inplace mode")
229 elif openhook and not callable(openhook):
230 raise ValueError("FileInput openhook must be callable")
231 self._openhook = openhook
Guido van Rossum7d5b99d1997-11-21 17:12:59 +0000232
233 def __del__(self):
Guido van Rossum45e2fbc1998-03-26 21:13:24 +0000234 self.close()
Guido van Rossum7d5b99d1997-11-21 17:12:59 +0000235
236 def close(self):
Guido van Rossum45e2fbc1998-03-26 21:13:24 +0000237 self.nextfile()
238 self._files = ()
Guido van Rossum7d5b99d1997-11-21 17:12:59 +0000239
Neil Schemenauer908632a2002-03-26 20:28:40 +0000240 def __iter__(self):
241 return self
242
243 def next(self):
Guido van Rossum47955242001-01-05 14:44:39 +0000244 try:
245 line = self._buffer[self._bufindex]
246 except IndexError:
247 pass
248 else:
249 self._bufindex += 1
250 self._lineno += 1
251 self._filelineno += 1
252 return line
Guido van Rossum45e2fbc1998-03-26 21:13:24 +0000253 line = self.readline()
254 if not line:
Neil Schemenauer908632a2002-03-26 20:28:40 +0000255 raise StopIteration
Guido van Rossum45e2fbc1998-03-26 21:13:24 +0000256 return line
Tim Peters863ac442002-04-16 01:38:40 +0000257
Neil Schemenauer908632a2002-03-26 20:28:40 +0000258 def __getitem__(self, i):
259 if i != self._lineno:
260 raise RuntimeError, "accessing lines out of order"
261 try:
262 return self.next()
263 except StopIteration:
264 raise IndexError, "end of input reached"
Guido van Rossum7d5b99d1997-11-21 17:12:59 +0000265
266 def nextfile(self):
Guido van Rossum45e2fbc1998-03-26 21:13:24 +0000267 savestdout = self._savestdout
268 self._savestdout = 0
269 if savestdout:
270 sys.stdout = savestdout
Guido van Rossum7d5b99d1997-11-21 17:12:59 +0000271
Guido van Rossum45e2fbc1998-03-26 21:13:24 +0000272 output = self._output
273 self._output = 0
274 if output:
275 output.close()
Guido van Rossum7d5b99d1997-11-21 17:12:59 +0000276
Guido van Rossum45e2fbc1998-03-26 21:13:24 +0000277 file = self._file
278 self._file = 0
279 if file and not self._isstdin:
280 file.close()
Guido van Rossum7d5b99d1997-11-21 17:12:59 +0000281
Guido van Rossum45e2fbc1998-03-26 21:13:24 +0000282 backupfilename = self._backupfilename
283 self._backupfilename = 0
284 if backupfilename and not self._backup:
285 try: os.unlink(backupfilename)
Skip Montanarocffac662002-08-14 02:58:16 +0000286 except OSError: pass
Guido van Rossum7d5b99d1997-11-21 17:12:59 +0000287
Guido van Rossum8ca162f2002-04-07 06:36:23 +0000288 self._isstdin = False
Guido van Rossum47955242001-01-05 14:44:39 +0000289 self._buffer = []
290 self._bufindex = 0
Guido van Rossum7d5b99d1997-11-21 17:12:59 +0000291
292 def readline(self):
Guido van Rossum47955242001-01-05 14:44:39 +0000293 try:
294 line = self._buffer[self._bufindex]
295 except IndexError:
296 pass
297 else:
298 self._bufindex += 1
299 self._lineno += 1
300 self._filelineno += 1
301 return line
Guido van Rossum45e2fbc1998-03-26 21:13:24 +0000302 if not self._file:
303 if not self._files:
304 return ""
305 self._filename = self._files[0]
306 self._files = self._files[1:]
307 self._filelineno = 0
308 self._file = None
Guido van Rossum8ca162f2002-04-07 06:36:23 +0000309 self._isstdin = False
Guido van Rossum45e2fbc1998-03-26 21:13:24 +0000310 self._backupfilename = 0
311 if self._filename == '-':
312 self._filename = '<stdin>'
313 self._file = sys.stdin
Guido van Rossum8ca162f2002-04-07 06:36:23 +0000314 self._isstdin = True
Guido van Rossum45e2fbc1998-03-26 21:13:24 +0000315 else:
316 if self._inplace:
317 self._backupfilename = (
Guido van Rossume2ae77b2001-10-24 20:42:55 +0000318 self._filename + (self._backup or os.extsep+"bak"))
Guido van Rossum45e2fbc1998-03-26 21:13:24 +0000319 try: os.unlink(self._backupfilename)
320 except os.error: pass
Guido van Rossumdcb85831999-10-18 21:41:43 +0000321 # The next few lines may raise IOError
Guido van Rossum45e2fbc1998-03-26 21:13:24 +0000322 os.rename(self._filename, self._backupfilename)
Georg Brandlc029f872006-02-19 14:12:34 +0000323 self._file = open(self._backupfilename, self._mode)
Guido van Rossumdcb85831999-10-18 21:41:43 +0000324 try:
Walter Dörwald294bbf32002-06-06 09:48:13 +0000325 perm = os.fstat(self._file.fileno()).st_mode
Skip Montanarocffac662002-08-14 02:58:16 +0000326 except OSError:
Guido van Rossumdcb85831999-10-18 21:41:43 +0000327 self._output = open(self._filename, "w")
328 else:
329 fd = os.open(self._filename,
330 os.O_CREAT | os.O_WRONLY | os.O_TRUNC,
331 perm)
332 self._output = os.fdopen(fd, "w")
333 try:
Jack Jansen52941a82003-01-08 16:33:16 +0000334 if hasattr(os, 'chmod'):
335 os.chmod(self._filename, perm)
Skip Montanarocffac662002-08-14 02:58:16 +0000336 except OSError:
Guido van Rossumdcb85831999-10-18 21:41:43 +0000337 pass
Guido van Rossum45e2fbc1998-03-26 21:13:24 +0000338 self._savestdout = sys.stdout
339 sys.stdout = self._output
340 else:
341 # This may raise IOError
Georg Brandlc98eeed2006-02-19 14:57:47 +0000342 if self._openhook:
343 self._file = self._openhook(self._filename, self._mode)
344 else:
345 self._file = open(self._filename, self._mode)
Guido van Rossum47955242001-01-05 14:44:39 +0000346 self._buffer = self._file.readlines(self._bufsize)
347 self._bufindex = 0
348 if not self._buffer:
349 self.nextfile()
Guido van Rossum45e2fbc1998-03-26 21:13:24 +0000350 # Recursive call
351 return self.readline()
Guido van Rossum7d5b99d1997-11-21 17:12:59 +0000352
353 def filename(self):
Guido van Rossum45e2fbc1998-03-26 21:13:24 +0000354 return self._filename
Guido van Rossum7d5b99d1997-11-21 17:12:59 +0000355
356 def lineno(self):
Guido van Rossum45e2fbc1998-03-26 21:13:24 +0000357 return self._lineno
Guido van Rossum7d5b99d1997-11-21 17:12:59 +0000358
359 def filelineno(self):
Guido van Rossum45e2fbc1998-03-26 21:13:24 +0000360 return self._filelineno
Guido van Rossum7d5b99d1997-11-21 17:12:59 +0000361
Georg Brandl67e9fb92006-02-19 13:56:17 +0000362 def fileno(self):
363 if self._file:
364 try:
365 return self._file.fileno()
366 except ValueError:
367 return -1
368 else:
369 return -1
370
Guido van Rossum7d5b99d1997-11-21 17:12:59 +0000371 def isfirstline(self):
Guido van Rossum45e2fbc1998-03-26 21:13:24 +0000372 return self._filelineno == 1
Guido van Rossum7d5b99d1997-11-21 17:12:59 +0000373
374 def isstdin(self):
Guido van Rossum45e2fbc1998-03-26 21:13:24 +0000375 return self._isstdin
Guido van Rossum7d5b99d1997-11-21 17:12:59 +0000376
Georg Brandlc98eeed2006-02-19 14:57:47 +0000377
378def hook_compressed(filename, mode):
379 ext = os.path.splitext(filename)[1]
380 if ext == '.gz':
381 import gzip
382 return gzip.open(filename, mode)
383 elif ext == '.bz2':
384 import bz2
385 return bz2.BZ2File(filename, mode)
386 else:
387 return open(filename, mode)
388
389
390def hook_encoded(encoding):
391 import codecs
392 def openhook(filename, mode):
393 return codecs.open(filename, mode, encoding)
394 return openhook
395
396
Guido van Rossum7d5b99d1997-11-21 17:12:59 +0000397def _test():
398 import getopt
399 inplace = 0
400 backup = 0
401 opts, args = getopt.getopt(sys.argv[1:], "ib:")
402 for o, a in opts:
Guido van Rossum45e2fbc1998-03-26 21:13:24 +0000403 if o == '-i': inplace = 1
404 if o == '-b': backup = a
Guido van Rossum7d5b99d1997-11-21 17:12:59 +0000405 for line in input(args, inplace=inplace, backup=backup):
Guido van Rossum45e2fbc1998-03-26 21:13:24 +0000406 if line[-1:] == '\n': line = line[:-1]
407 if line[-1:] == '\r': line = line[:-1]
408 print "%d: %s[%d]%s %s" % (lineno(), filename(), filelineno(),
409 isfirstline() and "*" or "", line)
Guido van Rossum7d5b99d1997-11-21 17:12:59 +0000410 print "%d: %s[%d]" % (lineno(), filename(), filelineno())
411
412if __name__ == '__main__':
413 _test()