Guido van Rossum | 7d5b99d | 1997-11-21 17:12:59 +0000 | [diff] [blame] | 1 | """Helper class to quickly write a loop over all standard input files. |
| 2 | |
| 3 | Typical use is: |
| 4 | |
| 5 | import fileinput |
| 6 | for line in fileinput.input(): |
| 7 | process(line) |
| 8 | |
| 9 | This iterates over the lines of all files listed in sys.argv[1:], |
| 10 | defaulting to sys.stdin if the list is empty. If a filename is '-' it |
| 11 | is also replaced by sys.stdin. To specify an alternative list of |
| 12 | filenames, pass it as the argument to input(). A single file name is |
| 13 | also allowed. |
| 14 | |
| 15 | Functions filename(), lineno() return the filename and cumulative line |
| 16 | number of the line that has just been read; filelineno() returns its |
| 17 | line number in the current file; isfirstline() returns true iff the |
| 18 | line just read is the first line of its file; isstdin() returns true |
| 19 | iff the line was read from sys.stdin. Function nextfile() closes the |
| 20 | current file so that the next iteration will read the first line from |
| 21 | the next file (if any); lines not read from the file will not count |
| 22 | towards the cumulative line count; the filename is not changed until |
| 23 | after the first line of the next file has been read. Function close() |
| 24 | closes the sequence. |
| 25 | |
| 26 | Before any lines have been read, filename() returns None and both line |
| 27 | numbers are zero; nextfile() has no effect. After all lines have been |
| 28 | read, filename() and the line number functions return the values |
| 29 | pertaining to the last line read; nextfile() has no effect. |
| 30 | |
| 31 | All files are opened in text mode. If an I/O error occurs during |
| 32 | opening or reading a file, the IOError exception is raised. |
| 33 | |
| 34 | If sys.stdin is used more than once, the second and further use will |
| 35 | return no lines, except perhaps for interactive use, or if it has been |
| 36 | explicitly reset (e.g. using sys.stdin.seek(0)). |
| 37 | |
| 38 | Empty files are opened and immediately closed; the only time their |
| 39 | presence in the list of filenames is noticeable at all is when the |
| 40 | last file opened is empty. |
| 41 | |
| 42 | It is possible that the last line of a file doesn't end in a newline |
| 43 | character; otherwise lines are returned including the trailing |
| 44 | newline. |
| 45 | |
| 46 | Class FileInput is the implementation; its methods filename(), |
| 47 | lineno(), fileline(), isfirstline(), isstdin(), nextfile() and close() |
| 48 | correspond to the functions in the module. In addition it has a |
| 49 | readline() method which returns the next input line, and a |
| 50 | __getitem__() method which implements the sequence behavior. The |
| 51 | sequence must be accessed in strictly sequential order; sequence |
| 52 | access and readline() cannot be mixed. |
| 53 | |
| 54 | Optional in-place filtering: if the keyword argument inplace=1 is |
| 55 | passed to input() or to the FileInput constructor, the file is moved |
| 56 | to a backup file and standard output is directed to the input file. |
| 57 | This makes it possible to write a filter that rewrites its input file |
| 58 | in place. If the keyword argument backup=".<some extension>" is also |
| 59 | given, it specifies the extension for the backup file, and the backup |
| 60 | file remains around; by default, the extension is ".bak" and it is |
| 61 | deleted when the output file is closed. In-place filtering is |
| 62 | disabled when standard input is read. XXX The current implementation |
| 63 | does not work for MS-DOS 8+3 filesystems. |
| 64 | |
Guido van Rossum | 4795524 | 2001-01-05 14:44:39 +0000 | [diff] [blame] | 65 | Performance: this module is unfortunately one of the slower ways of |
| 66 | processing large numbers of input lines. Nevertheless, a significant |
| 67 | speed-up has been obtained by using readlines(bufsize) instead of |
| 68 | readline(). A new keyword argument, bufsize=N, is present on the |
| 69 | input() function and the FileInput() class to override the default |
| 70 | buffer size. |
| 71 | |
Guido van Rossum | 7d5b99d | 1997-11-21 17:12:59 +0000 | [diff] [blame] | 72 | XXX Possible additions: |
| 73 | |
| 74 | - optional getopt argument processing |
| 75 | - specify open mode ('r' or 'rb') |
Guido van Rossum | 7d5b99d | 1997-11-21 17:12:59 +0000 | [diff] [blame] | 76 | - fileno() |
| 77 | - isatty() |
| 78 | - read(), read(size), even readlines() |
| 79 | |
| 80 | """ |
| 81 | |
Guido van Rossum | dcb8583 | 1999-10-18 21:41:43 +0000 | [diff] [blame] | 82 | import sys, os, stat |
Guido van Rossum | 7d5b99d | 1997-11-21 17:12:59 +0000 | [diff] [blame] | 83 | |
| 84 | _state = None |
| 85 | |
Guido van Rossum | 4795524 | 2001-01-05 14:44:39 +0000 | [diff] [blame] | 86 | DEFAULT_BUFSIZE = 8*1024 |
| 87 | |
| 88 | def input(files=None, inplace=0, backup="", bufsize=0): |
Guido van Rossum | 7d5b99d | 1997-11-21 17:12:59 +0000 | [diff] [blame] | 89 | global _state |
| 90 | if _state and _state._file: |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 91 | raise RuntimeError, "input() already active" |
Guido van Rossum | 4795524 | 2001-01-05 14:44:39 +0000 | [diff] [blame] | 92 | _state = FileInput(files, inplace, backup, bufsize) |
Guido van Rossum | 7d5b99d | 1997-11-21 17:12:59 +0000 | [diff] [blame] | 93 | return _state |
| 94 | |
| 95 | def close(): |
| 96 | global _state |
| 97 | state = _state |
| 98 | _state = None |
| 99 | if state: |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 100 | state.close() |
Guido van Rossum | 7d5b99d | 1997-11-21 17:12:59 +0000 | [diff] [blame] | 101 | |
| 102 | def nextfile(): |
| 103 | if not _state: |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 104 | raise RuntimeError, "no active input()" |
Guido van Rossum | 7d5b99d | 1997-11-21 17:12:59 +0000 | [diff] [blame] | 105 | return _state.nextfile() |
| 106 | |
| 107 | def filename(): |
| 108 | if not _state: |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 109 | raise RuntimeError, "no active input()" |
Guido van Rossum | 7d5b99d | 1997-11-21 17:12:59 +0000 | [diff] [blame] | 110 | return _state.filename() |
| 111 | |
| 112 | def lineno(): |
| 113 | if not _state: |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 114 | raise RuntimeError, "no active input()" |
Guido van Rossum | 7d5b99d | 1997-11-21 17:12:59 +0000 | [diff] [blame] | 115 | return _state.lineno() |
| 116 | |
| 117 | def filelineno(): |
| 118 | if not _state: |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 119 | raise RuntimeError, "no active input()" |
Guido van Rossum | 7d5b99d | 1997-11-21 17:12:59 +0000 | [diff] [blame] | 120 | return _state.filelineno() |
| 121 | |
| 122 | def isfirstline(): |
| 123 | if not _state: |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 124 | raise RuntimeError, "no active input()" |
Guido van Rossum | 7d5b99d | 1997-11-21 17:12:59 +0000 | [diff] [blame] | 125 | return _state.isfirstline() |
| 126 | |
| 127 | def isstdin(): |
| 128 | if not _state: |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 129 | raise RuntimeError, "no active input()" |
Guido van Rossum | 7d5b99d | 1997-11-21 17:12:59 +0000 | [diff] [blame] | 130 | return _state.isstdin() |
| 131 | |
| 132 | class FileInput: |
| 133 | |
Guido van Rossum | 4795524 | 2001-01-05 14:44:39 +0000 | [diff] [blame] | 134 | def __init__(self, files=None, inplace=0, backup="", bufsize=0): |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 135 | if type(files) == type(''): |
| 136 | files = (files,) |
| 137 | else: |
Guido van Rossum | 2516b39 | 2000-04-10 17:16:12 +0000 | [diff] [blame] | 138 | if files is None: |
| 139 | files = sys.argv[1:] |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 140 | if not files: |
Guido van Rossum | 2516b39 | 2000-04-10 17:16:12 +0000 | [diff] [blame] | 141 | files = ('-',) |
| 142 | else: |
| 143 | files = tuple(files) |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 144 | self._files = files |
| 145 | self._inplace = inplace |
| 146 | self._backup = backup |
Guido van Rossum | 4795524 | 2001-01-05 14:44:39 +0000 | [diff] [blame] | 147 | self._bufsize = bufsize or DEFAULT_BUFSIZE |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 148 | self._savestdout = None |
| 149 | self._output = None |
| 150 | self._filename = None |
| 151 | self._lineno = 0 |
| 152 | self._filelineno = 0 |
| 153 | self._file = None |
| 154 | self._isstdin = 0 |
Guido van Rossum | 0aec9fb | 1998-07-20 15:49:28 +0000 | [diff] [blame] | 155 | self._backupfilename = None |
Guido van Rossum | 4795524 | 2001-01-05 14:44:39 +0000 | [diff] [blame] | 156 | self._buffer = [] |
| 157 | self._bufindex = 0 |
Guido van Rossum | 7d5b99d | 1997-11-21 17:12:59 +0000 | [diff] [blame] | 158 | |
| 159 | def __del__(self): |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 160 | self.close() |
Guido van Rossum | 7d5b99d | 1997-11-21 17:12:59 +0000 | [diff] [blame] | 161 | |
| 162 | def close(self): |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 163 | self.nextfile() |
| 164 | self._files = () |
Guido van Rossum | 7d5b99d | 1997-11-21 17:12:59 +0000 | [diff] [blame] | 165 | |
| 166 | def __getitem__(self, i): |
Guido van Rossum | 4795524 | 2001-01-05 14:44:39 +0000 | [diff] [blame] | 167 | try: |
| 168 | line = self._buffer[self._bufindex] |
| 169 | except IndexError: |
| 170 | pass |
| 171 | else: |
| 172 | self._bufindex += 1 |
| 173 | self._lineno += 1 |
| 174 | self._filelineno += 1 |
| 175 | return line |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 176 | if i != self._lineno: |
| 177 | raise RuntimeError, "accessing lines out of order" |
| 178 | line = self.readline() |
| 179 | if not line: |
| 180 | raise IndexError, "end of input reached" |
| 181 | return line |
Guido van Rossum | 7d5b99d | 1997-11-21 17:12:59 +0000 | [diff] [blame] | 182 | |
| 183 | def nextfile(self): |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 184 | savestdout = self._savestdout |
| 185 | self._savestdout = 0 |
| 186 | if savestdout: |
| 187 | sys.stdout = savestdout |
Guido van Rossum | 7d5b99d | 1997-11-21 17:12:59 +0000 | [diff] [blame] | 188 | |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 189 | output = self._output |
| 190 | self._output = 0 |
| 191 | if output: |
| 192 | output.close() |
Guido van Rossum | 7d5b99d | 1997-11-21 17:12:59 +0000 | [diff] [blame] | 193 | |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 194 | file = self._file |
| 195 | self._file = 0 |
| 196 | if file and not self._isstdin: |
| 197 | file.close() |
Guido van Rossum | 7d5b99d | 1997-11-21 17:12:59 +0000 | [diff] [blame] | 198 | |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 199 | backupfilename = self._backupfilename |
| 200 | self._backupfilename = 0 |
| 201 | if backupfilename and not self._backup: |
| 202 | try: os.unlink(backupfilename) |
| 203 | except: pass |
Guido van Rossum | 7d5b99d | 1997-11-21 17:12:59 +0000 | [diff] [blame] | 204 | |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 205 | self._isstdin = 0 |
Guido van Rossum | 4795524 | 2001-01-05 14:44:39 +0000 | [diff] [blame] | 206 | self._buffer = [] |
| 207 | self._bufindex = 0 |
Guido van Rossum | 7d5b99d | 1997-11-21 17:12:59 +0000 | [diff] [blame] | 208 | |
| 209 | def readline(self): |
Guido van Rossum | 4795524 | 2001-01-05 14:44:39 +0000 | [diff] [blame] | 210 | try: |
| 211 | line = self._buffer[self._bufindex] |
| 212 | except IndexError: |
| 213 | pass |
| 214 | else: |
| 215 | self._bufindex += 1 |
| 216 | self._lineno += 1 |
| 217 | self._filelineno += 1 |
| 218 | return line |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 219 | if not self._file: |
| 220 | if not self._files: |
| 221 | return "" |
| 222 | self._filename = self._files[0] |
| 223 | self._files = self._files[1:] |
| 224 | self._filelineno = 0 |
| 225 | self._file = None |
| 226 | self._isstdin = 0 |
| 227 | self._backupfilename = 0 |
| 228 | if self._filename == '-': |
| 229 | self._filename = '<stdin>' |
| 230 | self._file = sys.stdin |
| 231 | self._isstdin = 1 |
| 232 | else: |
| 233 | if self._inplace: |
| 234 | self._backupfilename = ( |
| 235 | self._filename + (self._backup or ".bak")) |
| 236 | try: os.unlink(self._backupfilename) |
| 237 | except os.error: pass |
Guido van Rossum | dcb8583 | 1999-10-18 21:41:43 +0000 | [diff] [blame] | 238 | # The next few lines may raise IOError |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 239 | os.rename(self._filename, self._backupfilename) |
| 240 | self._file = open(self._backupfilename, "r") |
Guido van Rossum | dcb8583 | 1999-10-18 21:41:43 +0000 | [diff] [blame] | 241 | try: |
| 242 | perm = os.fstat(self._file.fileno())[stat.ST_MODE] |
| 243 | except: |
| 244 | self._output = open(self._filename, "w") |
| 245 | else: |
| 246 | fd = os.open(self._filename, |
| 247 | os.O_CREAT | os.O_WRONLY | os.O_TRUNC, |
| 248 | perm) |
| 249 | self._output = os.fdopen(fd, "w") |
| 250 | try: |
| 251 | os.chmod(self._filename, perm) |
| 252 | except: |
| 253 | pass |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 254 | self._savestdout = sys.stdout |
| 255 | sys.stdout = self._output |
| 256 | else: |
| 257 | # This may raise IOError |
| 258 | self._file = open(self._filename, "r") |
Guido van Rossum | 4795524 | 2001-01-05 14:44:39 +0000 | [diff] [blame] | 259 | self._buffer = self._file.readlines(self._bufsize) |
| 260 | self._bufindex = 0 |
| 261 | if not self._buffer: |
| 262 | self.nextfile() |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 263 | # Recursive call |
| 264 | return self.readline() |
Guido van Rossum | 7d5b99d | 1997-11-21 17:12:59 +0000 | [diff] [blame] | 265 | |
| 266 | def filename(self): |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 267 | return self._filename |
Guido van Rossum | 7d5b99d | 1997-11-21 17:12:59 +0000 | [diff] [blame] | 268 | |
| 269 | def lineno(self): |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 270 | return self._lineno |
Guido van Rossum | 7d5b99d | 1997-11-21 17:12:59 +0000 | [diff] [blame] | 271 | |
| 272 | def filelineno(self): |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 273 | return self._filelineno |
Guido van Rossum | 7d5b99d | 1997-11-21 17:12:59 +0000 | [diff] [blame] | 274 | |
| 275 | def isfirstline(self): |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 276 | return self._filelineno == 1 |
Guido van Rossum | 7d5b99d | 1997-11-21 17:12:59 +0000 | [diff] [blame] | 277 | |
| 278 | def isstdin(self): |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 279 | return self._isstdin |
Guido van Rossum | 7d5b99d | 1997-11-21 17:12:59 +0000 | [diff] [blame] | 280 | |
| 281 | def _test(): |
| 282 | import getopt |
| 283 | inplace = 0 |
| 284 | backup = 0 |
| 285 | opts, args = getopt.getopt(sys.argv[1:], "ib:") |
| 286 | for o, a in opts: |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 287 | if o == '-i': inplace = 1 |
| 288 | if o == '-b': backup = a |
Guido van Rossum | 7d5b99d | 1997-11-21 17:12:59 +0000 | [diff] [blame] | 289 | for line in input(args, inplace=inplace, backup=backup): |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 290 | if line[-1:] == '\n': line = line[:-1] |
| 291 | if line[-1:] == '\r': line = line[:-1] |
| 292 | print "%d: %s[%d]%s %s" % (lineno(), filename(), filelineno(), |
| 293 | isfirstline() and "*" or "", line) |
Guido van Rossum | 7d5b99d | 1997-11-21 17:12:59 +0000 | [diff] [blame] | 294 | print "%d: %s[%d]" % (lineno(), filename(), filelineno()) |
| 295 | |
| 296 | if __name__ == '__main__': |
| 297 | _test() |