blob: e63a0c6258aea1b1317d2293c16ff2a9b5eec5e0 [file] [log] [blame]
Guido van Rossum0612d842000-03-10 23:20:43 +00001""" codecs -- Python Codec Registry, API and helpers.
2
3
4Written by Marc-Andre Lemburg (mal@lemburg.com).
5
6(c) Copyright CNRI, All Rights Reserved. NO WARRANTY.
7
8"""#"
9
Georg Brandl1a3284e2007-12-02 09:40:06 +000010import builtins, sys
Guido van Rossum0612d842000-03-10 23:20:43 +000011
12### Registry and builtin stateless codec functions
13
Guido van Rossumb95de4f2000-03-31 17:25:23 +000014try:
15 from _codecs import *
Guido van Rossumb940e112007-01-10 16:19:56 +000016except ImportError as why:
Thomas Wouters49fd7fa2006-04-21 10:40:58 +000017 raise SystemError('Failed to load the builtin codecs: %s' % why)
Guido van Rossum0612d842000-03-10 23:20:43 +000018
Tim Peters30324a72001-05-15 17:19:16 +000019__all__ = ["register", "lookup", "open", "EncodedFile", "BOM", "BOM_BE",
Walter Dörwald474458d2002-06-04 15:16:29 +000020 "BOM_LE", "BOM32_BE", "BOM32_LE", "BOM64_BE", "BOM64_LE",
21 "BOM_UTF8", "BOM_UTF16", "BOM_UTF16_LE", "BOM_UTF16_BE",
Walter Dörwald3aeb6322002-09-02 13:14:32 +000022 "BOM_UTF32", "BOM_UTF32_LE", "BOM_UTF32_BE",
23 "strict_errors", "ignore_errors", "replace_errors",
24 "xmlcharrefreplace_errors",
25 "register_error", "lookup_error"]
Skip Montanaroe99d5ea2001-01-20 19:54:20 +000026
Guido van Rossum0612d842000-03-10 23:20:43 +000027### Constants
28
29#
Walter Dörwald474458d2002-06-04 15:16:29 +000030# Byte Order Mark (BOM = ZERO WIDTH NO-BREAK SPACE = U+FEFF)
31# and its possible byte string values
32# for UTF8/UTF16/UTF32 output and little/big endian machines
Guido van Rossum0612d842000-03-10 23:20:43 +000033#
Guido van Rossum0612d842000-03-10 23:20:43 +000034
Walter Dörwald474458d2002-06-04 15:16:29 +000035# UTF-8
Walter Dörwaldca8a8d02007-05-04 13:05:09 +000036BOM_UTF8 = b'\xef\xbb\xbf'
Walter Dörwald474458d2002-06-04 15:16:29 +000037
38# UTF-16, little endian
Walter Dörwaldca8a8d02007-05-04 13:05:09 +000039BOM_LE = BOM_UTF16_LE = b'\xff\xfe'
Walter Dörwald474458d2002-06-04 15:16:29 +000040
41# UTF-16, big endian
Walter Dörwaldca8a8d02007-05-04 13:05:09 +000042BOM_BE = BOM_UTF16_BE = b'\xfe\xff'
Walter Dörwald474458d2002-06-04 15:16:29 +000043
44# UTF-32, little endian
Walter Dörwaldca8a8d02007-05-04 13:05:09 +000045BOM_UTF32_LE = b'\xff\xfe\x00\x00'
Walter Dörwald474458d2002-06-04 15:16:29 +000046
47# UTF-32, big endian
Walter Dörwaldca8a8d02007-05-04 13:05:09 +000048BOM_UTF32_BE = b'\x00\x00\xfe\xff'
Walter Dörwald474458d2002-06-04 15:16:29 +000049
Marc-André Lemburgb28de0d2002-12-12 17:37:50 +000050if sys.byteorder == 'little':
Walter Dörwald474458d2002-06-04 15:16:29 +000051
Marc-André Lemburgb28de0d2002-12-12 17:37:50 +000052 # UTF-16, native endianness
53 BOM = BOM_UTF16 = BOM_UTF16_LE
54
55 # UTF-32, native endianness
56 BOM_UTF32 = BOM_UTF32_LE
57
58else:
59
60 # UTF-16, native endianness
61 BOM = BOM_UTF16 = BOM_UTF16_BE
62
63 # UTF-32, native endianness
64 BOM_UTF32 = BOM_UTF32_BE
Walter Dörwald474458d2002-06-04 15:16:29 +000065
66# Old broken names (don't use in new code)
67BOM32_LE = BOM_UTF16_LE
68BOM32_BE = BOM_UTF16_BE
69BOM64_LE = BOM_UTF32_LE
70BOM64_BE = BOM_UTF32_BE
Guido van Rossum0612d842000-03-10 23:20:43 +000071
72
73### Codec base classes (defining the API)
74
Thomas Woutersa9773292006-04-21 09:43:23 +000075class CodecInfo(tuple):
76
77 def __new__(cls, encode, decode, streamreader=None, streamwriter=None,
78 incrementalencoder=None, incrementaldecoder=None, name=None):
79 self = tuple.__new__(cls, (encode, decode, streamreader, streamwriter))
80 self.name = name
81 self.encode = encode
82 self.decode = decode
83 self.incrementalencoder = incrementalencoder
84 self.incrementaldecoder = incrementaldecoder
85 self.streamwriter = streamwriter
86 self.streamreader = streamreader
87 return self
88
89 def __repr__(self):
Walter Dörwald3abcb012007-04-16 22:10:50 +000090 return "<%s.%s object for encoding %s at 0x%x>" % \
91 (self.__class__.__module__, self.__class__.__name__,
92 self.name, id(self))
Thomas Woutersa9773292006-04-21 09:43:23 +000093
Guido van Rossum0612d842000-03-10 23:20:43 +000094class Codec:
95
96 """ Defines the interface for stateless encoders/decoders.
97
Walter Dörwald7f82f792002-11-19 21:42:53 +000098 The .encode()/.decode() methods may use different error
Guido van Rossum0612d842000-03-10 23:20:43 +000099 handling schemes by providing the errors argument. These
Walter Dörwald7f82f792002-11-19 21:42:53 +0000100 string values are predefined:
Guido van Rossum0612d842000-03-10 23:20:43 +0000101
Guido van Rossumd8855fd2000-03-24 22:14:19 +0000102 'strict' - raise a ValueError error (or a subclass)
Guido van Rossum0612d842000-03-10 23:20:43 +0000103 'ignore' - ignore the character and continue with the next
104 'replace' - replace with a suitable replacement character;
105 Python will use the official U+FFFD REPLACEMENT
Walter Dörwald7f82f792002-11-19 21:42:53 +0000106 CHARACTER for the builtin Unicode codecs on
107 decoding and '?' on encoding.
108 'xmlcharrefreplace' - Replace with the appropriate XML
109 character reference (only for encoding).
110 'backslashreplace' - Replace with backslashed escape sequences
111 (only for encoding).
112
113 The set of allowed values can be extended via register_error.
Guido van Rossum0612d842000-03-10 23:20:43 +0000114
115 """
Tim Peters30324a72001-05-15 17:19:16 +0000116 def encode(self, input, errors='strict'):
Guido van Rossum1c89b0e2000-04-11 15:41:38 +0000117
Fred Drake3e74c0d2000-03-17 15:40:35 +0000118 """ Encodes the object input and returns a tuple (output
Guido van Rossum0612d842000-03-10 23:20:43 +0000119 object, length consumed).
120
121 errors defines the error handling to apply. It defaults to
122 'strict' handling.
123
124 The method may not store state in the Codec instance. Use
125 StreamCodec for codecs which have to keep state in order to
126 make encoding/decoding efficient.
127
128 The encoder must be able to handle zero length input and
129 return an empty object of the output object type in this
130 situation.
131
132 """
133 raise NotImplementedError
134
Tim Peters30324a72001-05-15 17:19:16 +0000135 def decode(self, input, errors='strict'):
Guido van Rossum0612d842000-03-10 23:20:43 +0000136
137 """ Decodes the object input and returns a tuple (output
138 object, length consumed).
139
140 input must be an object which provides the bf_getreadbuf
141 buffer slot. Python strings, buffer objects and memory
142 mapped files are examples of objects providing this slot.
Guido van Rossum1c89b0e2000-04-11 15:41:38 +0000143
Guido van Rossum0612d842000-03-10 23:20:43 +0000144 errors defines the error handling to apply. It defaults to
145 'strict' handling.
146
147 The method may not store state in the Codec instance. Use
148 StreamCodec for codecs which have to keep state in order to
149 make encoding/decoding efficient.
150
151 The decoder must be able to handle zero length input and
152 return an empty object of the output object type in this
153 situation.
154
Guido van Rossum1c89b0e2000-04-11 15:41:38 +0000155 """
Guido van Rossum0612d842000-03-10 23:20:43 +0000156 raise NotImplementedError
157
Thomas Woutersa9773292006-04-21 09:43:23 +0000158class IncrementalEncoder(object):
159 """
Walter Dörwald3abcb012007-04-16 22:10:50 +0000160 An IncrementalEncoder encodes an input in multiple steps. The input can
161 be passed piece by piece to the encode() method. The IncrementalEncoder
162 remembers the state of the encoding process between calls to encode().
Thomas Woutersa9773292006-04-21 09:43:23 +0000163 """
164 def __init__(self, errors='strict'):
165 """
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000166 Creates an IncrementalEncoder instance.
Thomas Woutersa9773292006-04-21 09:43:23 +0000167
168 The IncrementalEncoder may use different error handling schemes by
169 providing the errors keyword argument. See the module docstring
170 for a list of possible values.
171 """
172 self.errors = errors
173 self.buffer = ""
174
175 def encode(self, input, final=False):
176 """
177 Encodes input and returns the resulting object.
178 """
179 raise NotImplementedError
180
181 def reset(self):
182 """
183 Resets the encoder to the initial state.
184 """
185
Walter Dörwald3abcb012007-04-16 22:10:50 +0000186 def getstate(self):
187 """
188 Return the current state of the encoder.
189 """
190 return 0
191
192 def setstate(self, state):
193 """
194 Set the current state of the encoder. state must have been
195 returned by getstate().
196 """
197
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000198class BufferedIncrementalEncoder(IncrementalEncoder):
199 """
200 This subclass of IncrementalEncoder can be used as the baseclass for an
201 incremental encoder if the encoder must keep some of the output in a
202 buffer between calls to encode().
203 """
204 def __init__(self, errors='strict'):
205 IncrementalEncoder.__init__(self, errors)
Walter Dörwald3abcb012007-04-16 22:10:50 +0000206 # unencoded input that is kept between calls to encode()
207 self.buffer = ""
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000208
209 def _buffer_encode(self, input, errors, final):
210 # Overwrite this method in subclasses: It must encode input
211 # and return an (output, length consumed) tuple
212 raise NotImplementedError
213
214 def encode(self, input, final=False):
215 # encode input (taking the buffer into account)
216 data = self.buffer + input
217 (result, consumed) = self._buffer_encode(data, self.errors, final)
218 # keep unencoded input until the next call
219 self.buffer = data[consumed:]
220 return result
221
222 def reset(self):
223 IncrementalEncoder.reset(self)
224 self.buffer = ""
225
Walter Dörwald3abcb012007-04-16 22:10:50 +0000226 def getstate(self):
227 return self.buffer or 0
228
229 def setstate(self, state):
230 self.buffer = state or ""
231
Thomas Woutersa9773292006-04-21 09:43:23 +0000232class IncrementalDecoder(object):
233 """
Walter Dörwald3abcb012007-04-16 22:10:50 +0000234 An IncrementalDecoder decodes an input in multiple steps. The input can
235 be passed piece by piece to the decode() method. The IncrementalDecoder
Thomas Woutersa9773292006-04-21 09:43:23 +0000236 remembers the state of the decoding process between calls to decode().
237 """
238 def __init__(self, errors='strict'):
239 """
Ka-Ping Yeef44c7e82008-03-18 04:51:32 +0000240 Create a IncrementalDecoder instance.
Thomas Woutersa9773292006-04-21 09:43:23 +0000241
242 The IncrementalDecoder may use different error handling schemes by
243 providing the errors keyword argument. See the module docstring
244 for a list of possible values.
245 """
246 self.errors = errors
247
248 def decode(self, input, final=False):
249 """
Ka-Ping Yeef44c7e82008-03-18 04:51:32 +0000250 Decode input and returns the resulting object.
Thomas Woutersa9773292006-04-21 09:43:23 +0000251 """
252 raise NotImplementedError
253
254 def reset(self):
255 """
Ka-Ping Yeef44c7e82008-03-18 04:51:32 +0000256 Reset the decoder to the initial state.
Thomas Woutersa9773292006-04-21 09:43:23 +0000257 """
258
Walter Dörwald3abcb012007-04-16 22:10:50 +0000259 def getstate(self):
260 """
Ka-Ping Yeef44c7e82008-03-18 04:51:32 +0000261 Return the current state of the decoder.
262
263 This must be a (buffered_input, additional_state_info) tuple.
264 buffered_input must be a bytes object containing bytes that
265 were passed to decode() that have not yet been converted.
266 additional_state_info must be a non-negative integer
267 representing the state of the decoder WITHOUT yet having
268 processed the contents of buffered_input. In the initial state
269 and after reset(), getstate() must return (b"", 0).
Walter Dörwald3abcb012007-04-16 22:10:50 +0000270 """
Walter Dörwaldca8a8d02007-05-04 13:05:09 +0000271 return (b"", 0)
Walter Dörwald3abcb012007-04-16 22:10:50 +0000272
273 def setstate(self, state):
274 """
Ka-Ping Yeef44c7e82008-03-18 04:51:32 +0000275 Set the current state of the decoder.
276
277 state must have been returned by getstate(). The effect of
278 setstate((b"", 0)) must be equivalent to reset().
Walter Dörwald3abcb012007-04-16 22:10:50 +0000279 """
280
Thomas Woutersa9773292006-04-21 09:43:23 +0000281class BufferedIncrementalDecoder(IncrementalDecoder):
282 """
283 This subclass of IncrementalDecoder can be used as the baseclass for an
Walter Dörwald3abcb012007-04-16 22:10:50 +0000284 incremental decoder if the decoder must be able to handle incomplete
285 byte sequences.
Thomas Woutersa9773292006-04-21 09:43:23 +0000286 """
287 def __init__(self, errors='strict'):
288 IncrementalDecoder.__init__(self, errors)
Walter Dörwald3abcb012007-04-16 22:10:50 +0000289 # undecoded input that is kept between calls to decode()
Walter Dörwaldca8a8d02007-05-04 13:05:09 +0000290 self.buffer = b""
Thomas Woutersa9773292006-04-21 09:43:23 +0000291
292 def _buffer_decode(self, input, errors, final):
293 # Overwrite this method in subclasses: It must decode input
294 # and return an (output, length consumed) tuple
295 raise NotImplementedError
296
297 def decode(self, input, final=False):
298 # decode input (taking the buffer into account)
299 data = self.buffer + input
300 (result, consumed) = self._buffer_decode(data, self.errors, final)
301 # keep undecoded input until the next call
302 self.buffer = data[consumed:]
303 return result
304
305 def reset(self):
306 IncrementalDecoder.reset(self)
Walter Dörwaldca8a8d02007-05-04 13:05:09 +0000307 self.buffer = b""
Thomas Woutersa9773292006-04-21 09:43:23 +0000308
Walter Dörwald3abcb012007-04-16 22:10:50 +0000309 def getstate(self):
310 # additional state info is always 0
311 return (self.buffer, 0)
312
313 def setstate(self, state):
314 # ignore additional state info
315 self.buffer = state[0]
316
Guido van Rossum0612d842000-03-10 23:20:43 +0000317#
318# The StreamWriter and StreamReader class provide generic working
Andrew M. Kuchling97c56352001-09-18 20:29:48 +0000319# interfaces which can be used to implement new encoding submodules
Guido van Rossum0612d842000-03-10 23:20:43 +0000320# very easily. See encodings/utf_8.py for an example on how this is
321# done.
Guido van Rossum1c89b0e2000-04-11 15:41:38 +0000322#
Guido van Rossum0612d842000-03-10 23:20:43 +0000323
324class StreamWriter(Codec):
325
Tim Peters30324a72001-05-15 17:19:16 +0000326 def __init__(self, stream, errors='strict'):
Guido van Rossum0612d842000-03-10 23:20:43 +0000327
328 """ Creates a StreamWriter instance.
329
330 stream must be a file-like object open for writing
331 (binary) data.
332
Walter Dörwald7f82f792002-11-19 21:42:53 +0000333 The StreamWriter may use different error handling
Guido van Rossum0612d842000-03-10 23:20:43 +0000334 schemes by providing the errors keyword argument. These
Walter Dörwald7f82f792002-11-19 21:42:53 +0000335 parameters are predefined:
Guido van Rossum0612d842000-03-10 23:20:43 +0000336
337 'strict' - raise a ValueError (or a subclass)
338 'ignore' - ignore the character and continue with the next
339 'replace'- replace with a suitable replacement character
Walter Dörwald7f82f792002-11-19 21:42:53 +0000340 'xmlcharrefreplace' - Replace with the appropriate XML
341 character reference.
342 'backslashreplace' - Replace with backslashed escape
343 sequences (only for encoding).
Guido van Rossum0612d842000-03-10 23:20:43 +0000344
Walter Dörwald7f82f792002-11-19 21:42:53 +0000345 The set of allowed parameter values can be extended via
346 register_error.
Guido van Rossum0612d842000-03-10 23:20:43 +0000347 """
348 self.stream = stream
349 self.errors = errors
350
Guido van Rossuma3277132000-04-11 15:37:43 +0000351 def write(self, object):
Guido van Rossum0612d842000-03-10 23:20:43 +0000352
353 """ Writes the object's contents encoded to self.stream.
354 """
Tim Peters30324a72001-05-15 17:19:16 +0000355 data, consumed = self.encode(object, self.errors)
Guido van Rossum0612d842000-03-10 23:20:43 +0000356 self.stream.write(data)
357
Guido van Rossuma3277132000-04-11 15:37:43 +0000358 def writelines(self, list):
359
360 """ Writes the concatenated list of strings to the stream
361 using .write().
362 """
363 self.write(''.join(list))
Guido van Rossum1c89b0e2000-04-11 15:41:38 +0000364
Guido van Rossum0612d842000-03-10 23:20:43 +0000365 def reset(self):
366
367 """ Flushes and resets the codec buffers used for keeping state.
368
369 Calling this method should ensure that the data on the
370 output is put into a clean state, that allows appending
371 of new fresh data without having to rescan the whole
372 stream to recover state.
373
374 """
375 pass
376
Victor Stinnera92ad7e2010-05-22 16:59:09 +0000377 def seek(self, offset, whence=0):
378 self.stream.seek(offset, whence)
379 if whence == 0 and offset == 0:
380 self.reset()
381
Tim Peters30324a72001-05-15 17:19:16 +0000382 def __getattr__(self, name,
Guido van Rossum0612d842000-03-10 23:20:43 +0000383 getattr=getattr):
384
385 """ Inherit all other methods from the underlying stream.
386 """
Tim Peters30324a72001-05-15 17:19:16 +0000387 return getattr(self.stream, name)
Guido van Rossum0612d842000-03-10 23:20:43 +0000388
Thomas Wouters89f507f2006-12-13 04:49:30 +0000389 def __enter__(self):
390 return self
391
392 def __exit__(self, type, value, tb):
393 self.stream.close()
394
Guido van Rossum0612d842000-03-10 23:20:43 +0000395###
396
397class StreamReader(Codec):
398
Georg Brandl02524622010-12-02 18:06:51 +0000399 charbuffertype = str
400
Tim Peters30324a72001-05-15 17:19:16 +0000401 def __init__(self, stream, errors='strict'):
Guido van Rossum0612d842000-03-10 23:20:43 +0000402
403 """ Creates a StreamReader instance.
404
405 stream must be a file-like object open for reading
406 (binary) data.
407
Walter Dörwald7f82f792002-11-19 21:42:53 +0000408 The StreamReader may use different error handling
Guido van Rossum0612d842000-03-10 23:20:43 +0000409 schemes by providing the errors keyword argument. These
Walter Dörwald7f82f792002-11-19 21:42:53 +0000410 parameters are predefined:
Guido van Rossum0612d842000-03-10 23:20:43 +0000411
412 'strict' - raise a ValueError (or a subclass)
413 'ignore' - ignore the character and continue with the next
414 'replace'- replace with a suitable replacement character;
415
Walter Dörwald7f82f792002-11-19 21:42:53 +0000416 The set of allowed parameter values can be extended via
417 register_error.
Guido van Rossum0612d842000-03-10 23:20:43 +0000418 """
419 self.stream = stream
420 self.errors = errors
Walter Dörwaldca8a8d02007-05-04 13:05:09 +0000421 self.bytebuffer = b""
Georg Brandl02524622010-12-02 18:06:51 +0000422 self._empty_charbuffer = self.charbuffertype()
423 self.charbuffer = self._empty_charbuffer
Martin v. Löwis4ed67382005-09-18 08:34:39 +0000424 self.linebuffer = None
Guido van Rossum0612d842000-03-10 23:20:43 +0000425
Walter Dörwald69652032004-09-07 20:24:22 +0000426 def decode(self, input, errors='strict'):
427 raise NotImplementedError
428
Martin v. Löwis56066d22005-08-24 07:38:12 +0000429 def read(self, size=-1, chars=-1, firstline=False):
Guido van Rossum0612d842000-03-10 23:20:43 +0000430
431 """ Decodes data from the stream self.stream and returns the
432 resulting object.
433
Walter Dörwald69652032004-09-07 20:24:22 +0000434 chars indicates the number of characters to read from the
435 stream. read() will never return more than chars
436 characters, but it might return less, if there are not enough
437 characters available.
438
Guido van Rossum0612d842000-03-10 23:20:43 +0000439 size indicates the approximate maximum number of bytes to
440 read from the stream for decoding purposes. The decoder
441 can modify this setting as appropriate. The default value
442 -1 indicates to read and decode as much as possible. size
443 is intended to prevent having to decode huge files in one
444 step.
445
Martin v. Löwis56066d22005-08-24 07:38:12 +0000446 If firstline is true, and a UnicodeDecodeError happens
447 after the first line terminator in the input only the first line
448 will be returned, the rest of the input will be kept until the
449 next call to read().
450
Guido van Rossum0612d842000-03-10 23:20:43 +0000451 The method should use a greedy read strategy meaning that
452 it should read as much data as is allowed within the
453 definition of the encoding and the given size, e.g. if
454 optional encoding endings or state markers are available
455 on the stream, these should be read too.
Guido van Rossum0612d842000-03-10 23:20:43 +0000456 """
Martin v. Löwis4ed67382005-09-18 08:34:39 +0000457 # If we have lines cached, first merge them back into characters
458 if self.linebuffer:
Georg Brandl02524622010-12-02 18:06:51 +0000459 self.charbuffer = self._empty_charbuffer.join(self.linebuffer)
Martin v. Löwis4ed67382005-09-18 08:34:39 +0000460 self.linebuffer = None
Tim Peters536cf992005-12-25 23:18:31 +0000461
Walter Dörwald69652032004-09-07 20:24:22 +0000462 # read until we get the required number of characters (if available)
Walter Dörwald69652032004-09-07 20:24:22 +0000463 while True:
464 # can the request can be satisfied from the character buffer?
465 if chars < 0:
Walter Dörwaldca199432006-03-06 22:39:12 +0000466 if size < 0:
467 if self.charbuffer:
468 break
469 elif len(self.charbuffer) >= size:
Walter Dörwalde57d7b12004-12-21 22:24:00 +0000470 break
Guido van Rossum0612d842000-03-10 23:20:43 +0000471 else:
Walter Dörwald69652032004-09-07 20:24:22 +0000472 if len(self.charbuffer) >= chars:
Walter Dörwald69652032004-09-07 20:24:22 +0000473 break
474 # we need more data
475 if size < 0:
476 newdata = self.stream.read()
477 else:
478 newdata = self.stream.read(size)
Walter Dörwalde57d7b12004-12-21 22:24:00 +0000479 # decode bytes (those remaining from the last call included)
Walter Dörwald69652032004-09-07 20:24:22 +0000480 data = self.bytebuffer + newdata
Martin v. Löwis56066d22005-08-24 07:38:12 +0000481 try:
482 newchars, decodedbytes = self.decode(data, self.errors)
Guido van Rossumb940e112007-01-10 16:19:56 +0000483 except UnicodeDecodeError as exc:
Martin v. Löwis56066d22005-08-24 07:38:12 +0000484 if firstline:
Walter Dörwald3abcb012007-04-16 22:10:50 +0000485 newchars, decodedbytes = \
486 self.decode(data[:exc.start], self.errors)
Ezio Melottid8b509b2011-09-28 17:37:55 +0300487 lines = newchars.splitlines(keepends=True)
Martin v. Löwis56066d22005-08-24 07:38:12 +0000488 if len(lines)<=1:
489 raise
490 else:
491 raise
Walter Dörwald69652032004-09-07 20:24:22 +0000492 # keep undecoded bytes until the next call
493 self.bytebuffer = data[decodedbytes:]
494 # put new characters in the character buffer
Walter Dörwalde57d7b12004-12-21 22:24:00 +0000495 self.charbuffer += newchars
Walter Dörwald69652032004-09-07 20:24:22 +0000496 # there was no data available
497 if not newdata:
Walter Dörwalde57d7b12004-12-21 22:24:00 +0000498 break
499 if chars < 0:
500 # Return everything we've got
501 result = self.charbuffer
Georg Brandl02524622010-12-02 18:06:51 +0000502 self.charbuffer = self._empty_charbuffer
Walter Dörwalde57d7b12004-12-21 22:24:00 +0000503 else:
504 # Return the first chars characters
505 result = self.charbuffer[:chars]
506 self.charbuffer = self.charbuffer[chars:]
Walter Dörwald69652032004-09-07 20:24:22 +0000507 return result
Guido van Rossum0612d842000-03-10 23:20:43 +0000508
Walter Dörwald69652032004-09-07 20:24:22 +0000509 def readline(self, size=None, keepends=True):
Guido van Rossuma3277132000-04-11 15:37:43 +0000510
511 """ Read one line from the input stream and return the
512 decoded data.
513
Walter Dörwald69652032004-09-07 20:24:22 +0000514 size, if given, is passed as size argument to the
515 read() method.
Guido van Rossum1c89b0e2000-04-11 15:41:38 +0000516
Guido van Rossuma3277132000-04-11 15:37:43 +0000517 """
Martin v. Löwis4ed67382005-09-18 08:34:39 +0000518 # If we have lines cached from an earlier read, return
519 # them unconditionally
520 if self.linebuffer:
521 line = self.linebuffer[0]
522 del self.linebuffer[0]
523 if len(self.linebuffer) == 1:
524 # revert to charbuffer mode; we might need more data
525 # next time
526 self.charbuffer = self.linebuffer[0]
527 self.linebuffer = None
528 if not keepends:
Ezio Melottid8b509b2011-09-28 17:37:55 +0300529 line = line.splitlines(keepends=False)[0]
Martin v. Löwis4ed67382005-09-18 08:34:39 +0000530 return line
Tim Peters536cf992005-12-25 23:18:31 +0000531
Walter Dörwalde57d7b12004-12-21 22:24:00 +0000532 readsize = size or 72
Georg Brandl02524622010-12-02 18:06:51 +0000533 line = self._empty_charbuffer
Walter Dörwalde57d7b12004-12-21 22:24:00 +0000534 # If size is given, we call read() only once
Walter Dörwald69652032004-09-07 20:24:22 +0000535 while True:
Martin v. Löwis56066d22005-08-24 07:38:12 +0000536 data = self.read(readsize, firstline=True)
Walter Dörwalde57d7b12004-12-21 22:24:00 +0000537 if data:
Walter Dörwalda4eb2d52005-04-21 21:42:35 +0000538 # If we're at a "\r" read one extra character (which might
539 # be a "\n") to get a proper line ending. If the stream is
Walter Dörwaldbc8e6422005-04-21 21:32:03 +0000540 # temporarily exhausted we return the wrong line ending.
Georg Brandl02524622010-12-02 18:06:51 +0000541 if (isinstance(data, str) and data.endswith("\r")) or \
542 (isinstance(data, bytes) and data.endswith(b"\r")):
Walter Dörwald7a6dc132005-04-04 21:38:47 +0000543 data += self.read(size=1, chars=1)
Walter Dörwald7a6dc132005-04-04 21:38:47 +0000544
Walter Dörwald69652032004-09-07 20:24:22 +0000545 line += data
Ezio Melottid8b509b2011-09-28 17:37:55 +0300546 lines = line.splitlines(keepends=True)
Walter Dörwalde57d7b12004-12-21 22:24:00 +0000547 if lines:
Martin v. Löwis4ed67382005-09-18 08:34:39 +0000548 if len(lines) > 1:
549 # More than one line result; the first line is a full line
550 # to return
551 line = lines[0]
552 del lines[0]
553 if len(lines) > 1:
554 # cache the remaining lines
555 lines[-1] += self.charbuffer
556 self.linebuffer = lines
557 self.charbuffer = None
558 else:
559 # only one remaining line, put it back into charbuffer
560 self.charbuffer = lines[0] + self.charbuffer
561 if not keepends:
Ezio Melottid8b509b2011-09-28 17:37:55 +0300562 line = line.splitlines(keepends=False)[0]
Martin v. Löwis4ed67382005-09-18 08:34:39 +0000563 break
Walter Dörwalde57d7b12004-12-21 22:24:00 +0000564 line0withend = lines[0]
Ezio Melottid8b509b2011-09-28 17:37:55 +0300565 line0withoutend = lines[0].splitlines(keepends=False)[0]
Walter Dörwalde57d7b12004-12-21 22:24:00 +0000566 if line0withend != line0withoutend: # We really have a line end
567 # Put the rest back together and keep it until the next call
Georg Brandl02524622010-12-02 18:06:51 +0000568 self.charbuffer = self._empty_charbuffer.join(lines[1:]) + \
569 self.charbuffer
Walter Dörwalde57d7b12004-12-21 22:24:00 +0000570 if keepends:
571 line = line0withend
572 else:
573 line = line0withoutend
Walter Dörwald9fa09462005-01-10 12:01:39 +0000574 break
Walter Dörwalde57d7b12004-12-21 22:24:00 +0000575 # we didn't get anything or this was our only try
Walter Dörwald9fa09462005-01-10 12:01:39 +0000576 if not data or size is not None:
Walter Dörwalde57d7b12004-12-21 22:24:00 +0000577 if line and not keepends:
Ezio Melottid8b509b2011-09-28 17:37:55 +0300578 line = line.splitlines(keepends=False)[0]
Walter Dörwalde57d7b12004-12-21 22:24:00 +0000579 break
Georg Brandl02524622010-12-02 18:06:51 +0000580 if readsize < 8000:
Walter Dörwalde57d7b12004-12-21 22:24:00 +0000581 readsize *= 2
582 return line
Guido van Rossum1c89b0e2000-04-11 15:41:38 +0000583
Walter Dörwald69652032004-09-07 20:24:22 +0000584 def readlines(self, sizehint=None, keepends=True):
Guido van Rossuma3277132000-04-11 15:37:43 +0000585
586 """ Read all lines available on the input stream
587 and return them as list of lines.
588
589 Line breaks are implemented using the codec's decoder
590 method and are included in the list entries.
Guido van Rossum1c89b0e2000-04-11 15:41:38 +0000591
Marc-André Lemburgd5948492004-02-26 15:22:17 +0000592 sizehint, if given, is ignored since there is no efficient
593 way to finding the true end-of-line.
Guido van Rossuma3277132000-04-11 15:37:43 +0000594
595 """
Walter Dörwald69652032004-09-07 20:24:22 +0000596 data = self.read()
Hye-Shik Changaf5c7cf2004-10-17 23:51:21 +0000597 return data.splitlines(keepends)
Guido van Rossum0612d842000-03-10 23:20:43 +0000598
599 def reset(self):
600
601 """ Resets the codec buffers used for keeping state.
602
603 Note that no stream repositioning should take place.
Thomas Wouters7e474022000-07-16 12:04:32 +0000604 This method is primarily intended to be able to recover
Guido van Rossum0612d842000-03-10 23:20:43 +0000605 from decoding errors.
606
607 """
Walter Dörwaldca8a8d02007-05-04 13:05:09 +0000608 self.bytebuffer = b""
Georg Brandl02524622010-12-02 18:06:51 +0000609 self.charbuffer = self._empty_charbuffer
Martin v. Löwis4ed67382005-09-18 08:34:39 +0000610 self.linebuffer = None
Walter Dörwald729c31f2005-03-14 19:06:30 +0000611
Walter Dörwald71fd90d2005-03-14 19:25:41 +0000612 def seek(self, offset, whence=0):
Walter Dörwald729c31f2005-03-14 19:06:30 +0000613 """ Set the input stream's current position.
614
615 Resets the codec buffers used for keeping state.
616 """
Walter Dörwald729c31f2005-03-14 19:06:30 +0000617 self.stream.seek(offset, whence)
Victor Stinnera92ad7e2010-05-22 16:59:09 +0000618 self.reset()
Guido van Rossum0612d842000-03-10 23:20:43 +0000619
Georg Brandla18af4e2007-04-21 15:47:16 +0000620 def __next__(self):
Walter Dörwald4dbf1922002-11-06 16:53:44 +0000621
622 """ Return the next decoded line from the input stream."""
623 line = self.readline()
624 if line:
625 return line
626 raise StopIteration
627
628 def __iter__(self):
629 return self
630
Tim Peters30324a72001-05-15 17:19:16 +0000631 def __getattr__(self, name,
Guido van Rossum0612d842000-03-10 23:20:43 +0000632 getattr=getattr):
633
634 """ Inherit all other methods from the underlying stream.
635 """
Tim Peters30324a72001-05-15 17:19:16 +0000636 return getattr(self.stream, name)
Guido van Rossum0612d842000-03-10 23:20:43 +0000637
Thomas Wouters89f507f2006-12-13 04:49:30 +0000638 def __enter__(self):
639 return self
640
641 def __exit__(self, type, value, tb):
642 self.stream.close()
643
Guido van Rossum0612d842000-03-10 23:20:43 +0000644###
645
646class StreamReaderWriter:
647
Fred Drake49fd1072000-04-13 14:11:21 +0000648 """ StreamReaderWriter instances allow wrapping streams which
649 work in both read and write modes.
650
651 The design is such that one can use the factory functions
Thomas Wouters7e474022000-07-16 12:04:32 +0000652 returned by the codec.lookup() function to construct the
Fred Drake49fd1072000-04-13 14:11:21 +0000653 instance.
654
655 """
Guido van Rossuma3277132000-04-11 15:37:43 +0000656 # Optional attributes set by the file wrappers below
657 encoding = 'unknown'
658
Tim Peters30324a72001-05-15 17:19:16 +0000659 def __init__(self, stream, Reader, Writer, errors='strict'):
Guido van Rossum0612d842000-03-10 23:20:43 +0000660
661 """ Creates a StreamReaderWriter instance.
662
663 stream must be a Stream-like object.
664
665 Reader, Writer must be factory functions or classes
666 providing the StreamReader, StreamWriter interface resp.
667
668 Error handling is done in the same way as defined for the
669 StreamWriter/Readers.
670
671 """
672 self.stream = stream
673 self.reader = Reader(stream, errors)
674 self.writer = Writer(stream, errors)
675 self.errors = errors
676
Tim Peters30324a72001-05-15 17:19:16 +0000677 def read(self, size=-1):
Guido van Rossum0612d842000-03-10 23:20:43 +0000678
679 return self.reader.read(size)
680
Guido van Rossumd58c26f2000-05-01 16:17:32 +0000681 def readline(self, size=None):
Guido van Rossuma3277132000-04-11 15:37:43 +0000682
683 return self.reader.readline(size)
684
Guido van Rossumd58c26f2000-05-01 16:17:32 +0000685 def readlines(self, sizehint=None):
Guido van Rossuma3277132000-04-11 15:37:43 +0000686
687 return self.reader.readlines(sizehint)
688
Georg Brandla18af4e2007-04-21 15:47:16 +0000689 def __next__(self):
Walter Dörwald4dbf1922002-11-06 16:53:44 +0000690
691 """ Return the next decoded line from the input stream."""
Georg Brandla18af4e2007-04-21 15:47:16 +0000692 return next(self.reader)
Walter Dörwald4dbf1922002-11-06 16:53:44 +0000693
694 def __iter__(self):
695 return self
696
Tim Peters30324a72001-05-15 17:19:16 +0000697 def write(self, data):
Guido van Rossum0612d842000-03-10 23:20:43 +0000698
699 return self.writer.write(data)
700
Tim Peters30324a72001-05-15 17:19:16 +0000701 def writelines(self, list):
Guido van Rossuma3277132000-04-11 15:37:43 +0000702
703 return self.writer.writelines(list)
704
Guido van Rossum0612d842000-03-10 23:20:43 +0000705 def reset(self):
706
707 self.reader.reset()
708 self.writer.reset()
709
Victor Stinner3fed0872010-05-22 02:16:27 +0000710 def seek(self, offset, whence=0):
Victor Stinnera92ad7e2010-05-22 16:59:09 +0000711 self.stream.seek(offset, whence)
712 self.reader.reset()
713 if whence == 0 and offset == 0:
714 self.writer.reset()
Victor Stinner3fed0872010-05-22 02:16:27 +0000715
Tim Peters30324a72001-05-15 17:19:16 +0000716 def __getattr__(self, name,
Guido van Rossum0612d842000-03-10 23:20:43 +0000717 getattr=getattr):
718
719 """ Inherit all other methods from the underlying stream.
720 """
Tim Peters30324a72001-05-15 17:19:16 +0000721 return getattr(self.stream, name)
Guido van Rossum0612d842000-03-10 23:20:43 +0000722
Thomas Wouters89f507f2006-12-13 04:49:30 +0000723 # these are needed to make "with codecs.open(...)" work properly
724
725 def __enter__(self):
726 return self
727
728 def __exit__(self, type, value, tb):
729 self.stream.close()
730
Guido van Rossum0612d842000-03-10 23:20:43 +0000731###
732
733class StreamRecoder:
734
Fred Drake49fd1072000-04-13 14:11:21 +0000735 """ StreamRecoder instances provide a frontend - backend
736 view of encoding data.
737
738 They use the complete set of APIs returned by the
739 codecs.lookup() function to implement their task.
740
741 Data written to the stream is first decoded into an
742 intermediate format (which is dependent on the given codec
743 combination) and then written to the stream using an instance
744 of the provided Writer class.
745
746 In the other direction, data is read from the stream using a
747 Reader instance and then return encoded data to the caller.
748
749 """
Guido van Rossuma3277132000-04-11 15:37:43 +0000750 # Optional attributes set by the file wrappers below
751 data_encoding = 'unknown'
752 file_encoding = 'unknown'
753
Tim Peters30324a72001-05-15 17:19:16 +0000754 def __init__(self, stream, encode, decode, Reader, Writer,
755 errors='strict'):
Guido van Rossum0612d842000-03-10 23:20:43 +0000756
757 """ Creates a StreamRecoder instance which implements a two-way
758 conversion: encode and decode work on the frontend (the
Guido van Rossum1c89b0e2000-04-11 15:41:38 +0000759 input to .read() and output of .write()) while
Guido van Rossum0612d842000-03-10 23:20:43 +0000760 Reader and Writer work on the backend (reading and
Fred Drake908670c2000-03-17 15:42:11 +0000761 writing to the stream).
Guido van Rossum0612d842000-03-10 23:20:43 +0000762
763 You can use these objects to do transparent direct
764 recodings from e.g. latin-1 to utf-8 and back.
765
766 stream must be a file-like object.
767
768 encode, decode must adhere to the Codec interface, Reader,
769 Writer must be factory functions or classes providing the
770 StreamReader, StreamWriter interface resp.
771
772 encode and decode are needed for the frontend translation,
773 Reader and Writer for the backend translation. Unicode is
774 used as intermediate encoding.
775
776 Error handling is done in the same way as defined for the
777 StreamWriter/Readers.
778
779 """
780 self.stream = stream
781 self.encode = encode
782 self.decode = decode
783 self.reader = Reader(stream, errors)
784 self.writer = Writer(stream, errors)
785 self.errors = errors
786
Tim Peters30324a72001-05-15 17:19:16 +0000787 def read(self, size=-1):
Guido van Rossum0612d842000-03-10 23:20:43 +0000788
789 data = self.reader.read(size)
790 data, bytesencoded = self.encode(data, self.errors)
791 return data
792
Tim Peters30324a72001-05-15 17:19:16 +0000793 def readline(self, size=None):
Guido van Rossuma3277132000-04-11 15:37:43 +0000794
795 if size is None:
796 data = self.reader.readline()
797 else:
798 data = self.reader.readline(size)
799 data, bytesencoded = self.encode(data, self.errors)
800 return data
801
Tim Peters30324a72001-05-15 17:19:16 +0000802 def readlines(self, sizehint=None):
Guido van Rossuma3277132000-04-11 15:37:43 +0000803
Marc-André Lemburgd5948492004-02-26 15:22:17 +0000804 data = self.reader.read()
Guido van Rossuma3277132000-04-11 15:37:43 +0000805 data, bytesencoded = self.encode(data, self.errors)
Ezio Melottid8b509b2011-09-28 17:37:55 +0300806 return data.splitlines(keepends=True)
Guido van Rossuma3277132000-04-11 15:37:43 +0000807
Georg Brandla18af4e2007-04-21 15:47:16 +0000808 def __next__(self):
Walter Dörwald4dbf1922002-11-06 16:53:44 +0000809
810 """ Return the next decoded line from the input stream."""
Georg Brandla18af4e2007-04-21 15:47:16 +0000811 data = next(self.reader)
Walter Dörwaldc5238b82005-09-01 11:56:53 +0000812 data, bytesencoded = self.encode(data, self.errors)
813 return data
Walter Dörwald4dbf1922002-11-06 16:53:44 +0000814
815 def __iter__(self):
816 return self
817
Tim Peters30324a72001-05-15 17:19:16 +0000818 def write(self, data):
Guido van Rossum0612d842000-03-10 23:20:43 +0000819
820 data, bytesdecoded = self.decode(data, self.errors)
821 return self.writer.write(data)
822
Tim Peters30324a72001-05-15 17:19:16 +0000823 def writelines(self, list):
Guido van Rossuma3277132000-04-11 15:37:43 +0000824
825 data = ''.join(list)
826 data, bytesdecoded = self.decode(data, self.errors)
827 return self.writer.write(data)
Guido van Rossum0612d842000-03-10 23:20:43 +0000828
829 def reset(self):
830
831 self.reader.reset()
832 self.writer.reset()
833
Tim Peters30324a72001-05-15 17:19:16 +0000834 def __getattr__(self, name,
Guido van Rossum0612d842000-03-10 23:20:43 +0000835 getattr=getattr):
836
837 """ Inherit all other methods from the underlying stream.
838 """
Tim Peters30324a72001-05-15 17:19:16 +0000839 return getattr(self.stream, name)
Guido van Rossum0612d842000-03-10 23:20:43 +0000840
Thomas Wouters89f507f2006-12-13 04:49:30 +0000841 def __enter__(self):
842 return self
843
844 def __exit__(self, type, value, tb):
845 self.stream.close()
846
Guido van Rossum0612d842000-03-10 23:20:43 +0000847### Shortcuts
848
Marc-André Lemburg349a3d32000-06-21 21:21:04 +0000849def open(filename, mode='rb', encoding=None, errors='strict', buffering=1):
Guido van Rossum0612d842000-03-10 23:20:43 +0000850
851 """ Open an encoded file using the given mode and return
852 a wrapped version providing transparent encoding/decoding.
853
854 Note: The wrapped version will only accept the object format
855 defined by the codecs, i.e. Unicode objects for most builtin
Skip Montanaro9f5f9d92005-03-16 03:51:56 +0000856 codecs. Output is also codec dependent and will usually be
Guido van Rossum0612d842000-03-10 23:20:43 +0000857 Unicode as well.
858
Marc-André Lemburg349a3d32000-06-21 21:21:04 +0000859 Files are always opened in binary mode, even if no binary mode
Walter Dörwald7f3ed742003-02-02 23:08:27 +0000860 was specified. This is done to avoid data loss due to encodings
Marc-André Lemburg349a3d32000-06-21 21:21:04 +0000861 using 8-bit values. The default file mode is 'rb' meaning to
862 open the file in binary read mode.
863
Guido van Rossum0612d842000-03-10 23:20:43 +0000864 encoding specifies the encoding which is to be used for the
Walter Dörwald7f3ed742003-02-02 23:08:27 +0000865 file.
Guido van Rossum0612d842000-03-10 23:20:43 +0000866
867 errors may be given to define the error handling. It defaults
868 to 'strict' which causes ValueErrors to be raised in case an
869 encoding error occurs.
870
871 buffering has the same meaning as for the builtin open() API.
872 It defaults to line buffered.
873
Fred Drake49fd1072000-04-13 14:11:21 +0000874 The returned wrapped file object provides an extra attribute
875 .encoding which allows querying the used encoding. This
876 attribute is only available if an encoding was specified as
877 parameter.
878
Guido van Rossum0612d842000-03-10 23:20:43 +0000879 """
880 if encoding is not None and \
881 'b' not in mode:
882 # Force opening of the file in binary mode
883 mode = mode + 'b'
Georg Brandl1a3284e2007-12-02 09:40:06 +0000884 file = builtins.open(filename, mode, buffering)
Guido van Rossum0612d842000-03-10 23:20:43 +0000885 if encoding is None:
886 return file
Thomas Woutersa9773292006-04-21 09:43:23 +0000887 info = lookup(encoding)
888 srw = StreamReaderWriter(file, info.streamreader, info.streamwriter, errors)
Guido van Rossuma3277132000-04-11 15:37:43 +0000889 # Add attributes to simplify introspection
890 srw.encoding = encoding
891 return srw
Guido van Rossum0612d842000-03-10 23:20:43 +0000892
Guido van Rossuma3277132000-04-11 15:37:43 +0000893def EncodedFile(file, data_encoding, file_encoding=None, errors='strict'):
Guido van Rossum0612d842000-03-10 23:20:43 +0000894
895 """ Return a wrapped version of file which provides transparent
896 encoding translation.
897
898 Strings written to the wrapped file are interpreted according
Guido van Rossuma3277132000-04-11 15:37:43 +0000899 to the given data_encoding and then written to the original
900 file as string using file_encoding. The intermediate encoding
901 will usually be Unicode but depends on the specified codecs.
Guido van Rossum0612d842000-03-10 23:20:43 +0000902
Guido van Rossuma3277132000-04-11 15:37:43 +0000903 Strings are read from the file using file_encoding and then
904 passed back to the caller as string using data_encoding.
905
906 If file_encoding is not given, it defaults to data_encoding.
Guido van Rossum0612d842000-03-10 23:20:43 +0000907
908 errors may be given to define the error handling. It defaults
909 to 'strict' which causes ValueErrors to be raised in case an
910 encoding error occurs.
911
Fred Drake49fd1072000-04-13 14:11:21 +0000912 The returned wrapped file object provides two extra attributes
913 .data_encoding and .file_encoding which reflect the given
914 parameters of the same name. The attributes can be used for
915 introspection by Python programs.
916
Guido van Rossum0612d842000-03-10 23:20:43 +0000917 """
Guido van Rossuma3277132000-04-11 15:37:43 +0000918 if file_encoding is None:
919 file_encoding = data_encoding
Thomas Wouters89f507f2006-12-13 04:49:30 +0000920 data_info = lookup(data_encoding)
921 file_info = lookup(file_encoding)
922 sr = StreamRecoder(file, data_info.encode, data_info.decode,
923 file_info.streamreader, file_info.streamwriter, errors)
Guido van Rossuma3277132000-04-11 15:37:43 +0000924 # Add attributes to simplify introspection
925 sr.data_encoding = data_encoding
926 sr.file_encoding = file_encoding
927 return sr
Guido van Rossum0612d842000-03-10 23:20:43 +0000928
Marc-André Lemburgaa32c5a2001-09-19 11:24:48 +0000929### Helpers for codec lookup
930
931def getencoder(encoding):
932
933 """ Lookup up the codec for the given encoding and return
934 its encoder function.
935
936 Raises a LookupError in case the encoding cannot be found.
937
938 """
Thomas Woutersa9773292006-04-21 09:43:23 +0000939 return lookup(encoding).encode
Marc-André Lemburgaa32c5a2001-09-19 11:24:48 +0000940
941def getdecoder(encoding):
942
943 """ Lookup up the codec for the given encoding and return
944 its decoder function.
945
946 Raises a LookupError in case the encoding cannot be found.
947
948 """
Thomas Woutersa9773292006-04-21 09:43:23 +0000949 return lookup(encoding).decode
950
951def getincrementalencoder(encoding):
952
953 """ Lookup up the codec for the given encoding and return
954 its IncrementalEncoder class or factory function.
955
956 Raises a LookupError in case the encoding cannot be found
957 or the codecs doesn't provide an incremental encoder.
958
959 """
960 encoder = lookup(encoding).incrementalencoder
961 if encoder is None:
962 raise LookupError(encoding)
963 return encoder
964
965def getincrementaldecoder(encoding):
966
967 """ Lookup up the codec for the given encoding and return
968 its IncrementalDecoder class or factory function.
969
970 Raises a LookupError in case the encoding cannot be found
971 or the codecs doesn't provide an incremental decoder.
972
973 """
974 decoder = lookup(encoding).incrementaldecoder
975 if decoder is None:
976 raise LookupError(encoding)
977 return decoder
Marc-André Lemburgaa32c5a2001-09-19 11:24:48 +0000978
979def getreader(encoding):
980
981 """ Lookup up the codec for the given encoding and return
982 its StreamReader class or factory function.
983
984 Raises a LookupError in case the encoding cannot be found.
985
986 """
Thomas Woutersa9773292006-04-21 09:43:23 +0000987 return lookup(encoding).streamreader
Marc-André Lemburgaa32c5a2001-09-19 11:24:48 +0000988
989def getwriter(encoding):
990
991 """ Lookup up the codec for the given encoding and return
992 its StreamWriter class or factory function.
993
994 Raises a LookupError in case the encoding cannot be found.
995
996 """
Thomas Woutersa9773292006-04-21 09:43:23 +0000997 return lookup(encoding).streamwriter
998
999def iterencode(iterator, encoding, errors='strict', **kwargs):
1000 """
1001 Encoding iterator.
1002
1003 Encodes the input strings from the iterator using a IncrementalEncoder.
1004
1005 errors and kwargs are passed through to the IncrementalEncoder
1006 constructor.
1007 """
1008 encoder = getincrementalencoder(encoding)(errors, **kwargs)
1009 for input in iterator:
1010 output = encoder.encode(input)
1011 if output:
1012 yield output
1013 output = encoder.encode("", True)
1014 if output:
1015 yield output
1016
1017def iterdecode(iterator, encoding, errors='strict', **kwargs):
1018 """
1019 Decoding iterator.
1020
1021 Decodes the input strings from the iterator using a IncrementalDecoder.
1022
1023 errors and kwargs are passed through to the IncrementalDecoder
1024 constructor.
1025 """
1026 decoder = getincrementaldecoder(encoding)(errors, **kwargs)
1027 for input in iterator:
1028 output = decoder.decode(input)
1029 if output:
1030 yield output
Walter Dörwaldca8a8d02007-05-04 13:05:09 +00001031 output = decoder.decode(b"", True)
Thomas Woutersa9773292006-04-21 09:43:23 +00001032 if output:
1033 yield output
Marc-André Lemburgaa32c5a2001-09-19 11:24:48 +00001034
Marc-André Lemburga866df82001-01-03 21:29:14 +00001035### Helpers for charmap-based codecs
1036
1037def make_identity_dict(rng):
1038
1039 """ make_identity_dict(rng) -> dict
1040
1041 Return a dictionary where elements of the rng sequence are
1042 mapped to themselves.
Tim Peters88869f92001-01-14 23:36:06 +00001043
Marc-André Lemburga866df82001-01-03 21:29:14 +00001044 """
1045 res = {}
1046 for i in rng:
1047 res[i]=i
1048 return res
1049
Marc-André Lemburg716cf912001-05-16 09:41:45 +00001050def make_encoding_map(decoding_map):
1051
1052 """ Creates an encoding map from a decoding map.
1053
Walter Dörwald7f3ed742003-02-02 23:08:27 +00001054 If a target mapping in the decoding map occurs multiple
Marc-André Lemburg716cf912001-05-16 09:41:45 +00001055 times, then that target is mapped to None (undefined mapping),
1056 causing an exception when encountered by the charmap codec
1057 during translation.
1058
1059 One example where this happens is cp875.py which decodes
1060 multiple character to \u001a.
1061
1062 """
1063 m = {}
1064 for k,v in decoding_map.items():
Raymond Hettinger54f02222002-06-01 14:18:47 +00001065 if not v in m:
Marc-André Lemburg716cf912001-05-16 09:41:45 +00001066 m[v] = k
1067 else:
1068 m[v] = None
1069 return m
Tim Peters3a2ab1a2001-05-29 06:06:54 +00001070
Walter Dörwald3aeb6322002-09-02 13:14:32 +00001071### error handlers
1072
Martin v. Löwise2713be2005-03-08 15:03:08 +00001073try:
1074 strict_errors = lookup_error("strict")
1075 ignore_errors = lookup_error("ignore")
1076 replace_errors = lookup_error("replace")
1077 xmlcharrefreplace_errors = lookup_error("xmlcharrefreplace")
1078 backslashreplace_errors = lookup_error("backslashreplace")
1079except LookupError:
1080 # In --disable-unicode builds, these error handler are missing
1081 strict_errors = None
1082 ignore_errors = None
1083 replace_errors = None
1084 xmlcharrefreplace_errors = None
1085 backslashreplace_errors = None
Walter Dörwald3aeb6322002-09-02 13:14:32 +00001086
Martin v. Löwis6cd441d2001-07-31 08:54:55 +00001087# Tell modulefinder that using codecs probably needs the encodings
1088# package
1089_false = 0
1090if _false:
1091 import encodings
1092
Guido van Rossum0612d842000-03-10 23:20:43 +00001093### Tests
Guido van Rossum1c89b0e2000-04-11 15:41:38 +00001094
Guido van Rossum0612d842000-03-10 23:20:43 +00001095if __name__ == '__main__':
1096
Guido van Rossuma3277132000-04-11 15:37:43 +00001097 # Make stdout translate Latin-1 output into UTF-8 output
1098 sys.stdout = EncodedFile(sys.stdout, 'latin-1', 'utf-8')
Guido van Rossum1c89b0e2000-04-11 15:41:38 +00001099
Guido van Rossuma3277132000-04-11 15:37:43 +00001100 # Have stdin translate Latin-1 input into UTF-8 input
1101 sys.stdin = EncodedFile(sys.stdin, 'utf-8', 'latin-1')