Martin v. Löwis | 412ed3b | 2006-01-08 10:45:39 +0000 | [diff] [blame] | 1 | """ Python 'utf-8-sig' Codec |
| 2 | This work similar to UTF-8 with the following changes: |
| 3 | |
| 4 | * On encoding/writing a UTF-8 encoded BOM will be prepended/written as the |
| 5 | first three bytes. |
| 6 | |
| 7 | * On decoding/reading if the first three bytes are a UTF-8 encoded BOM, these |
| 8 | bytes will be skipped. |
| 9 | """ |
| 10 | import codecs |
| 11 | |
| 12 | ### Codec APIs |
| 13 | |
| 14 | def encode(input, errors='strict'): |
Walter Dörwald | 3abcb01 | 2007-04-16 22:10:50 +0000 | [diff] [blame] | 15 | return (codecs.BOM_UTF8 + codecs.utf_8_encode(input, errors)[0], |
| 16 | len(input)) |
Martin v. Löwis | 412ed3b | 2006-01-08 10:45:39 +0000 | [diff] [blame] | 17 | |
| 18 | def decode(input, errors='strict'): |
| 19 | prefix = 0 |
Thomas Wouters | 89f507f | 2006-12-13 04:49:30 +0000 | [diff] [blame] | 20 | if input[:3] == codecs.BOM_UTF8: |
Martin v. Löwis | 412ed3b | 2006-01-08 10:45:39 +0000 | [diff] [blame] | 21 | input = input[3:] |
| 22 | prefix = 3 |
| 23 | (output, consumed) = codecs.utf_8_decode(input, errors, True) |
| 24 | return (output, consumed+prefix) |
| 25 | |
Thomas Wouters | a977329 | 2006-04-21 09:43:23 +0000 | [diff] [blame] | 26 | class IncrementalEncoder(codecs.IncrementalEncoder): |
| 27 | def __init__(self, errors='strict'): |
| 28 | codecs.IncrementalEncoder.__init__(self, errors) |
Walter Dörwald | 3abcb01 | 2007-04-16 22:10:50 +0000 | [diff] [blame] | 29 | self.first = 1 |
Thomas Wouters | a977329 | 2006-04-21 09:43:23 +0000 | [diff] [blame] | 30 | |
| 31 | def encode(self, input, final=False): |
| 32 | if self.first: |
Walter Dörwald | 3abcb01 | 2007-04-16 22:10:50 +0000 | [diff] [blame] | 33 | self.first = 0 |
| 34 | return codecs.BOM_UTF8 + \ |
| 35 | codecs.utf_8_encode(input, self.errors)[0] |
Thomas Wouters | a977329 | 2006-04-21 09:43:23 +0000 | [diff] [blame] | 36 | else: |
Thomas Wouters | 0e3f591 | 2006-08-11 14:57:12 +0000 | [diff] [blame] | 37 | return codecs.utf_8_encode(input, self.errors)[0] |
Thomas Wouters | a977329 | 2006-04-21 09:43:23 +0000 | [diff] [blame] | 38 | |
| 39 | def reset(self): |
| 40 | codecs.IncrementalEncoder.reset(self) |
Walter Dörwald | 3abcb01 | 2007-04-16 22:10:50 +0000 | [diff] [blame] | 41 | self.first = 1 |
| 42 | |
| 43 | def getstate(self): |
| 44 | return self.first |
| 45 | |
| 46 | def setstate(self, state): |
| 47 | self.first = state |
Thomas Wouters | a977329 | 2006-04-21 09:43:23 +0000 | [diff] [blame] | 48 | |
| 49 | class IncrementalDecoder(codecs.BufferedIncrementalDecoder): |
| 50 | def __init__(self, errors='strict'): |
| 51 | codecs.BufferedIncrementalDecoder.__init__(self, errors) |
Walter Dörwald | 3abcb01 | 2007-04-16 22:10:50 +0000 | [diff] [blame] | 52 | self.first = 1 |
Thomas Wouters | a977329 | 2006-04-21 09:43:23 +0000 | [diff] [blame] | 53 | |
| 54 | def _buffer_decode(self, input, errors, final): |
Walter Dörwald | 3abcb01 | 2007-04-16 22:10:50 +0000 | [diff] [blame] | 55 | if self.first: |
Thomas Wouters | a977329 | 2006-04-21 09:43:23 +0000 | [diff] [blame] | 56 | if len(input) < 3: |
Walter Dörwald | 3abcb01 | 2007-04-16 22:10:50 +0000 | [diff] [blame] | 57 | if codecs.BOM_UTF8.startswith(input): |
| 58 | # not enough data to decide if this really is a BOM |
| 59 | # => try again on the next call |
Guido van Rossum | ef87d6e | 2007-05-02 19:09:54 +0000 | [diff] [blame] | 60 | return ("", 0) |
Walter Dörwald | 3abcb01 | 2007-04-16 22:10:50 +0000 | [diff] [blame] | 61 | else: |
| 62 | self.first = 0 |
| 63 | else: |
| 64 | self.first = 0 |
| 65 | if input[:3] == codecs.BOM_UTF8: |
| 66 | (output, consumed) = \ |
| 67 | codecs.utf_8_decode(input[3:], errors, final) |
| 68 | return (output, consumed+3) |
Thomas Wouters | a977329 | 2006-04-21 09:43:23 +0000 | [diff] [blame] | 69 | return codecs.utf_8_decode(input, errors, final) |
| 70 | |
| 71 | def reset(self): |
| 72 | codecs.BufferedIncrementalDecoder.reset(self) |
Walter Dörwald | 3abcb01 | 2007-04-16 22:10:50 +0000 | [diff] [blame] | 73 | self.first = 1 |
| 74 | |
| 75 | def getstate(self): |
| 76 | state = codecs.BufferedIncrementalDecoder.getstate(self) |
| 77 | # state[1] must be 0 here, as it isn't passed along to the caller |
| 78 | return (state[0], self.first) |
| 79 | |
| 80 | def setstate(self, state): |
| 81 | # state[1] will be ignored by BufferedIncrementalDecoder.setstate() |
| 82 | codecs.BufferedIncrementalDecoder.setstate(self, state) |
| 83 | self.first = state[1] |
Thomas Wouters | a977329 | 2006-04-21 09:43:23 +0000 | [diff] [blame] | 84 | |
Martin v. Löwis | 412ed3b | 2006-01-08 10:45:39 +0000 | [diff] [blame] | 85 | class StreamWriter(codecs.StreamWriter): |
| 86 | def reset(self): |
| 87 | codecs.StreamWriter.reset(self) |
| 88 | try: |
| 89 | del self.encode |
| 90 | except AttributeError: |
| 91 | pass |
| 92 | |
| 93 | def encode(self, input, errors='strict'): |
| 94 | self.encode = codecs.utf_8_encode |
| 95 | return encode(input, errors) |
| 96 | |
| 97 | class StreamReader(codecs.StreamReader): |
| 98 | def reset(self): |
| 99 | codecs.StreamReader.reset(self) |
| 100 | try: |
| 101 | del self.decode |
| 102 | except AttributeError: |
| 103 | pass |
| 104 | |
| 105 | def decode(self, input, errors='strict'): |
Guido van Rossum | 87c0f1d | 2007-11-19 18:03:44 +0000 | [diff] [blame] | 106 | if len(input) < 3: |
| 107 | if codecs.BOM_UTF8.startswith(input): |
| 108 | # not enough data to decide if this is a BOM |
| 109 | # => try again on the next call |
| 110 | return ("", 0) |
| 111 | elif input[:3] == codecs.BOM_UTF8: |
| 112 | self.decode = codecs.utf_8_decode |
| 113 | (output, consumed) = codecs.utf_8_decode(input[3:],errors) |
| 114 | return (output, consumed+3) |
| 115 | # (else) no BOM present |
Martin v. Löwis | 412ed3b | 2006-01-08 10:45:39 +0000 | [diff] [blame] | 116 | self.decode = codecs.utf_8_decode |
Guido van Rossum | 87c0f1d | 2007-11-19 18:03:44 +0000 | [diff] [blame] | 117 | return codecs.utf_8_decode(input, errors) |
Martin v. Löwis | 412ed3b | 2006-01-08 10:45:39 +0000 | [diff] [blame] | 118 | |
| 119 | ### encodings module API |
| 120 | |
| 121 | def getregentry(): |
Thomas Wouters | a977329 | 2006-04-21 09:43:23 +0000 | [diff] [blame] | 122 | return codecs.CodecInfo( |
| 123 | name='utf-8-sig', |
| 124 | encode=encode, |
| 125 | decode=decode, |
| 126 | incrementalencoder=IncrementalEncoder, |
| 127 | incrementaldecoder=IncrementalDecoder, |
| 128 | streamreader=StreamReader, |
| 129 | streamwriter=StreamWriter, |
| 130 | ) |