blob: 885e740b7663bb0ae3dcfd29d542e64064dfad1f [file] [log] [blame]
Guido van Rossuma831cac2000-03-10 23:23:21 +00001""" Test script for the Unicode implementation.
2
Guido van Rossuma831cac2000-03-10 23:23:21 +00003Written by Marc-Andre Lemburg (mal@lemburg.com).
4
5(c) Copyright CNRI, All Rights Reserved. NO WARRANTY.
6
Marc-André Lemburg36619082001-01-17 19:11:13 +00007"""#"
Guido van Rossum98297ee2007-11-06 21:34:58 +00008import codecs
9import struct
10import sys
11import unittest
12import warnings
Benjamin Petersonee8712c2008-05-20 21:35:26 +000013from test import support, string_tests
Eric Smitha1eac722011-01-29 11:15:35 +000014import _string
Guido van Rossuma831cac2000-03-10 23:23:21 +000015
Neal Norwitz430f68b2005-11-24 22:00:56 +000016# Error handling (bad decoder return)
17def search_function(encoding):
18 def decode1(input, errors="strict"):
19 return 42 # not a tuple
20 def encode1(input, errors="strict"):
21 return 42 # not a tuple
22 def encode2(input, errors="strict"):
23 return (42, 42) # no unicode
24 def decode2(input, errors="strict"):
25 return (42, 42) # no unicode
26 if encoding=="test.unicode1":
27 return (encode1, decode1, None, None)
28 elif encoding=="test.unicode2":
29 return (encode2, decode2, None, None)
30 else:
31 return None
32codecs.register(search_function)
33
Brett Cannon226b2302010-03-20 22:22:22 +000034class UnicodeTest(string_tests.CommonTest,
35 string_tests.MixinStrUnicodeUserStringTest,
36 string_tests.MixinStrUnicodeTest):
37
Guido van Rossumef87d6e2007-05-02 19:09:54 +000038 type2test = str
Walter Dörwald0fd583c2003-02-21 12:53:50 +000039
40 def checkequalnofix(self, result, object, methodname, *args):
41 method = getattr(object, methodname)
42 realresult = method(*args)
43 self.assertEqual(realresult, result)
Benjamin Petersonc9c0f202009-06-30 23:06:06 +000044 self.assertTrue(type(realresult) is type(result))
Walter Dörwald0fd583c2003-02-21 12:53:50 +000045
46 # if the original is returned make sure that
47 # this doesn't happen with subclasses
48 if realresult is object:
Guido van Rossumef87d6e2007-05-02 19:09:54 +000049 class usub(str):
Walter Dörwald0fd583c2003-02-21 12:53:50 +000050 def __repr__(self):
Guido van Rossumef87d6e2007-05-02 19:09:54 +000051 return 'usub(%r)' % str.__repr__(self)
Walter Dörwald0fd583c2003-02-21 12:53:50 +000052 object = usub(object)
53 method = getattr(object, methodname)
54 realresult = method(*args)
55 self.assertEqual(realresult, result)
Benjamin Petersonc9c0f202009-06-30 23:06:06 +000056 self.assertTrue(object is not realresult)
Guido van Rossume4874ae2001-09-21 15:36:41 +000057
Jeremy Hylton504de6b2003-10-06 05:08:26 +000058 def test_literals(self):
Guido van Rossumef87d6e2007-05-02 19:09:54 +000059 self.assertEqual('\xff', '\u00ff')
60 self.assertEqual('\uffff', '\U0000ffff')
Guido van Rossum36e0a922007-07-20 04:05:57 +000061 self.assertRaises(SyntaxError, eval, '\'\\Ufffffffe\'')
62 self.assertRaises(SyntaxError, eval, '\'\\Uffffffff\'')
63 self.assertRaises(SyntaxError, eval, '\'\\U%08x\'' % 0x110000)
Benjamin Petersoncd76c272008-04-05 15:09:30 +000064 # raw strings should not have unicode escapes
Florent Xiclunaa87b3832010-09-13 02:28:18 +000065 self.assertNotEqual(r"\u0020", " ")
Jeremy Hylton504de6b2003-10-06 05:08:26 +000066
Georg Brandl559e5d72008-06-11 18:37:52 +000067 def test_ascii(self):
68 if not sys.platform.startswith('java'):
69 # Test basic sanity of repr()
70 self.assertEqual(ascii('abc'), "'abc'")
71 self.assertEqual(ascii('ab\\c'), "'ab\\\\c'")
72 self.assertEqual(ascii('ab\\'), "'ab\\\\'")
73 self.assertEqual(ascii('\\c'), "'\\\\c'")
74 self.assertEqual(ascii('\\'), "'\\\\'")
75 self.assertEqual(ascii('\n'), "'\\n'")
76 self.assertEqual(ascii('\r'), "'\\r'")
77 self.assertEqual(ascii('\t'), "'\\t'")
78 self.assertEqual(ascii('\b'), "'\\x08'")
79 self.assertEqual(ascii("'\""), """'\\'"'""")
80 self.assertEqual(ascii("'\""), """'\\'"'""")
81 self.assertEqual(ascii("'"), '''"'"''')
82 self.assertEqual(ascii('"'), """'"'""")
83 latin1repr = (
84 "'\\x00\\x01\\x02\\x03\\x04\\x05\\x06\\x07\\x08\\t\\n\\x0b\\x0c\\r"
85 "\\x0e\\x0f\\x10\\x11\\x12\\x13\\x14\\x15\\x16\\x17\\x18\\x19\\x1a"
86 "\\x1b\\x1c\\x1d\\x1e\\x1f !\"#$%&\\'()*+,-./0123456789:;<=>?@ABCDEFGHI"
87 "JKLMNOPQRSTUVWXYZ[\\\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\\x7f"
88 "\\x80\\x81\\x82\\x83\\x84\\x85\\x86\\x87\\x88\\x89\\x8a\\x8b\\x8c\\x8d"
89 "\\x8e\\x8f\\x90\\x91\\x92\\x93\\x94\\x95\\x96\\x97\\x98\\x99\\x9a\\x9b"
90 "\\x9c\\x9d\\x9e\\x9f\\xa0\\xa1\\xa2\\xa3\\xa4\\xa5\\xa6\\xa7\\xa8\\xa9"
91 "\\xaa\\xab\\xac\\xad\\xae\\xaf\\xb0\\xb1\\xb2\\xb3\\xb4\\xb5\\xb6\\xb7"
92 "\\xb8\\xb9\\xba\\xbb\\xbc\\xbd\\xbe\\xbf\\xc0\\xc1\\xc2\\xc3\\xc4\\xc5"
93 "\\xc6\\xc7\\xc8\\xc9\\xca\\xcb\\xcc\\xcd\\xce\\xcf\\xd0\\xd1\\xd2\\xd3"
94 "\\xd4\\xd5\\xd6\\xd7\\xd8\\xd9\\xda\\xdb\\xdc\\xdd\\xde\\xdf\\xe0\\xe1"
95 "\\xe2\\xe3\\xe4\\xe5\\xe6\\xe7\\xe8\\xe9\\xea\\xeb\\xec\\xed\\xee\\xef"
96 "\\xf0\\xf1\\xf2\\xf3\\xf4\\xf5\\xf6\\xf7\\xf8\\xf9\\xfa\\xfb\\xfc\\xfd"
97 "\\xfe\\xff'")
98 testrepr = ascii(''.join(map(chr, range(256))))
99 self.assertEqual(testrepr, latin1repr)
100 # Test ascii works on wide unicode escapes without overflow.
101 self.assertEqual(ascii("\U00010000" * 39 + "\uffff" * 4096),
102 ascii("\U00010000" * 39 + "\uffff" * 4096))
103
104 class WrongRepr:
105 def __repr__(self):
106 return b'byte-repr'
107 self.assertRaises(TypeError, ascii, WrongRepr())
108
Walter Dörwald28256f22003-01-19 16:59:20 +0000109 def test_repr(self):
110 if not sys.platform.startswith('java'):
111 # Test basic sanity of repr()
Walter Dörwald67e83882007-05-05 12:26:27 +0000112 self.assertEqual(repr('abc'), "'abc'")
113 self.assertEqual(repr('ab\\c'), "'ab\\\\c'")
114 self.assertEqual(repr('ab\\'), "'ab\\\\'")
115 self.assertEqual(repr('\\c'), "'\\\\c'")
116 self.assertEqual(repr('\\'), "'\\\\'")
117 self.assertEqual(repr('\n'), "'\\n'")
118 self.assertEqual(repr('\r'), "'\\r'")
119 self.assertEqual(repr('\t'), "'\\t'")
120 self.assertEqual(repr('\b'), "'\\x08'")
121 self.assertEqual(repr("'\""), """'\\'"'""")
122 self.assertEqual(repr("'\""), """'\\'"'""")
123 self.assertEqual(repr("'"), '''"'"''')
124 self.assertEqual(repr('"'), """'"'""")
Walter Dörwald28256f22003-01-19 16:59:20 +0000125 latin1repr = (
Walter Dörwald67e83882007-05-05 12:26:27 +0000126 "'\\x00\\x01\\x02\\x03\\x04\\x05\\x06\\x07\\x08\\t\\n\\x0b\\x0c\\r"
Walter Dörwald28256f22003-01-19 16:59:20 +0000127 "\\x0e\\x0f\\x10\\x11\\x12\\x13\\x14\\x15\\x16\\x17\\x18\\x19\\x1a"
128 "\\x1b\\x1c\\x1d\\x1e\\x1f !\"#$%&\\'()*+,-./0123456789:;<=>?@ABCDEFGHI"
129 "JKLMNOPQRSTUVWXYZ[\\\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\\x7f"
130 "\\x80\\x81\\x82\\x83\\x84\\x85\\x86\\x87\\x88\\x89\\x8a\\x8b\\x8c\\x8d"
131 "\\x8e\\x8f\\x90\\x91\\x92\\x93\\x94\\x95\\x96\\x97\\x98\\x99\\x9a\\x9b"
Georg Brandl559e5d72008-06-11 18:37:52 +0000132 "\\x9c\\x9d\\x9e\\x9f\\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9"
133 "\xaa\xab\xac\\xad\xae\xaf\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7"
134 "\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf\xc0\xc1\xc2\xc3\xc4\xc5"
135 "\xc6\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf\xd0\xd1\xd2\xd3"
136 "\xd4\xd5\xd6\xd7\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf\xe0\xe1"
137 "\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef"
138 "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd"
139 "\xfe\xff'")
Guido van Rossum805365e2007-05-07 22:24:25 +0000140 testrepr = repr(''.join(map(chr, range(256))))
Walter Dörwald28256f22003-01-19 16:59:20 +0000141 self.assertEqual(testrepr, latin1repr)
Thomas Wouters89f507f2006-12-13 04:49:30 +0000142 # Test repr works on wide unicode escapes without overflow.
Guido van Rossumef87d6e2007-05-02 19:09:54 +0000143 self.assertEqual(repr("\U00010000" * 39 + "\uffff" * 4096),
144 repr("\U00010000" * 39 + "\uffff" * 4096))
Walter Dörwald28256f22003-01-19 16:59:20 +0000145
Georg Brandl559e5d72008-06-11 18:37:52 +0000146 class WrongRepr:
147 def __repr__(self):
148 return b'byte-repr'
149 self.assertRaises(TypeError, repr, WrongRepr())
150
Guido van Rossum49d6b072006-08-17 21:11:47 +0000151 def test_iterators(self):
152 # Make sure unicode objects have an __iter__ method
Guido van Rossumef87d6e2007-05-02 19:09:54 +0000153 it = "\u1111\u2222\u3333".__iter__()
154 self.assertEqual(next(it), "\u1111")
155 self.assertEqual(next(it), "\u2222")
156 self.assertEqual(next(it), "\u3333")
Georg Brandla18af4e2007-04-21 15:47:16 +0000157 self.assertRaises(StopIteration, next, it)
Guido van Rossum49d6b072006-08-17 21:11:47 +0000158
Walter Dörwald28256f22003-01-19 16:59:20 +0000159 def test_count(self):
Walter Dörwald0fd583c2003-02-21 12:53:50 +0000160 string_tests.CommonTest.test_count(self)
161 # check mixed argument types
Guido van Rossumef87d6e2007-05-02 19:09:54 +0000162 self.checkequalnofix(3, 'aaa', 'count', 'a')
163 self.checkequalnofix(0, 'aaa', 'count', 'b')
164 self.checkequalnofix(3, 'aaa', 'count', 'a')
165 self.checkequalnofix(0, 'aaa', 'count', 'b')
166 self.checkequalnofix(0, 'aaa', 'count', 'b')
167 self.checkequalnofix(1, 'aaa', 'count', 'a', -1)
168 self.checkequalnofix(3, 'aaa', 'count', 'a', -10)
169 self.checkequalnofix(2, 'aaa', 'count', 'a', 0, -1)
170 self.checkequalnofix(0, 'aaa', 'count', 'a', 0, -10)
Guido van Rossuma831cac2000-03-10 23:23:21 +0000171
Walter Dörwald28256f22003-01-19 16:59:20 +0000172 def test_find(self):
Guido van Rossumef87d6e2007-05-02 19:09:54 +0000173 self.checkequalnofix(0, 'abcdefghiabc', 'find', 'abc')
174 self.checkequalnofix(9, 'abcdefghiabc', 'find', 'abc', 1)
175 self.checkequalnofix(-1, 'abcdefghiabc', 'find', 'def', 4)
Guido van Rossuma831cac2000-03-10 23:23:21 +0000176
Guido van Rossumef87d6e2007-05-02 19:09:54 +0000177 self.assertRaises(TypeError, 'hello'.find)
178 self.assertRaises(TypeError, 'hello'.find, 42)
Guido van Rossuma831cac2000-03-10 23:23:21 +0000179
Walter Dörwald28256f22003-01-19 16:59:20 +0000180 def test_rfind(self):
Walter Dörwald0fd583c2003-02-21 12:53:50 +0000181 string_tests.CommonTest.test_rfind(self)
182 # check mixed argument types
Guido van Rossumef87d6e2007-05-02 19:09:54 +0000183 self.checkequalnofix(9, 'abcdefghiabc', 'rfind', 'abc')
184 self.checkequalnofix(12, 'abcdefghiabc', 'rfind', '')
185 self.checkequalnofix(12, 'abcdefghiabc', 'rfind', '')
Guido van Rossum8b264542000-12-19 02:22:31 +0000186
Walter Dörwald28256f22003-01-19 16:59:20 +0000187 def test_index(self):
Walter Dörwald0fd583c2003-02-21 12:53:50 +0000188 string_tests.CommonTest.test_index(self)
Walter Dörwaldaa97f042007-05-03 21:05:51 +0000189 self.checkequalnofix(0, 'abcdefghiabc', 'index', '')
190 self.checkequalnofix(3, 'abcdefghiabc', 'index', 'def')
191 self.checkequalnofix(0, 'abcdefghiabc', 'index', 'abc')
192 self.checkequalnofix(9, 'abcdefghiabc', 'index', 'abc', 1)
193 self.assertRaises(ValueError, 'abcdefghiabc'.index, 'hib')
194 self.assertRaises(ValueError, 'abcdefghiab'.index, 'abc', 1)
195 self.assertRaises(ValueError, 'abcdefghi'.index, 'ghi', 8)
196 self.assertRaises(ValueError, 'abcdefghi'.index, 'ghi', -1)
Guido van Rossuma831cac2000-03-10 23:23:21 +0000197
Walter Dörwald28256f22003-01-19 16:59:20 +0000198 def test_rindex(self):
Walter Dörwald0fd583c2003-02-21 12:53:50 +0000199 string_tests.CommonTest.test_rindex(self)
Walter Dörwaldaa97f042007-05-03 21:05:51 +0000200 self.checkequalnofix(12, 'abcdefghiabc', 'rindex', '')
201 self.checkequalnofix(3, 'abcdefghiabc', 'rindex', 'def')
202 self.checkequalnofix(9, 'abcdefghiabc', 'rindex', 'abc')
203 self.checkequalnofix(0, 'abcdefghiabc', 'rindex', 'abc', 0, -1)
Guido van Rossuma831cac2000-03-10 23:23:21 +0000204
Walter Dörwaldaa97f042007-05-03 21:05:51 +0000205 self.assertRaises(ValueError, 'abcdefghiabc'.rindex, 'hib')
206 self.assertRaises(ValueError, 'defghiabc'.rindex, 'def', 1)
207 self.assertRaises(ValueError, 'defghiabc'.rindex, 'abc', 0, -1)
208 self.assertRaises(ValueError, 'abcdefghi'.rindex, 'ghi', 0, 8)
209 self.assertRaises(ValueError, 'abcdefghi'.rindex, 'ghi', 0, -1)
Guido van Rossuma831cac2000-03-10 23:23:21 +0000210
Georg Brandlceee0772007-11-27 23:48:05 +0000211 def test_maketrans_translate(self):
212 # these work with plain translate()
213 self.checkequalnofix('bbbc', 'abababc', 'translate',
214 {ord('a'): None})
215 self.checkequalnofix('iiic', 'abababc', 'translate',
216 {ord('a'): None, ord('b'): ord('i')})
217 self.checkequalnofix('iiix', 'abababc', 'translate',
218 {ord('a'): None, ord('b'): ord('i'), ord('c'): 'x'})
219 self.checkequalnofix('c', 'abababc', 'translate',
220 {ord('a'): None, ord('b'): ''})
221 self.checkequalnofix('xyyx', 'xzx', 'translate',
222 {ord('z'): 'yy'})
223 # this needs maketrans()
224 self.checkequalnofix('abababc', 'abababc', 'translate',
225 {'b': '<i>'})
226 tbl = self.type2test.maketrans({'a': None, 'b': '<i>'})
227 self.checkequalnofix('<i><i><i>c', 'abababc', 'translate', tbl)
228 # test alternative way of calling maketrans()
229 tbl = self.type2test.maketrans('abc', 'xyz', 'd')
230 self.checkequalnofix('xyzzy', 'abdcdcbdddd', 'translate', tbl)
231
232 self.assertRaises(TypeError, self.type2test.maketrans)
233 self.assertRaises(ValueError, self.type2test.maketrans, 'abc', 'defg')
234 self.assertRaises(TypeError, self.type2test.maketrans, 2, 'def')
235 self.assertRaises(TypeError, self.type2test.maketrans, 'abc', 2)
236 self.assertRaises(TypeError, self.type2test.maketrans, 'abc', 'def', 2)
237 self.assertRaises(ValueError, self.type2test.maketrans, {'xy': 2})
238 self.assertRaises(TypeError, self.type2test.maketrans, {(1,): 2})
Guido van Rossuma831cac2000-03-10 23:23:21 +0000239
Guido van Rossumef87d6e2007-05-02 19:09:54 +0000240 self.assertRaises(TypeError, 'hello'.translate)
Walter Dörwald67e83882007-05-05 12:26:27 +0000241 self.assertRaises(TypeError, 'abababc'.translate, 'abc', 'xyz')
Guido van Rossuma831cac2000-03-10 23:23:21 +0000242
Walter Dörwald28256f22003-01-19 16:59:20 +0000243 def test_split(self):
Walter Dörwald0fd583c2003-02-21 12:53:50 +0000244 string_tests.CommonTest.test_split(self)
Andrew M. Kuchlingeddd68d2002-03-29 16:21:44 +0000245
Walter Dörwald0fd583c2003-02-21 12:53:50 +0000246 # Mixed arguments
Guido van Rossumef87d6e2007-05-02 19:09:54 +0000247 self.checkequalnofix(['a', 'b', 'c', 'd'], 'a//b//c//d', 'split', '//')
248 self.checkequalnofix(['a', 'b', 'c', 'd'], 'a//b//c//d', 'split', '//')
249 self.checkequalnofix(['endcase ', ''], 'endcase test', 'split', 'test')
Guido van Rossuma831cac2000-03-10 23:23:21 +0000250
Walter Dörwald28256f22003-01-19 16:59:20 +0000251 def test_join(self):
Walter Dörwald0fd583c2003-02-21 12:53:50 +0000252 string_tests.MixinStrUnicodeUserStringTest.test_join(self)
Marc-André Lemburgd6d06ad2000-07-07 17:48:52 +0000253
Guido van Rossumf1044292007-09-27 18:01:22 +0000254 class MyWrapper:
255 def __init__(self, sval): self.sval = sval
256 def __str__(self): return self.sval
257
Walter Dörwald0fd583c2003-02-21 12:53:50 +0000258 # mixed arguments
Guido van Rossumef87d6e2007-05-02 19:09:54 +0000259 self.checkequalnofix('a b c d', ' ', 'join', ['a', 'b', 'c', 'd'])
260 self.checkequalnofix('abcd', '', 'join', ('a', 'b', 'c', 'd'))
261 self.checkequalnofix('w x y z', ' ', 'join', string_tests.Sequence('wxyz'))
262 self.checkequalnofix('a b c d', ' ', 'join', ['a', 'b', 'c', 'd'])
263 self.checkequalnofix('a b c d', ' ', 'join', ['a', 'b', 'c', 'd'])
264 self.checkequalnofix('abcd', '', 'join', ('a', 'b', 'c', 'd'))
265 self.checkequalnofix('w x y z', ' ', 'join', string_tests.Sequence('wxyz'))
Guido van Rossum98297ee2007-11-06 21:34:58 +0000266 self.checkraises(TypeError, ' ', 'join', ['1', '2', MyWrapper('foo')])
267 self.checkraises(TypeError, ' ', 'join', ['1', '2', '3', bytes()])
268 self.checkraises(TypeError, ' ', 'join', [1, 2, 3])
269 self.checkraises(TypeError, ' ', 'join', ['1', '2', 3])
Marc-André Lemburge5034372000-08-08 08:04:29 +0000270
Walter Dörwald28256f22003-01-19 16:59:20 +0000271 def test_replace(self):
Walter Dörwald0fd583c2003-02-21 12:53:50 +0000272 string_tests.CommonTest.test_replace(self)
Guido van Rossuma831cac2000-03-10 23:23:21 +0000273
Walter Dörwald28256f22003-01-19 16:59:20 +0000274 # method call forwarded from str implementation because of unicode argument
Guido van Rossumef87d6e2007-05-02 19:09:54 +0000275 self.checkequalnofix('one@two!three!', 'one!two!three!', 'replace', '!', '@', 1)
276 self.assertRaises(TypeError, 'replace'.replace, "r", 42)
Guido van Rossuma831cac2000-03-10 23:23:21 +0000277
Guido van Rossum98297ee2007-11-06 21:34:58 +0000278 def test_bytes_comparison(self):
Brett Cannon226b2302010-03-20 22:22:22 +0000279 with support.check_warnings():
280 warnings.simplefilter('ignore', BytesWarning)
281 self.assertEqual('abc' == b'abc', False)
282 self.assertEqual('abc' != b'abc', True)
283 self.assertEqual('abc' == bytearray(b'abc'), False)
284 self.assertEqual('abc' != bytearray(b'abc'), True)
Brett Cannon40430012007-10-22 20:24:51 +0000285
Walter Dörwald28256f22003-01-19 16:59:20 +0000286 def test_comparison(self):
287 # Comparisons:
Guido van Rossumef87d6e2007-05-02 19:09:54 +0000288 self.assertEqual('abc', 'abc')
Benjamin Petersonc9c0f202009-06-30 23:06:06 +0000289 self.assertTrue('abcd' > 'abc')
Benjamin Petersonc9c0f202009-06-30 23:06:06 +0000290 self.assertTrue('abc' < 'abcd')
Walter Dörwald28256f22003-01-19 16:59:20 +0000291
292 if 0:
293 # Move these tests to a Unicode collation module test...
294 # Testing UTF-16 code point order comparisons...
295
296 # No surrogates, no fixup required.
Benjamin Petersonc9c0f202009-06-30 23:06:06 +0000297 self.assertTrue('\u0061' < '\u20ac')
Walter Dörwald28256f22003-01-19 16:59:20 +0000298 # Non surrogate below surrogate value, no fixup required
Benjamin Petersonc9c0f202009-06-30 23:06:06 +0000299 self.assertTrue('\u0061' < '\ud800\udc02')
Walter Dörwald28256f22003-01-19 16:59:20 +0000300
301 # Non surrogate above surrogate value, fixup required
302 def test_lecmp(s, s2):
Benjamin Petersonc9c0f202009-06-30 23:06:06 +0000303 self.assertTrue(s < s2)
Walter Dörwald28256f22003-01-19 16:59:20 +0000304
305 def test_fixup(s):
Guido van Rossumef87d6e2007-05-02 19:09:54 +0000306 s2 = '\ud800\udc01'
Walter Dörwald28256f22003-01-19 16:59:20 +0000307 test_lecmp(s, s2)
Guido van Rossumef87d6e2007-05-02 19:09:54 +0000308 s2 = '\ud900\udc01'
Walter Dörwald28256f22003-01-19 16:59:20 +0000309 test_lecmp(s, s2)
Guido van Rossumef87d6e2007-05-02 19:09:54 +0000310 s2 = '\uda00\udc01'
Walter Dörwald28256f22003-01-19 16:59:20 +0000311 test_lecmp(s, s2)
Guido van Rossumef87d6e2007-05-02 19:09:54 +0000312 s2 = '\udb00\udc01'
Walter Dörwald28256f22003-01-19 16:59:20 +0000313 test_lecmp(s, s2)
Guido van Rossumef87d6e2007-05-02 19:09:54 +0000314 s2 = '\ud800\udd01'
Walter Dörwald28256f22003-01-19 16:59:20 +0000315 test_lecmp(s, s2)
Guido van Rossumef87d6e2007-05-02 19:09:54 +0000316 s2 = '\ud900\udd01'
Walter Dörwald28256f22003-01-19 16:59:20 +0000317 test_lecmp(s, s2)
Guido van Rossumef87d6e2007-05-02 19:09:54 +0000318 s2 = '\uda00\udd01'
Walter Dörwald28256f22003-01-19 16:59:20 +0000319 test_lecmp(s, s2)
Guido van Rossumef87d6e2007-05-02 19:09:54 +0000320 s2 = '\udb00\udd01'
Walter Dörwald28256f22003-01-19 16:59:20 +0000321 test_lecmp(s, s2)
Guido van Rossumef87d6e2007-05-02 19:09:54 +0000322 s2 = '\ud800\ude01'
Walter Dörwald28256f22003-01-19 16:59:20 +0000323 test_lecmp(s, s2)
Guido van Rossumef87d6e2007-05-02 19:09:54 +0000324 s2 = '\ud900\ude01'
Walter Dörwald28256f22003-01-19 16:59:20 +0000325 test_lecmp(s, s2)
Guido van Rossumef87d6e2007-05-02 19:09:54 +0000326 s2 = '\uda00\ude01'
Walter Dörwald28256f22003-01-19 16:59:20 +0000327 test_lecmp(s, s2)
Guido van Rossumef87d6e2007-05-02 19:09:54 +0000328 s2 = '\udb00\ude01'
Walter Dörwald28256f22003-01-19 16:59:20 +0000329 test_lecmp(s, s2)
Guido van Rossumef87d6e2007-05-02 19:09:54 +0000330 s2 = '\ud800\udfff'
Walter Dörwald28256f22003-01-19 16:59:20 +0000331 test_lecmp(s, s2)
Guido van Rossumef87d6e2007-05-02 19:09:54 +0000332 s2 = '\ud900\udfff'
Walter Dörwald28256f22003-01-19 16:59:20 +0000333 test_lecmp(s, s2)
Guido van Rossumef87d6e2007-05-02 19:09:54 +0000334 s2 = '\uda00\udfff'
Walter Dörwald28256f22003-01-19 16:59:20 +0000335 test_lecmp(s, s2)
Guido van Rossumef87d6e2007-05-02 19:09:54 +0000336 s2 = '\udb00\udfff'
Walter Dörwald28256f22003-01-19 16:59:20 +0000337 test_lecmp(s, s2)
338
Guido van Rossumef87d6e2007-05-02 19:09:54 +0000339 test_fixup('\ue000')
340 test_fixup('\uff61')
Walter Dörwald28256f22003-01-19 16:59:20 +0000341
342 # Surrogates on both sides, no fixup required
Benjamin Petersonc9c0f202009-06-30 23:06:06 +0000343 self.assertTrue('\ud800\udc02' < '\ud84d\udc56')
Walter Dörwald28256f22003-01-19 16:59:20 +0000344
Walter Dörwald28256f22003-01-19 16:59:20 +0000345 def test_islower(self):
Walter Dörwald0fd583c2003-02-21 12:53:50 +0000346 string_tests.MixinStrUnicodeUserStringTest.test_islower(self)
Guido van Rossumef87d6e2007-05-02 19:09:54 +0000347 self.checkequalnofix(False, '\u1FFc', 'islower')
Walter Dörwald28256f22003-01-19 16:59:20 +0000348
349 def test_isupper(self):
Walter Dörwald0fd583c2003-02-21 12:53:50 +0000350 string_tests.MixinStrUnicodeUserStringTest.test_isupper(self)
351 if not sys.platform.startswith('java'):
Guido van Rossumef87d6e2007-05-02 19:09:54 +0000352 self.checkequalnofix(False, '\u1FFc', 'isupper')
Walter Dörwald28256f22003-01-19 16:59:20 +0000353
354 def test_istitle(self):
Walter Dörwald0fd583c2003-02-21 12:53:50 +0000355 string_tests.MixinStrUnicodeUserStringTest.test_title(self)
Guido van Rossumef87d6e2007-05-02 19:09:54 +0000356 self.checkequalnofix(True, '\u1FFc', 'istitle')
357 self.checkequalnofix(True, 'Greek \u1FFcitlecases ...', 'istitle')
Walter Dörwald28256f22003-01-19 16:59:20 +0000358
359 def test_isspace(self):
Walter Dörwald0fd583c2003-02-21 12:53:50 +0000360 string_tests.MixinStrUnicodeUserStringTest.test_isspace(self)
Guido van Rossumef87d6e2007-05-02 19:09:54 +0000361 self.checkequalnofix(True, '\u2000', 'isspace')
362 self.checkequalnofix(True, '\u200a', 'isspace')
363 self.checkequalnofix(False, '\u2014', 'isspace')
Walter Dörwald28256f22003-01-19 16:59:20 +0000364
365 def test_isalpha(self):
Walter Dörwald0fd583c2003-02-21 12:53:50 +0000366 string_tests.MixinStrUnicodeUserStringTest.test_isalpha(self)
Guido van Rossumef87d6e2007-05-02 19:09:54 +0000367 self.checkequalnofix(True, '\u1FFc', 'isalpha')
Walter Dörwald28256f22003-01-19 16:59:20 +0000368
369 def test_isdecimal(self):
Guido van Rossumef87d6e2007-05-02 19:09:54 +0000370 self.checkequalnofix(False, '', 'isdecimal')
371 self.checkequalnofix(False, 'a', 'isdecimal')
372 self.checkequalnofix(True, '0', 'isdecimal')
373 self.checkequalnofix(False, '\u2460', 'isdecimal') # CIRCLED DIGIT ONE
374 self.checkequalnofix(False, '\xbc', 'isdecimal') # VULGAR FRACTION ONE QUARTER
375 self.checkequalnofix(True, '\u0660', 'isdecimal') # ARABIC-INDIC DIGIT ZERO
376 self.checkequalnofix(True, '0123456789', 'isdecimal')
377 self.checkequalnofix(False, '0123456789a', 'isdecimal')
Walter Dörwald28256f22003-01-19 16:59:20 +0000378
Walter Dörwald0fd583c2003-02-21 12:53:50 +0000379 self.checkraises(TypeError, 'abc', 'isdecimal', 42)
Walter Dörwald28256f22003-01-19 16:59:20 +0000380
381 def test_isdigit(self):
Walter Dörwald0fd583c2003-02-21 12:53:50 +0000382 string_tests.MixinStrUnicodeUserStringTest.test_isdigit(self)
Guido van Rossumef87d6e2007-05-02 19:09:54 +0000383 self.checkequalnofix(True, '\u2460', 'isdigit')
384 self.checkequalnofix(False, '\xbc', 'isdigit')
385 self.checkequalnofix(True, '\u0660', 'isdigit')
Walter Dörwald28256f22003-01-19 16:59:20 +0000386
387 def test_isnumeric(self):
Guido van Rossumef87d6e2007-05-02 19:09:54 +0000388 self.checkequalnofix(False, '', 'isnumeric')
389 self.checkequalnofix(False, 'a', 'isnumeric')
390 self.checkequalnofix(True, '0', 'isnumeric')
391 self.checkequalnofix(True, '\u2460', 'isnumeric')
392 self.checkequalnofix(True, '\xbc', 'isnumeric')
393 self.checkequalnofix(True, '\u0660', 'isnumeric')
394 self.checkequalnofix(True, '0123456789', 'isnumeric')
395 self.checkequalnofix(False, '0123456789a', 'isnumeric')
Walter Dörwald28256f22003-01-19 16:59:20 +0000396
Guido van Rossumef87d6e2007-05-02 19:09:54 +0000397 self.assertRaises(TypeError, "abc".isnumeric, 42)
Walter Dörwald28256f22003-01-19 16:59:20 +0000398
Martin v. Löwis47383402007-08-15 07:32:56 +0000399 def test_isidentifier(self):
400 self.assertTrue("a".isidentifier())
401 self.assertTrue("Z".isidentifier())
402 self.assertTrue("_".isidentifier())
403 self.assertTrue("b0".isidentifier())
404 self.assertTrue("bc".isidentifier())
405 self.assertTrue("b_".isidentifier())
Antoine Pitroud72402e2010-10-27 18:52:48 +0000406 self.assertTrue("µ".isidentifier())
Martin v. Löwis47383402007-08-15 07:32:56 +0000407
408 self.assertFalse(" ".isidentifier())
409 self.assertFalse("[".isidentifier())
Antoine Pitroud72402e2010-10-27 18:52:48 +0000410 self.assertFalse("©".isidentifier())
Georg Brandld52429f2008-07-04 15:55:02 +0000411 self.assertFalse("0".isidentifier())
Martin v. Löwis47383402007-08-15 07:32:56 +0000412
Georg Brandl559e5d72008-06-11 18:37:52 +0000413 def test_isprintable(self):
414 self.assertTrue("".isprintable())
Benjamin Peterson09832742009-03-26 17:15:46 +0000415 self.assertTrue(" ".isprintable())
Georg Brandl559e5d72008-06-11 18:37:52 +0000416 self.assertTrue("abcdefg".isprintable())
417 self.assertFalse("abcdefg\n".isprintable())
Georg Brandld52429f2008-07-04 15:55:02 +0000418 # some defined Unicode character
419 self.assertTrue("\u0374".isprintable())
420 # undefined character
Amaury Forgeot d'Arca083f1e2008-09-10 23:51:42 +0000421 self.assertFalse("\u0378".isprintable())
Georg Brandld52429f2008-07-04 15:55:02 +0000422 # single surrogate character
Georg Brandl559e5d72008-06-11 18:37:52 +0000423 self.assertFalse("\ud800".isprintable())
424
Walter Dörwald28256f22003-01-19 16:59:20 +0000425 def test_contains(self):
426 # Testing Unicode contains method
Benjamin Peterson577473f2010-01-19 00:09:57 +0000427 self.assertIn('a', 'abdb')
428 self.assertIn('a', 'bdab')
429 self.assertIn('a', 'bdaba')
430 self.assertIn('a', 'bdba')
431 self.assertNotIn('a', 'bdb')
432 self.assertIn('a', 'bdba')
433 self.assertIn('a', ('a',1,None))
434 self.assertIn('a', (1,None,'a'))
435 self.assertIn('a', ('a',1,None))
436 self.assertIn('a', (1,None,'a'))
437 self.assertNotIn('a', ('x',1,'y'))
438 self.assertNotIn('a', ('x',1,None))
439 self.assertNotIn('abcd', 'abcxxxx')
440 self.assertIn('ab', 'abcd')
441 self.assertIn('ab', 'abc')
442 self.assertIn('ab', (1,None,'ab'))
443 self.assertIn('', 'abc')
444 self.assertIn('', '')
445 self.assertIn('', 'abc')
446 self.assertNotIn('\0', 'abc')
447 self.assertIn('\0', '\0abc')
448 self.assertIn('\0', 'abc\0')
449 self.assertIn('a', '\0abc')
450 self.assertIn('asdf', 'asdf')
451 self.assertNotIn('asdf', 'asd')
452 self.assertNotIn('asdf', '')
Walter Dörwald28256f22003-01-19 16:59:20 +0000453
Guido van Rossumef87d6e2007-05-02 19:09:54 +0000454 self.assertRaises(TypeError, "abc".__contains__)
Walter Dörwald28256f22003-01-19 16:59:20 +0000455
Eric Smith8c663262007-08-25 02:26:07 +0000456 def test_format(self):
457 self.assertEqual(''.format(), '')
458 self.assertEqual('a'.format(), 'a')
459 self.assertEqual('ab'.format(), 'ab')
460 self.assertEqual('a{{'.format(), 'a{')
461 self.assertEqual('a}}'.format(), 'a}')
462 self.assertEqual('{{b'.format(), '{b')
463 self.assertEqual('}}b'.format(), '}b')
464 self.assertEqual('a{{b'.format(), 'a{b')
465
466 # examples from the PEP:
467 import datetime
468 self.assertEqual("My name is {0}".format('Fred'), "My name is Fred")
469 self.assertEqual("My name is {0[name]}".format(dict(name='Fred')),
470 "My name is Fred")
471 self.assertEqual("My name is {0} :-{{}}".format('Fred'),
472 "My name is Fred :-{}")
473
474 d = datetime.date(2007, 8, 18)
475 self.assertEqual("The year is {0.year}".format(d),
476 "The year is 2007")
477
Eric Smith8c663262007-08-25 02:26:07 +0000478 # classes we'll use for testing
479 class C:
480 def __init__(self, x=100):
481 self._x = x
482 def __format__(self, spec):
483 return spec
484
485 class D:
486 def __init__(self, x):
487 self.x = x
488 def __format__(self, spec):
489 return str(self.x)
490
491 # class with __str__, but no __format__
492 class E:
493 def __init__(self, x):
494 self.x = x
495 def __str__(self):
496 return 'E(' + self.x + ')'
497
498 # class with __repr__, but no __format__ or __str__
499 class F:
500 def __init__(self, x):
501 self.x = x
502 def __repr__(self):
503 return 'F(' + self.x + ')'
504
505 # class with __format__ that forwards to string, for some format_spec's
506 class G:
507 def __init__(self, x):
508 self.x = x
509 def __str__(self):
510 return "string is " + self.x
511 def __format__(self, format_spec):
512 if format_spec == 'd':
513 return 'G(' + self.x + ')'
514 return object.__format__(self, format_spec)
515
Eric Smith739e2ad2007-08-27 19:07:22 +0000516 class I(datetime.date):
517 def __format__(self, format_spec):
518 return self.strftime(format_spec)
519
Eric Smith185e30c2007-08-30 22:23:08 +0000520 class J(int):
521 def __format__(self, format_spec):
522 return int.__format__(self * 2, format_spec)
523
Eric Smith8c663262007-08-25 02:26:07 +0000524
525 self.assertEqual(''.format(), '')
526 self.assertEqual('abc'.format(), 'abc')
527 self.assertEqual('{0}'.format('abc'), 'abc')
528 self.assertEqual('{0:}'.format('abc'), 'abc')
529# self.assertEqual('{ 0 }'.format('abc'), 'abc')
530 self.assertEqual('X{0}'.format('abc'), 'Xabc')
531 self.assertEqual('{0}X'.format('abc'), 'abcX')
532 self.assertEqual('X{0}Y'.format('abc'), 'XabcY')
533 self.assertEqual('{1}'.format(1, 'abc'), 'abc')
534 self.assertEqual('X{1}'.format(1, 'abc'), 'Xabc')
535 self.assertEqual('{1}X'.format(1, 'abc'), 'abcX')
536 self.assertEqual('X{1}Y'.format(1, 'abc'), 'XabcY')
537 self.assertEqual('{0}'.format(-15), '-15')
538 self.assertEqual('{0}{1}'.format(-15, 'abc'), '-15abc')
539 self.assertEqual('{0}X{1}'.format(-15, 'abc'), '-15Xabc')
540 self.assertEqual('{{'.format(), '{')
541 self.assertEqual('}}'.format(), '}')
542 self.assertEqual('{{}}'.format(), '{}')
543 self.assertEqual('{{x}}'.format(), '{x}')
544 self.assertEqual('{{{0}}}'.format(123), '{123}')
545 self.assertEqual('{{{{0}}}}'.format(), '{{0}}')
546 self.assertEqual('}}{{'.format(), '}{')
547 self.assertEqual('}}x{{'.format(), '}x{')
548
Eric Smith7ade6482007-08-26 22:27:13 +0000549 # weird field names
550 self.assertEqual("{0[foo-bar]}".format({'foo-bar':'baz'}), 'baz')
551 self.assertEqual("{0[foo bar]}".format({'foo bar':'baz'}), 'baz')
Eric Smith4cb4e4e2007-09-03 08:40:29 +0000552 self.assertEqual("{0[ ]}".format({' ':3}), '3')
Eric Smith7ade6482007-08-26 22:27:13 +0000553
Eric Smith8c663262007-08-25 02:26:07 +0000554 self.assertEqual('{foo._x}'.format(foo=C(20)), '20')
555 self.assertEqual('{1}{0}'.format(D(10), D(20)), '2010')
556 self.assertEqual('{0._x.x}'.format(C(D('abc'))), 'abc')
557 self.assertEqual('{0[0]}'.format(['abc', 'def']), 'abc')
558 self.assertEqual('{0[1]}'.format(['abc', 'def']), 'def')
559 self.assertEqual('{0[1][0]}'.format(['abc', ['def']]), 'def')
560 self.assertEqual('{0[1][0].x}'.format(['abc', [D('def')]]), 'def')
561
Eric Smith8c663262007-08-25 02:26:07 +0000562 # strings
563 self.assertEqual('{0:.3s}'.format('abc'), 'abc')
564 self.assertEqual('{0:.3s}'.format('ab'), 'ab')
565 self.assertEqual('{0:.3s}'.format('abcdef'), 'abc')
566 self.assertEqual('{0:.0s}'.format('abcdef'), '')
567 self.assertEqual('{0:3.3s}'.format('abc'), 'abc')
568 self.assertEqual('{0:2.3s}'.format('abc'), 'abc')
569 self.assertEqual('{0:2.2s}'.format('abc'), 'ab')
570 self.assertEqual('{0:3.2s}'.format('abc'), 'ab ')
571 self.assertEqual('{0:x<0s}'.format('result'), 'result')
572 self.assertEqual('{0:x<5s}'.format('result'), 'result')
573 self.assertEqual('{0:x<6s}'.format('result'), 'result')
574 self.assertEqual('{0:x<7s}'.format('result'), 'resultx')
575 self.assertEqual('{0:x<8s}'.format('result'), 'resultxx')
576 self.assertEqual('{0: <7s}'.format('result'), 'result ')
577 self.assertEqual('{0:<7s}'.format('result'), 'result ')
578 self.assertEqual('{0:>7s}'.format('result'), ' result')
579 self.assertEqual('{0:>8s}'.format('result'), ' result')
580 self.assertEqual('{0:^8s}'.format('result'), ' result ')
581 self.assertEqual('{0:^9s}'.format('result'), ' result ')
582 self.assertEqual('{0:^10s}'.format('result'), ' result ')
583 self.assertEqual('{0:10000}'.format('a'), 'a' + ' ' * 9999)
584 self.assertEqual('{0:10000}'.format(''), ' ' * 10000)
585 self.assertEqual('{0:10000000}'.format(''), ' ' * 10000000)
586
587 # format specifiers for user defined type
588 self.assertEqual('{0:abc}'.format(C()), 'abc')
589
Georg Brandld52429f2008-07-04 15:55:02 +0000590 # !r, !s and !a coercions
Eric Smith8c663262007-08-25 02:26:07 +0000591 self.assertEqual('{0!s}'.format('Hello'), 'Hello')
592 self.assertEqual('{0!s:}'.format('Hello'), 'Hello')
593 self.assertEqual('{0!s:15}'.format('Hello'), 'Hello ')
594 self.assertEqual('{0!s:15s}'.format('Hello'), 'Hello ')
595 self.assertEqual('{0!r}'.format('Hello'), "'Hello'")
596 self.assertEqual('{0!r:}'.format('Hello'), "'Hello'")
597 self.assertEqual('{0!r}'.format(F('Hello')), 'F(Hello)')
Amaury Forgeot d'Arca083f1e2008-09-10 23:51:42 +0000598 self.assertEqual('{0!r}'.format('\u0378'), "'\\u0378'") # nonprintable
Georg Brandld52429f2008-07-04 15:55:02 +0000599 self.assertEqual('{0!r}'.format('\u0374'), "'\u0374'") # printable
600 self.assertEqual('{0!r}'.format(F('\u0374')), 'F(\u0374)')
Georg Brandl559e5d72008-06-11 18:37:52 +0000601 self.assertEqual('{0!a}'.format('Hello'), "'Hello'")
Amaury Forgeot d'Arca083f1e2008-09-10 23:51:42 +0000602 self.assertEqual('{0!a}'.format('\u0378'), "'\\u0378'") # nonprintable
Georg Brandld52429f2008-07-04 15:55:02 +0000603 self.assertEqual('{0!a}'.format('\u0374'), "'\\u0374'") # printable
Georg Brandl559e5d72008-06-11 18:37:52 +0000604 self.assertEqual('{0!a:}'.format('Hello'), "'Hello'")
605 self.assertEqual('{0!a}'.format(F('Hello')), 'F(Hello)')
Georg Brandld52429f2008-07-04 15:55:02 +0000606 self.assertEqual('{0!a}'.format(F('\u0374')), 'F(\\u0374)')
Eric Smith8c663262007-08-25 02:26:07 +0000607
Eric Smith8c663262007-08-25 02:26:07 +0000608 # test fallback to object.__format__
609 self.assertEqual('{0}'.format({}), '{}')
610 self.assertEqual('{0}'.format([]), '[]')
611 self.assertEqual('{0}'.format([1]), '[1]')
Eric Smithe4d63172010-09-13 20:48:43 +0000612
Eric Smith8c663262007-08-25 02:26:07 +0000613 self.assertEqual('{0:d}'.format(G('data')), 'G(data)')
Eric Smith8c663262007-08-25 02:26:07 +0000614 self.assertEqual('{0!s}'.format(G('data')), 'string is data')
615
Eric Smithe4d63172010-09-13 20:48:43 +0000616 msg = 'object.__format__ with a non-empty format string is deprecated'
617 with support.check_warnings((msg, PendingDeprecationWarning)):
618 self.assertEqual('{0:^10}'.format(E('data')), ' E(data) ')
619 self.assertEqual('{0:^10s}'.format(E('data')), ' E(data) ')
620 self.assertEqual('{0:>15s}'.format(G('data')), ' string is data')
621
Eric Smith739e2ad2007-08-27 19:07:22 +0000622 self.assertEqual("{0:date: %Y-%m-%d}".format(I(year=2007,
623 month=8,
624 day=27)),
625 "date: 2007-08-27")
626
Eric Smith185e30c2007-08-30 22:23:08 +0000627 # test deriving from a builtin type and overriding __format__
628 self.assertEqual("{0}".format(J(10)), "20")
629
630
Eric Smith8c663262007-08-25 02:26:07 +0000631 # string format specifiers
632 self.assertEqual('{0:}'.format('a'), 'a')
633
634 # computed format specifiers
635 self.assertEqual("{0:.{1}}".format('hello world', 5), 'hello')
636 self.assertEqual("{0:.{1}s}".format('hello world', 5), 'hello')
637 self.assertEqual("{0:.{precision}s}".format('hello world', precision=5), 'hello')
638 self.assertEqual("{0:{width}.{precision}s}".format('hello world', width=10, precision=5), 'hello ')
639 self.assertEqual("{0:{width}.{precision}s}".format('hello world', width='10', precision='5'), 'hello ')
640
641 # test various errors
642 self.assertRaises(ValueError, '{'.format)
643 self.assertRaises(ValueError, '}'.format)
644 self.assertRaises(ValueError, 'a{'.format)
645 self.assertRaises(ValueError, 'a}'.format)
646 self.assertRaises(ValueError, '{a'.format)
647 self.assertRaises(ValueError, '}a'.format)
Eric Smith11529192007-09-04 23:04:22 +0000648 self.assertRaises(IndexError, '{0}'.format)
649 self.assertRaises(IndexError, '{1}'.format, 'abc')
650 self.assertRaises(KeyError, '{x}'.format)
Eric Smith8c663262007-08-25 02:26:07 +0000651 self.assertRaises(ValueError, "}{".format)
Eric Smith8c663262007-08-25 02:26:07 +0000652 self.assertRaises(ValueError, "abc{0:{}".format)
653 self.assertRaises(ValueError, "{0".format)
Eric Smith11529192007-09-04 23:04:22 +0000654 self.assertRaises(IndexError, "{0.}".format)
655 self.assertRaises(ValueError, "{0.}".format, 0)
656 self.assertRaises(IndexError, "{0[}".format)
Eric Smith4cb4e4e2007-09-03 08:40:29 +0000657 self.assertRaises(ValueError, "{0[}".format, [])
Eric Smith11529192007-09-04 23:04:22 +0000658 self.assertRaises(KeyError, "{0]}".format)
659 self.assertRaises(ValueError, "{0.[]}".format, 0)
Eric Smith7ade6482007-08-26 22:27:13 +0000660 self.assertRaises(ValueError, "{0..foo}".format, 0)
Eric Smith11529192007-09-04 23:04:22 +0000661 self.assertRaises(ValueError, "{0[0}".format, 0)
662 self.assertRaises(ValueError, "{0[0:foo}".format, 0)
663 self.assertRaises(KeyError, "{c]}".format)
664 self.assertRaises(ValueError, "{{ {{{0}}".format, 0)
665 self.assertRaises(ValueError, "{0}}".format, 0)
666 self.assertRaises(KeyError, "{foo}".format, bar=3)
Eric Smith8c663262007-08-25 02:26:07 +0000667 self.assertRaises(ValueError, "{0!x}".format, 3)
Eric Smith11529192007-09-04 23:04:22 +0000668 self.assertRaises(ValueError, "{0!}".format, 0)
669 self.assertRaises(ValueError, "{0!rs}".format, 0)
Eric Smith8c663262007-08-25 02:26:07 +0000670 self.assertRaises(ValueError, "{!}".format)
Eric Smith8ec90442009-03-14 12:29:34 +0000671 self.assertRaises(IndexError, "{:}".format)
672 self.assertRaises(IndexError, "{:s}".format)
673 self.assertRaises(IndexError, "{}".format)
Benjamin Peterson59a1b2f2010-06-07 22:31:26 +0000674 big = "23098475029384702983476098230754973209482573"
675 self.assertRaises(ValueError, ("{" + big + "}").format)
676 self.assertRaises(ValueError, ("{[" + big + "]}").format, [0])
Eric Smith8c663262007-08-25 02:26:07 +0000677
Eric Smith41669ca2009-05-23 14:23:22 +0000678 # issue 6089
679 self.assertRaises(ValueError, "{0[0]x}".format, [None])
680 self.assertRaises(ValueError, "{0[0](10)}".format, [None])
681
Eric Smith8c663262007-08-25 02:26:07 +0000682 # can't have a replacement on the field name portion
683 self.assertRaises(TypeError, '{0[{1}]}'.format, 'abcdefg', 4)
684
685 # exceed maximum recursion depth
686 self.assertRaises(ValueError, "{0:{1:{2}}}".format, 'abc', 's', '')
687 self.assertRaises(ValueError, "{0:{1:{2:{3:{4:{5:{6}}}}}}}".format,
688 0, 1, 2, 3, 4, 5, 6, 7)
689
690 # string format spec errors
691 self.assertRaises(ValueError, "{0:-s}".format, '')
692 self.assertRaises(ValueError, format, "", "-")
693 self.assertRaises(ValueError, "{0:=s}".format, '')
694
Eric Smithb1ebcc62008-07-15 13:02:41 +0000695 # Alternate formatting is not supported
696 self.assertRaises(ValueError, format, '', '#')
697 self.assertRaises(ValueError, format, '', '#20')
698
Eric Smith27bbca62010-11-04 17:06:58 +0000699 def test_format_map(self):
700 self.assertEqual(''.format_map({}), '')
701 self.assertEqual('a'.format_map({}), 'a')
702 self.assertEqual('ab'.format_map({}), 'ab')
703 self.assertEqual('a{{'.format_map({}), 'a{')
704 self.assertEqual('a}}'.format_map({}), 'a}')
705 self.assertEqual('{{b'.format_map({}), '{b')
706 self.assertEqual('}}b'.format_map({}), '}b')
707 self.assertEqual('a{{b'.format_map({}), 'a{b')
708
709 # using mappings
710 class Mapping(dict):
711 def __missing__(self, key):
712 return key
713 self.assertEqual('{hello}'.format_map(Mapping()), 'hello')
714 self.assertEqual('{a} {world}'.format_map(Mapping(a='hello')), 'hello world')
715
716 class InternalMapping:
717 def __init__(self):
718 self.mapping = {'a': 'hello'}
719 def __getitem__(self, key):
720 return self.mapping[key]
721 self.assertEqual('{a}'.format_map(InternalMapping()), 'hello')
722
723
Eric Smith27bbca62010-11-04 17:06:58 +0000724 class C:
725 def __init__(self, x=100):
726 self._x = x
727 def __format__(self, spec):
728 return spec
Eric Smith27bbca62010-11-04 17:06:58 +0000729 self.assertEqual('{foo._x}'.format_map({'foo': C(20)}), '20')
730
731 # test various errors
732 self.assertRaises(TypeError, '{'.format_map)
733 self.assertRaises(TypeError, '}'.format_map)
734 self.assertRaises(TypeError, 'a{'.format_map)
735 self.assertRaises(TypeError, 'a}'.format_map)
736 self.assertRaises(TypeError, '{a'.format_map)
737 self.assertRaises(TypeError, '}a'.format_map)
738
Eric Smith8ec90442009-03-14 12:29:34 +0000739 def test_format_auto_numbering(self):
740 class C:
741 def __init__(self, x=100):
742 self._x = x
743 def __format__(self, spec):
744 return spec
745
746 self.assertEqual('{}'.format(10), '10')
747 self.assertEqual('{:5}'.format('s'), 's ')
748 self.assertEqual('{!r}'.format('s'), "'s'")
749 self.assertEqual('{._x}'.format(C(10)), '10')
750 self.assertEqual('{[1]}'.format([1, 2]), '2')
751 self.assertEqual('{[a]}'.format({'a':4, 'b':2}), '4')
752 self.assertEqual('a{}b{}c'.format(0, 1), 'a0b1c')
753
754 self.assertEqual('a{:{}}b'.format('x', '^10'), 'a x b')
755 self.assertEqual('a{:{}x}b'.format(20, '#'), 'a0x14b')
756
757 # can't mix and match numbering and auto-numbering
758 self.assertRaises(ValueError, '{}{1}'.format, 1, 2)
759 self.assertRaises(ValueError, '{1}{}'.format, 1, 2)
760 self.assertRaises(ValueError, '{:{1}}'.format, 1, 2)
761 self.assertRaises(ValueError, '{0:{}}'.format, 1, 2)
762
763 # can mix and match auto-numbering and named
764 self.assertEqual('{f}{}'.format(4, f='test'), 'test4')
765 self.assertEqual('{}{f}'.format(4, f='test'), '4test')
766 self.assertEqual('{:{f}}{g}{}'.format(1, 3, g='g', f=2), ' 1g3')
767 self.assertEqual('{f:{}}{}{g}'.format(2, 4, f=1, g='g'), ' 14g')
768
Walter Dörwald28256f22003-01-19 16:59:20 +0000769 def test_formatting(self):
Walter Dörwald0fd583c2003-02-21 12:53:50 +0000770 string_tests.MixinStrUnicodeUserStringTest.test_formatting(self)
Walter Dörwald28256f22003-01-19 16:59:20 +0000771 # Testing Unicode formatting strings...
Guido van Rossumef87d6e2007-05-02 19:09:54 +0000772 self.assertEqual("%s, %s" % ("abc", "abc"), 'abc, abc')
773 self.assertEqual("%s, %s, %i, %f, %5.2f" % ("abc", "abc", 1, 2, 3), 'abc, abc, 1, 2.000000, 3.00')
774 self.assertEqual("%s, %s, %i, %f, %5.2f" % ("abc", "abc", 1, -2, 3), 'abc, abc, 1, -2.000000, 3.00')
775 self.assertEqual("%s, %s, %i, %f, %5.2f" % ("abc", "abc", -1, -2, 3.5), 'abc, abc, -1, -2.000000, 3.50')
776 self.assertEqual("%s, %s, %i, %f, %5.2f" % ("abc", "abc", -1, -2, 3.57), 'abc, abc, -1, -2.000000, 3.57')
777 self.assertEqual("%s, %s, %i, %f, %5.2f" % ("abc", "abc", -1, -2, 1003.57), 'abc, abc, -1, -2.000000, 1003.57')
Walter Dörwald28256f22003-01-19 16:59:20 +0000778 if not sys.platform.startswith('java'):
Walter Dörwald67e83882007-05-05 12:26:27 +0000779 self.assertEqual("%r, %r" % (b"abc", "abc"), "b'abc', 'abc'")
Georg Brandl559e5d72008-06-11 18:37:52 +0000780 self.assertEqual("%r" % ("\u1234",), "'\u1234'")
781 self.assertEqual("%a" % ("\u1234",), "'\\u1234'")
Guido van Rossumef87d6e2007-05-02 19:09:54 +0000782 self.assertEqual("%(x)s, %(y)s" % {'x':"abc", 'y':"def"}, 'abc, def')
783 self.assertEqual("%(x)s, %(\xfc)s" % {'x':"abc", '\xfc':"def"}, 'abc, def')
Walter Dörwald56fbcb52003-03-31 18:18:41 +0000784
Guido van Rossumef87d6e2007-05-02 19:09:54 +0000785 self.assertEqual('%c' % 0x1234, '\u1234')
Amaury Forgeot d'Arca4db6862008-07-04 21:26:43 +0000786 self.assertEqual('%c' % 0x21483, '\U00021483')
787 self.assertRaises(OverflowError, "%c".__mod__, (0x110000,))
788 self.assertEqual('%c' % '\U00021483', '\U00021483')
789 self.assertRaises(TypeError, "%c".__mod__, "aa")
Stefan Krah99212f62010-07-19 17:58:26 +0000790 self.assertRaises(ValueError, "%.1\u1032f".__mod__, (1.0/3))
Senthil Kumaran9ebe08d2011-07-03 21:03:16 -0700791 self.assertRaises(TypeError, "%i".__mod__, "aa")
Walter Dörwald28256f22003-01-19 16:59:20 +0000792
793 # formatting jobs delegated from the string implementation:
Walter Dörwald28256f22003-01-19 16:59:20 +0000794 self.assertEqual('...%(foo)s...' % {'foo':"abc"}, '...abc...')
Guido van Rossumef87d6e2007-05-02 19:09:54 +0000795 self.assertEqual('...%(foo)s...' % {'foo':"abc"}, '...abc...')
796 self.assertEqual('...%(foo)s...' % {'foo':"abc"}, '...abc...')
797 self.assertEqual('...%(foo)s...' % {'foo':"abc"}, '...abc...')
798 self.assertEqual('...%(foo)s...' % {'foo':"abc",'def':123}, '...abc...')
799 self.assertEqual('...%(foo)s...' % {'foo':"abc",'def':123}, '...abc...')
800 self.assertEqual('...%s...%s...%s...%s...' % (1,2,3,"abc"), '...1...2...3...abc...')
801 self.assertEqual('...%%...%%s...%s...%s...%s...%s...' % (1,2,3,"abc"), '...%...%s...1...2...3...abc...')
802 self.assertEqual('...%s...' % "abc", '...abc...')
803 self.assertEqual('%*s' % (5,'abc',), ' abc')
804 self.assertEqual('%*s' % (-5,'abc',), 'abc ')
805 self.assertEqual('%*.*s' % (5,2,'abc',), ' ab')
806 self.assertEqual('%*.*s' % (5,3,'abc',), ' abc')
807 self.assertEqual('%i %*.*s' % (10, 5,3,'abc',), '10 abc')
808 self.assertEqual('%i%s %*.*s' % (10, 3, 5, 3, 'abc',), '103 abc')
809 self.assertEqual('%c' % 'a', 'a')
Neil Schemenauercf52c072005-08-12 17:34:58 +0000810 class Wrapper:
811 def __str__(self):
Guido van Rossumef87d6e2007-05-02 19:09:54 +0000812 return '\u1234'
813 self.assertEqual('%s' % Wrapper(), '\u1234')
Walter Dörwald28256f22003-01-19 16:59:20 +0000814
Eric Smith741191f2009-05-06 13:08:15 +0000815 # issue 3382
816 NAN = float('nan')
817 INF = float('inf')
818 self.assertEqual('%f' % NAN, 'nan')
819 self.assertEqual('%F' % NAN, 'NAN')
820 self.assertEqual('%f' % INF, 'inf')
821 self.assertEqual('%F' % INF, 'INF')
822
Ezio Melottiba42fd52011-04-26 06:09:45 +0300823 def test_startswith_endswith_errors(self):
824 for meth in ('foo'.startswith, 'foo'.endswith):
Ezio Melottif2b3f782011-04-26 06:40:59 +0300825 with self.assertRaises(TypeError) as cm:
Ezio Melottiba42fd52011-04-26 06:09:45 +0300826 meth(['f'])
Ezio Melottif2b3f782011-04-26 06:40:59 +0300827 exc = str(cm.exception)
Ezio Melottiba42fd52011-04-26 06:09:45 +0300828 self.assertIn('str', exc)
829 self.assertIn('tuple', exc)
830
Benjamin Petersonee8712c2008-05-20 21:35:26 +0000831 @support.run_with_locale('LC_ALL', 'de_DE', 'fr_FR')
Georg Brandlda6b1072006-01-20 17:48:54 +0000832 def test_format_float(self):
Thomas Wouters477c8d52006-05-27 19:21:47 +0000833 # should not format with a comma, but always with C locale
Guido van Rossumef87d6e2007-05-02 19:09:54 +0000834 self.assertEqual('1.0', '%.1f' % 1.0)
Georg Brandlda6b1072006-01-20 17:48:54 +0000835
Walter Dörwald28256f22003-01-19 16:59:20 +0000836 def test_constructor(self):
837 # unicode(obj) tests (this maps to PyObject_Unicode() at C level)
838
839 self.assertEqual(
Guido van Rossumef87d6e2007-05-02 19:09:54 +0000840 str('unicode remains unicode'),
841 'unicode remains unicode'
Walter Dörwald28256f22003-01-19 16:59:20 +0000842 )
843
Guido van Rossumef87d6e2007-05-02 19:09:54 +0000844 class UnicodeSubclass(str):
Marc-André Lemburg79f57832002-12-29 19:44:06 +0000845 pass
Guido van Rossuma831cac2000-03-10 23:23:21 +0000846
Walter Dörwald28256f22003-01-19 16:59:20 +0000847 self.assertEqual(
Guido van Rossumef87d6e2007-05-02 19:09:54 +0000848 str(UnicodeSubclass('unicode subclass becomes unicode')),
849 'unicode subclass becomes unicode'
Walter Dörwald28256f22003-01-19 16:59:20 +0000850 )
Marc-André Lemburgb5507ec2001-10-19 12:02:29 +0000851
Walter Dörwald28256f22003-01-19 16:59:20 +0000852 self.assertEqual(
Guido van Rossumef87d6e2007-05-02 19:09:54 +0000853 str('strings are converted to unicode'),
854 'strings are converted to unicode'
Walter Dörwald28256f22003-01-19 16:59:20 +0000855 )
Marc-André Lemburgb5507ec2001-10-19 12:02:29 +0000856
Walter Dörwald28256f22003-01-19 16:59:20 +0000857 class StringCompat:
858 def __init__(self, x):
859 self.x = x
860 def __str__(self):
861 return self.x
Marc-André Lemburgb5507ec2001-10-19 12:02:29 +0000862
Walter Dörwald28256f22003-01-19 16:59:20 +0000863 self.assertEqual(
Guido van Rossumef87d6e2007-05-02 19:09:54 +0000864 str(StringCompat('__str__ compatible objects are recognized')),
865 '__str__ compatible objects are recognized'
Walter Dörwald28256f22003-01-19 16:59:20 +0000866 )
Marc-André Lemburgb5507ec2001-10-19 12:02:29 +0000867
Walter Dörwald28256f22003-01-19 16:59:20 +0000868 # unicode(obj) is compatible to str():
Marc-André Lemburgb5507ec2001-10-19 12:02:29 +0000869
Walter Dörwald28256f22003-01-19 16:59:20 +0000870 o = StringCompat('unicode(obj) is compatible to str()')
Guido van Rossumef87d6e2007-05-02 19:09:54 +0000871 self.assertEqual(str(o), 'unicode(obj) is compatible to str()')
Walter Dörwald28256f22003-01-19 16:59:20 +0000872 self.assertEqual(str(o), 'unicode(obj) is compatible to str()')
Marc-André Lemburgb5507ec2001-10-19 12:02:29 +0000873
Guido van Rossume2a383d2007-01-15 16:59:06 +0000874 for obj in (123, 123.45, 123):
Guido van Rossumef87d6e2007-05-02 19:09:54 +0000875 self.assertEqual(str(obj), str(str(obj)))
Marc-André Lemburgb5507ec2001-10-19 12:02:29 +0000876
Walter Dörwald28256f22003-01-19 16:59:20 +0000877 # unicode(obj, encoding, error) tests (this maps to
878 # PyUnicode_FromEncodedObject() at C level)
Marc-André Lemburgb5507ec2001-10-19 12:02:29 +0000879
Walter Dörwald28256f22003-01-19 16:59:20 +0000880 if not sys.platform.startswith('java'):
881 self.assertRaises(
882 TypeError,
Guido van Rossumef87d6e2007-05-02 19:09:54 +0000883 str,
884 'decoding unicode is not supported',
Walter Dörwald28256f22003-01-19 16:59:20 +0000885 'utf-8',
886 'strict'
887 )
Marc-André Lemburgb5507ec2001-10-19 12:02:29 +0000888
Walter Dörwald28256f22003-01-19 16:59:20 +0000889 self.assertEqual(
Walter Dörwald67e83882007-05-05 12:26:27 +0000890 str(b'strings are decoded to unicode', 'utf-8', 'strict'),
Guido van Rossumef87d6e2007-05-02 19:09:54 +0000891 'strings are decoded to unicode'
Walter Dörwald28256f22003-01-19 16:59:20 +0000892 )
Marc-André Lemburgb5507ec2001-10-19 12:02:29 +0000893
Walter Dörwald28256f22003-01-19 16:59:20 +0000894 if not sys.platform.startswith('java'):
895 self.assertEqual(
Guido van Rossumef87d6e2007-05-02 19:09:54 +0000896 str(
Guido van Rossumbae07c92007-10-08 02:46:15 +0000897 memoryview(b'character buffers are decoded to unicode'),
Walter Dörwald28256f22003-01-19 16:59:20 +0000898 'utf-8',
899 'strict'
900 ),
Guido van Rossumef87d6e2007-05-02 19:09:54 +0000901 'character buffers are decoded to unicode'
Walter Dörwald28256f22003-01-19 16:59:20 +0000902 )
Marc-André Lemburgb5507ec2001-10-19 12:02:29 +0000903
Guido van Rossumef87d6e2007-05-02 19:09:54 +0000904 self.assertRaises(TypeError, str, 42, 42, 42)
Marc-André Lemburgb5507ec2001-10-19 12:02:29 +0000905
Walter Dörwald28256f22003-01-19 16:59:20 +0000906 def test_codecs_utf7(self):
907 utfTests = [
Walter Dörwald67e83882007-05-05 12:26:27 +0000908 ('A\u2262\u0391.', b'A+ImIDkQ.'), # RFC2152 example
909 ('Hi Mom -\u263a-!', b'Hi Mom -+Jjo--!'), # RFC2152 example
910 ('\u65E5\u672C\u8A9E', b'+ZeVnLIqe-'), # RFC2152 example
911 ('Item 3 is \u00a31.', b'Item 3 is +AKM-1.'), # RFC2152 example
912 ('+', b'+-'),
913 ('+-', b'+--'),
914 ('+?', b'+-?'),
915 ('\?', b'+AFw?'),
916 ('+?', b'+-?'),
917 (r'\\?', b'+AFwAXA?'),
918 (r'\\\?', b'+AFwAXABc?'),
Antoine Pitrou244651a2009-05-04 18:56:13 +0000919 (r'++--', b'+-+---'),
920 ('\U000abcde', b'+2m/c3g-'), # surrogate pairs
921 ('/', b'/'),
Walter Dörwald28256f22003-01-19 16:59:20 +0000922 ]
Marc-André Lemburgb5507ec2001-10-19 12:02:29 +0000923
Walter Dörwald28256f22003-01-19 16:59:20 +0000924 for (x, y) in utfTests:
925 self.assertEqual(x.encode('utf-7'), y)
Marc-André Lemburgb5507ec2001-10-19 12:02:29 +0000926
Antoine Pitrou244651a2009-05-04 18:56:13 +0000927 # Unpaired surrogates not supported
Walter Dörwald67e83882007-05-05 12:26:27 +0000928 self.assertRaises(UnicodeError, str, b'+3ADYAA-', 'utf-7')
Marc-André Lemburgb5507ec2001-10-19 12:02:29 +0000929
Antoine Pitrou244651a2009-05-04 18:56:13 +0000930 self.assertEqual(str(b'+3ADYAA-', 'utf-7', 'replace'), '\ufffd\ufffd')
Marc-André Lemburgb5507ec2001-10-19 12:02:29 +0000931
Antoine Pitrou5ffd9e92008-07-25 18:05:24 +0000932 # Issue #2242: crash on some Windows/MSVC versions
Antoine Pitrou244651a2009-05-04 18:56:13 +0000933 self.assertEqual(b'+\xc1'.decode('utf-7'), '\xc1')
934
935 # Direct encoded characters
936 set_d = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789'(),-./:?"
937 # Optional direct characters
938 set_o = '!"#$%&*;<=>@[]^_`{|}'
939 for c in set_d:
940 self.assertEqual(c.encode('utf7'), c.encode('ascii'))
941 self.assertEqual(c.encode('ascii').decode('utf7'), c)
942 for c in set_o:
943 self.assertEqual(c.encode('ascii').decode('utf7'), c)
Antoine Pitrou5ffd9e92008-07-25 18:05:24 +0000944
Walter Dörwald28256f22003-01-19 16:59:20 +0000945 def test_codecs_utf8(self):
Walter Dörwald67e83882007-05-05 12:26:27 +0000946 self.assertEqual(''.encode('utf-8'), b'')
947 self.assertEqual('\u20ac'.encode('utf-8'), b'\xe2\x82\xac')
Martin v. Löwis74b7e442009-06-01 04:23:07 +0000948 if sys.maxunicode == 65535:
949 self.assertEqual('\ud800\udc02'.encode('utf-8'), b'\xf0\x90\x80\x82')
950 self.assertEqual('\ud84d\udc56'.encode('utf-8'), b'\xf0\xa3\x91\x96')
Martin v. Löwise0a2b722009-05-10 08:08:56 +0000951 self.assertEqual('\ud800'.encode('utf-8', 'surrogatepass'), b'\xed\xa0\x80')
952 self.assertEqual('\udc00'.encode('utf-8', 'surrogatepass'), b'\xed\xb0\x80')
Martin v. Löwis74b7e442009-06-01 04:23:07 +0000953 if sys.maxunicode == 65535:
954 self.assertEqual(
955 ('\ud800\udc02'*1000).encode('utf-8'),
956 b'\xf0\x90\x80\x82'*1000)
Walter Dörwald28256f22003-01-19 16:59:20 +0000957 self.assertEqual(
Guido van Rossumef87d6e2007-05-02 19:09:54 +0000958 '\u6b63\u78ba\u306b\u8a00\u3046\u3068\u7ffb\u8a33\u306f'
959 '\u3055\u308c\u3066\u3044\u307e\u305b\u3093\u3002\u4e00'
960 '\u90e8\u306f\u30c9\u30a4\u30c4\u8a9e\u3067\u3059\u304c'
961 '\u3001\u3042\u3068\u306f\u3067\u305f\u3089\u3081\u3067'
962 '\u3059\u3002\u5b9f\u969b\u306b\u306f\u300cWenn ist das'
963 ' Nunstuck git und'.encode('utf-8'),
Walter Dörwald67e83882007-05-05 12:26:27 +0000964 b'\xe6\xad\xa3\xe7\xa2\xba\xe3\x81\xab\xe8\xa8\x80\xe3\x81'
965 b'\x86\xe3\x81\xa8\xe7\xbf\xbb\xe8\xa8\xb3\xe3\x81\xaf\xe3'
966 b'\x81\x95\xe3\x82\x8c\xe3\x81\xa6\xe3\x81\x84\xe3\x81\xbe'
967 b'\xe3\x81\x9b\xe3\x82\x93\xe3\x80\x82\xe4\xb8\x80\xe9\x83'
968 b'\xa8\xe3\x81\xaf\xe3\x83\x89\xe3\x82\xa4\xe3\x83\x84\xe8'
969 b'\xaa\x9e\xe3\x81\xa7\xe3\x81\x99\xe3\x81\x8c\xe3\x80\x81'
970 b'\xe3\x81\x82\xe3\x81\xa8\xe3\x81\xaf\xe3\x81\xa7\xe3\x81'
971 b'\x9f\xe3\x82\x89\xe3\x82\x81\xe3\x81\xa7\xe3\x81\x99\xe3'
972 b'\x80\x82\xe5\xae\x9f\xe9\x9a\x9b\xe3\x81\xab\xe3\x81\xaf'
973 b'\xe3\x80\x8cWenn ist das Nunstuck git und'
Walter Dörwald28256f22003-01-19 16:59:20 +0000974 )
Guido van Rossumd8855fd2000-03-24 22:14:19 +0000975
Walter Dörwald28256f22003-01-19 16:59:20 +0000976 # UTF-8 specific decoding tests
Walter Dörwald67e83882007-05-05 12:26:27 +0000977 self.assertEqual(str(b'\xf0\xa3\x91\x96', 'utf-8'), '\U00023456' )
978 self.assertEqual(str(b'\xf0\x90\x80\x82', 'utf-8'), '\U00010002' )
979 self.assertEqual(str(b'\xe2\x82\xac', 'utf-8'), '\u20ac' )
Marc-André Lemburgc60e6f72001-09-20 10:35:46 +0000980
Walter Dörwald28256f22003-01-19 16:59:20 +0000981 # Other possible utf-8 test cases:
982 # * strict decoding testing for all of the
983 # UTF8_ERROR cases in PyUnicode_DecodeUTF8
Marc-André Lemburgc60e6f72001-09-20 10:35:46 +0000984
Ezio Melotti57221d02010-07-01 07:32:02 +0000985 def test_utf8_decode_valid_sequences(self):
986 sequences = [
987 # single byte
988 (b'\x00', '\x00'), (b'a', 'a'), (b'\x7f', '\x7f'),
989 # 2 bytes
990 (b'\xc2\x80', '\x80'), (b'\xdf\xbf', '\u07ff'),
991 # 3 bytes
992 (b'\xe0\xa0\x80', '\u0800'), (b'\xed\x9f\xbf', '\ud7ff'),
993 (b'\xee\x80\x80', '\uE000'), (b'\xef\xbf\xbf', '\uffff'),
994 # 4 bytes
995 (b'\xF0\x90\x80\x80', '\U00010000'),
996 (b'\xf4\x8f\xbf\xbf', '\U0010FFFF')
997 ]
998 for seq, res in sequences:
999 self.assertEqual(seq.decode('utf-8'), res)
1000
1001
1002 def test_utf8_decode_invalid_sequences(self):
1003 # continuation bytes in a sequence of 2, 3, or 4 bytes
1004 continuation_bytes = [bytes([x]) for x in range(0x80, 0xC0)]
1005 # start bytes of a 2-byte sequence equivalent to codepoints < 0x7F
1006 invalid_2B_seq_start_bytes = [bytes([x]) for x in range(0xC0, 0xC2)]
1007 # start bytes of a 4-byte sequence equivalent to codepoints > 0x10FFFF
1008 invalid_4B_seq_start_bytes = [bytes([x]) for x in range(0xF5, 0xF8)]
1009 invalid_start_bytes = (
1010 continuation_bytes + invalid_2B_seq_start_bytes +
1011 invalid_4B_seq_start_bytes + [bytes([x]) for x in range(0xF7, 0x100)]
1012 )
1013
1014 for byte in invalid_start_bytes:
1015 self.assertRaises(UnicodeDecodeError, byte.decode, 'utf-8')
1016
1017 for sb in invalid_2B_seq_start_bytes:
1018 for cb in continuation_bytes:
1019 self.assertRaises(UnicodeDecodeError, (sb+cb).decode, 'utf-8')
1020
1021 for sb in invalid_4B_seq_start_bytes:
1022 for cb1 in continuation_bytes[:3]:
1023 for cb3 in continuation_bytes[:3]:
1024 self.assertRaises(UnicodeDecodeError,
1025 (sb+cb1+b'\x80'+cb3).decode, 'utf-8')
1026
1027 for cb in [bytes([x]) for x in range(0x80, 0xA0)]:
1028 self.assertRaises(UnicodeDecodeError,
1029 (b'\xE0'+cb+b'\x80').decode, 'utf-8')
1030 self.assertRaises(UnicodeDecodeError,
1031 (b'\xE0'+cb+b'\xBF').decode, 'utf-8')
1032 # surrogates
1033 for cb in [bytes([x]) for x in range(0xA0, 0xC0)]:
1034 self.assertRaises(UnicodeDecodeError,
1035 (b'\xED'+cb+b'\x80').decode, 'utf-8')
1036 self.assertRaises(UnicodeDecodeError,
1037 (b'\xED'+cb+b'\xBF').decode, 'utf-8')
1038 for cb in [bytes([x]) for x in range(0x80, 0x90)]:
1039 self.assertRaises(UnicodeDecodeError,
1040 (b'\xF0'+cb+b'\x80\x80').decode, 'utf-8')
1041 self.assertRaises(UnicodeDecodeError,
1042 (b'\xF0'+cb+b'\xBF\xBF').decode, 'utf-8')
1043 for cb in [bytes([x]) for x in range(0x90, 0xC0)]:
1044 self.assertRaises(UnicodeDecodeError,
1045 (b'\xF4'+cb+b'\x80\x80').decode, 'utf-8')
1046 self.assertRaises(UnicodeDecodeError,
1047 (b'\xF4'+cb+b'\xBF\xBF').decode, 'utf-8')
1048
1049 def test_issue8271(self):
1050 # Issue #8271: during the decoding of an invalid UTF-8 byte sequence,
1051 # only the start byte and the continuation byte(s) are now considered
1052 # invalid, instead of the number of bytes specified by the start byte.
1053 # See http://www.unicode.org/versions/Unicode5.2.0/ch03.pdf (page 95,
1054 # table 3-8, Row 2) for more information about the algorithm used.
1055 FFFD = '\ufffd'
1056 sequences = [
1057 # invalid start bytes
1058 (b'\x80', FFFD), # continuation byte
1059 (b'\x80\x80', FFFD*2), # 2 continuation bytes
1060 (b'\xc0', FFFD),
1061 (b'\xc0\xc0', FFFD*2),
1062 (b'\xc1', FFFD),
1063 (b'\xc1\xc0', FFFD*2),
1064 (b'\xc0\xc1', FFFD*2),
1065 # with start byte of a 2-byte sequence
1066 (b'\xc2', FFFD), # only the start byte
1067 (b'\xc2\xc2', FFFD*2), # 2 start bytes
1068 (b'\xc2\xc2\xc2', FFFD*3), # 2 start bytes
1069 (b'\xc2\x41', FFFD+'A'), # invalid continuation byte
1070 # with start byte of a 3-byte sequence
1071 (b'\xe1', FFFD), # only the start byte
1072 (b'\xe1\xe1', FFFD*2), # 2 start bytes
1073 (b'\xe1\xe1\xe1', FFFD*3), # 3 start bytes
1074 (b'\xe1\xe1\xe1\xe1', FFFD*4), # 4 start bytes
1075 (b'\xe1\x80', FFFD), # only 1 continuation byte
1076 (b'\xe1\x41', FFFD+'A'), # invalid continuation byte
1077 (b'\xe1\x41\x80', FFFD+'A'+FFFD), # invalid cb followed by valid cb
1078 (b'\xe1\x41\x41', FFFD+'AA'), # 2 invalid continuation bytes
1079 (b'\xe1\x80\x41', FFFD+'A'), # only 1 valid continuation byte
1080 (b'\xe1\x80\xe1\x41', FFFD*2+'A'), # 1 valid and the other invalid
1081 (b'\xe1\x41\xe1\x80', FFFD+'A'+FFFD), # 1 invalid and the other valid
1082 # with start byte of a 4-byte sequence
1083 (b'\xf1', FFFD), # only the start byte
1084 (b'\xf1\xf1', FFFD*2), # 2 start bytes
1085 (b'\xf1\xf1\xf1', FFFD*3), # 3 start bytes
1086 (b'\xf1\xf1\xf1\xf1', FFFD*4), # 4 start bytes
1087 (b'\xf1\xf1\xf1\xf1\xf1', FFFD*5), # 5 start bytes
1088 (b'\xf1\x80', FFFD), # only 1 continuation bytes
1089 (b'\xf1\x80\x80', FFFD), # only 2 continuation bytes
1090 (b'\xf1\x80\x41', FFFD+'A'), # 1 valid cb and 1 invalid
1091 (b'\xf1\x80\x41\x41', FFFD+'AA'), # 1 valid cb and 1 invalid
1092 (b'\xf1\x80\x80\x41', FFFD+'A'), # 2 valid cb and 1 invalid
1093 (b'\xf1\x41\x80', FFFD+'A'+FFFD), # 1 invalid cv and 1 valid
1094 (b'\xf1\x41\x80\x80', FFFD+'A'+FFFD*2), # 1 invalid cb and 2 invalid
1095 (b'\xf1\x41\x80\x41', FFFD+'A'+FFFD+'A'), # 2 invalid cb and 1 invalid
1096 (b'\xf1\x41\x41\x80', FFFD+'AA'+FFFD), # 1 valid cb and 1 invalid
1097 (b'\xf1\x41\xf1\x80', FFFD+'A'+FFFD),
1098 (b'\xf1\x41\x80\xf1', FFFD+'A'+FFFD*2),
1099 (b'\xf1\xf1\x80\x41', FFFD*2+'A'),
1100 (b'\xf1\x41\xf1\xf1', FFFD+'A'+FFFD*2),
1101 # with invalid start byte of a 4-byte sequence (rfc2279)
1102 (b'\xf5', FFFD), # only the start byte
1103 (b'\xf5\xf5', FFFD*2), # 2 start bytes
1104 (b'\xf5\x80', FFFD*2), # only 1 continuation byte
1105 (b'\xf5\x80\x80', FFFD*3), # only 2 continuation byte
1106 (b'\xf5\x80\x80\x80', FFFD*4), # 3 continuation bytes
1107 (b'\xf5\x80\x41', FFFD*2+'A'), # 1 valid cb and 1 invalid
1108 (b'\xf5\x80\x41\xf5', FFFD*2+'A'+FFFD),
1109 (b'\xf5\x41\x80\x80\x41', FFFD+'A'+FFFD*2+'A'),
1110 # with invalid start byte of a 5-byte sequence (rfc2279)
1111 (b'\xf8', FFFD), # only the start byte
1112 (b'\xf8\xf8', FFFD*2), # 2 start bytes
1113 (b'\xf8\x80', FFFD*2), # only one continuation byte
1114 (b'\xf8\x80\x41', FFFD*2 + 'A'), # 1 valid cb and 1 invalid
1115 (b'\xf8\x80\x80\x80\x80', FFFD*5), # invalid 5 bytes seq with 5 bytes
1116 # with invalid start byte of a 6-byte sequence (rfc2279)
1117 (b'\xfc', FFFD), # only the start byte
1118 (b'\xfc\xfc', FFFD*2), # 2 start bytes
1119 (b'\xfc\x80\x80', FFFD*3), # only 2 continuation bytes
1120 (b'\xfc\x80\x80\x80\x80\x80', FFFD*6), # 6 continuation bytes
1121 # invalid start byte
1122 (b'\xfe', FFFD),
1123 (b'\xfe\x80\x80', FFFD*3),
1124 # other sequences
1125 (b'\xf1\x80\x41\x42\x43', '\ufffd\x41\x42\x43'),
1126 (b'\xf1\x80\xff\x42\x43', '\ufffd\ufffd\x42\x43'),
1127 (b'\xf1\x80\xc2\x81\x43', '\ufffd\x81\x43'),
1128 (b'\x61\xF1\x80\x80\xE1\x80\xC2\x62\x80\x63\x80\xBF\x64',
1129 '\x61\uFFFD\uFFFD\uFFFD\x62\uFFFD\x63\uFFFD\uFFFD\x64'),
1130 ]
1131 for n, (seq, res) in enumerate(sequences):
1132 self.assertRaises(UnicodeDecodeError, seq.decode, 'utf-8', 'strict')
1133 self.assertEqual(seq.decode('utf-8', 'replace'), res)
1134 self.assertEqual((seq+b'b').decode('utf-8', 'replace'), res+'b')
1135 self.assertEqual(seq.decode('utf-8', 'ignore'),
1136 res.replace('\uFFFD', ''))
1137
Martin v. Löwis0d8e16c2003-08-05 06:19:47 +00001138 def test_codecs_idna(self):
1139 # Test whether trailing dot is preserved
Walter Dörwald1324c6f2007-05-11 19:57:05 +00001140 self.assertEqual("www.python.org.".encode("idna"), b"www.python.org.")
Martin v. Löwis0d8e16c2003-08-05 06:19:47 +00001141
Walter Dörwald28256f22003-01-19 16:59:20 +00001142 def test_codecs_errors(self):
1143 # Error handling (encoding)
Guido van Rossumef87d6e2007-05-02 19:09:54 +00001144 self.assertRaises(UnicodeError, 'Andr\202 x'.encode, 'ascii')
1145 self.assertRaises(UnicodeError, 'Andr\202 x'.encode, 'ascii','strict')
Walter Dörwald67e83882007-05-05 12:26:27 +00001146 self.assertEqual('Andr\202 x'.encode('ascii','ignore'), b"Andr x")
1147 self.assertEqual('Andr\202 x'.encode('ascii','replace'), b"Andr? x")
Benjamin Peterson308d6372009-09-18 21:42:35 +00001148 self.assertEqual('Andr\202 x'.encode('ascii', 'replace'),
1149 'Andr\202 x'.encode('ascii', errors='replace'))
1150 self.assertEqual('Andr\202 x'.encode('ascii', 'ignore'),
1151 'Andr\202 x'.encode(encoding='ascii', errors='ignore'))
Marc-André Lemburgc60e6f72001-09-20 10:35:46 +00001152
Walter Dörwald28256f22003-01-19 16:59:20 +00001153 # Error handling (decoding)
Walter Dörwald67e83882007-05-05 12:26:27 +00001154 self.assertRaises(UnicodeError, str, b'Andr\202 x', 'ascii')
1155 self.assertRaises(UnicodeError, str, b'Andr\202 x', 'ascii', 'strict')
1156 self.assertEqual(str(b'Andr\202 x', 'ascii', 'ignore'), "Andr x")
1157 self.assertEqual(str(b'Andr\202 x', 'ascii', 'replace'), 'Andr\uFFFD x')
Marc-André Lemburgc60e6f72001-09-20 10:35:46 +00001158
Walter Dörwald28256f22003-01-19 16:59:20 +00001159 # Error handling (unknown character names)
Guido van Rossum39478e82007-08-27 17:23:59 +00001160 self.assertEqual(b"\\N{foo}xx".decode("unicode-escape", "ignore"), "xx")
Marc-André Lemburg3688a882002-02-06 18:09:02 +00001161
Walter Dörwald28256f22003-01-19 16:59:20 +00001162 # Error handling (truncated escape sequence)
Guido van Rossum9c627722007-08-27 18:31:48 +00001163 self.assertRaises(UnicodeError, b"\\".decode, "unicode-escape")
Marc-André Lemburgd6d06ad2000-07-07 17:48:52 +00001164
Guido van Rossum9c627722007-08-27 18:31:48 +00001165 self.assertRaises(TypeError, b"hello".decode, "test.unicode1")
1166 self.assertRaises(TypeError, str, b"hello", "test.unicode2")
Guido van Rossumef87d6e2007-05-02 19:09:54 +00001167 self.assertRaises(TypeError, "hello".encode, "test.unicode1")
1168 self.assertRaises(TypeError, "hello".encode, "test.unicode2")
Walter Dörwald28256f22003-01-19 16:59:20 +00001169 # executes PyUnicode_Encode()
1170 import imp
1171 self.assertRaises(
1172 ImportError,
1173 imp.find_module,
1174 "non-existing module",
Guido van Rossumef87d6e2007-05-02 19:09:54 +00001175 ["non-existing dir"]
Walter Dörwald28256f22003-01-19 16:59:20 +00001176 )
Marc-André Lemburgd6d06ad2000-07-07 17:48:52 +00001177
Walter Dörwald28256f22003-01-19 16:59:20 +00001178 # Error handling (wrong arguments)
Guido van Rossumef87d6e2007-05-02 19:09:54 +00001179 self.assertRaises(TypeError, "hello".encode, 42, 42, 42)
Guido van Rossumd8855fd2000-03-24 22:14:19 +00001180
Alexander Belopolsky942af5a2010-12-04 03:38:46 +00001181 # Error handling (lone surrogate in PyUnicode_TransformDecimalToASCII())
1182 self.assertRaises(UnicodeError, int, "\ud800")
1183 self.assertRaises(UnicodeError, int, "\udf00")
1184 self.assertRaises(UnicodeError, float, "\ud800")
1185 self.assertRaises(UnicodeError, float, "\udf00")
1186 self.assertRaises(UnicodeError, complex, "\ud800")
1187 self.assertRaises(UnicodeError, complex, "\udf00")
Guido van Rossum97064862000-04-10 13:52:48 +00001188
Walter Dörwald28256f22003-01-19 16:59:20 +00001189 def test_codecs(self):
1190 # Encoding
Walter Dörwald67e83882007-05-05 12:26:27 +00001191 self.assertEqual('hello'.encode('ascii'), b'hello')
1192 self.assertEqual('hello'.encode('utf-7'), b'hello')
1193 self.assertEqual('hello'.encode('utf-8'), b'hello')
1194 self.assertEqual('hello'.encode('utf8'), b'hello')
1195 self.assertEqual('hello'.encode('utf-16-le'), b'h\000e\000l\000l\000o\000')
1196 self.assertEqual('hello'.encode('utf-16-be'), b'\000h\000e\000l\000l\000o')
1197 self.assertEqual('hello'.encode('latin-1'), b'hello')
Guido van Rossum97064862000-04-10 13:52:48 +00001198
Walter Dörwald28256f22003-01-19 16:59:20 +00001199 # Roundtrip safety for BMP (just the first 1024 chars)
Guido van Rossum805365e2007-05-07 22:24:25 +00001200 for c in range(1024):
Guido van Rossum84fc66d2007-05-03 17:18:26 +00001201 u = chr(c)
Hye-Shik Chang835b2432005-12-17 04:38:31 +00001202 for encoding in ('utf-7', 'utf-8', 'utf-16', 'utf-16-le',
1203 'utf-16-be', 'raw_unicode_escape',
1204 'unicode_escape', 'unicode_internal'):
Guido van Rossumef87d6e2007-05-02 19:09:54 +00001205 self.assertEqual(str(u.encode(encoding),encoding), u)
Martin v. Löwis047c05e2002-03-21 08:55:28 +00001206
Walter Dörwald28256f22003-01-19 16:59:20 +00001207 # Roundtrip safety for BMP (just the first 256 chars)
Guido van Rossum805365e2007-05-07 22:24:25 +00001208 for c in range(256):
Guido van Rossum84fc66d2007-05-03 17:18:26 +00001209 u = chr(c)
Hye-Shik Chang835b2432005-12-17 04:38:31 +00001210 for encoding in ('latin-1',):
Guido van Rossumef87d6e2007-05-02 19:09:54 +00001211 self.assertEqual(str(u.encode(encoding),encoding), u)
Guido van Rossumd8855fd2000-03-24 22:14:19 +00001212
Walter Dörwald28256f22003-01-19 16:59:20 +00001213 # Roundtrip safety for BMP (just the first 128 chars)
Guido van Rossum805365e2007-05-07 22:24:25 +00001214 for c in range(128):
Guido van Rossum84fc66d2007-05-03 17:18:26 +00001215 u = chr(c)
Hye-Shik Chang835b2432005-12-17 04:38:31 +00001216 for encoding in ('ascii',):
Guido van Rossumef87d6e2007-05-02 19:09:54 +00001217 self.assertEqual(str(u.encode(encoding),encoding), u)
Guido van Rossumd8855fd2000-03-24 22:14:19 +00001218
Walter Dörwald28256f22003-01-19 16:59:20 +00001219 # Roundtrip safety for non-BMP (just a few chars)
Guido van Rossumef87d6e2007-05-02 19:09:54 +00001220 u = '\U00010001\U00020002\U00030003\U00040004\U00050005'
Walter Dörwald28256f22003-01-19 16:59:20 +00001221 for encoding in ('utf-8', 'utf-16', 'utf-16-le', 'utf-16-be',
1222 #'raw_unicode_escape',
1223 'unicode_escape', 'unicode_internal'):
Guido van Rossumef87d6e2007-05-02 19:09:54 +00001224 self.assertEqual(str(u.encode(encoding),encoding), u)
Guido van Rossumd8855fd2000-03-24 22:14:19 +00001225
Walter Dörwald28256f22003-01-19 16:59:20 +00001226 # UTF-8 must be roundtrip safe for all UCS-2 code points
1227 # This excludes surrogates: in the full range, there would be
1228 # a surrogate pair (\udbff\udc00), which gets converted back
1229 # to a non-BMP character (\U0010fc00)
Walter Dörwald1324c6f2007-05-11 19:57:05 +00001230 u = ''.join(map(chr, list(range(0,0xd800)) +
1231 list(range(0xe000,0x10000))))
Walter Dörwald28256f22003-01-19 16:59:20 +00001232 for encoding in ('utf-8',):
Guido van Rossumef87d6e2007-05-02 19:09:54 +00001233 self.assertEqual(str(u.encode(encoding),encoding), u)
Guido van Rossum9e896b32000-04-05 20:11:21 +00001234
Walter Dörwald28256f22003-01-19 16:59:20 +00001235 def test_codecs_charmap(self):
1236 # 0-127
Guido van Rossum805365e2007-05-07 22:24:25 +00001237 s = bytes(range(128))
Walter Dörwald28256f22003-01-19 16:59:20 +00001238 for encoding in (
1239 'cp037', 'cp1026',
Benjamin Peterson5a6214a2010-06-27 22:41:29 +00001240 'cp437', 'cp500', 'cp720', 'cp737', 'cp775', 'cp850',
1241 'cp852', 'cp855', 'cp858', 'cp860', 'cp861', 'cp862',
Walter Dörwald28256f22003-01-19 16:59:20 +00001242 'cp863', 'cp865', 'cp866',
1243 'iso8859_10', 'iso8859_13', 'iso8859_14', 'iso8859_15',
1244 'iso8859_2', 'iso8859_3', 'iso8859_4', 'iso8859_5', 'iso8859_6',
1245 'iso8859_7', 'iso8859_9', 'koi8_r', 'latin_1',
1246 'mac_cyrillic', 'mac_latin2',
Marc-André Lemburgbd3be8f2002-02-07 11:33:49 +00001247
Walter Dörwald28256f22003-01-19 16:59:20 +00001248 'cp1250', 'cp1251', 'cp1252', 'cp1253', 'cp1254', 'cp1255',
1249 'cp1256', 'cp1257', 'cp1258',
1250 'cp856', 'cp857', 'cp864', 'cp869', 'cp874',
Marc-André Lemburgbd3be8f2002-02-07 11:33:49 +00001251
Walter Dörwald28256f22003-01-19 16:59:20 +00001252 'mac_greek', 'mac_iceland','mac_roman', 'mac_turkish',
1253 'cp1006', 'iso8859_8',
Guido van Rossum9e896b32000-04-05 20:11:21 +00001254
Walter Dörwald28256f22003-01-19 16:59:20 +00001255 ### These have undefined mappings:
1256 #'cp424',
Guido van Rossum9e896b32000-04-05 20:11:21 +00001257
Walter Dörwald28256f22003-01-19 16:59:20 +00001258 ### These fail the round-trip:
1259 #'cp875'
Guido van Rossum9e896b32000-04-05 20:11:21 +00001260
Walter Dörwald28256f22003-01-19 16:59:20 +00001261 ):
Guido van Rossumef87d6e2007-05-02 19:09:54 +00001262 self.assertEqual(str(s, encoding).encode(encoding), s)
Guido van Rossum9e896b32000-04-05 20:11:21 +00001263
Walter Dörwald28256f22003-01-19 16:59:20 +00001264 # 128-255
Guido van Rossum805365e2007-05-07 22:24:25 +00001265 s = bytes(range(128, 256))
Walter Dörwald28256f22003-01-19 16:59:20 +00001266 for encoding in (
1267 'cp037', 'cp1026',
Benjamin Peterson5a6214a2010-06-27 22:41:29 +00001268 'cp437', 'cp500', 'cp720', 'cp737', 'cp775', 'cp850',
1269 'cp852', 'cp855', 'cp858', 'cp860', 'cp861', 'cp862',
Walter Dörwald28256f22003-01-19 16:59:20 +00001270 'cp863', 'cp865', 'cp866',
1271 'iso8859_10', 'iso8859_13', 'iso8859_14', 'iso8859_15',
1272 'iso8859_2', 'iso8859_4', 'iso8859_5',
1273 'iso8859_9', 'koi8_r', 'latin_1',
1274 'mac_cyrillic', 'mac_latin2',
Fred Drake004d5e62000-10-23 17:22:08 +00001275
Walter Dörwald28256f22003-01-19 16:59:20 +00001276 ### These have undefined mappings:
1277 #'cp1250', 'cp1251', 'cp1252', 'cp1253', 'cp1254', 'cp1255',
1278 #'cp1256', 'cp1257', 'cp1258',
1279 #'cp424', 'cp856', 'cp857', 'cp864', 'cp869', 'cp874',
1280 #'iso8859_3', 'iso8859_6', 'iso8859_7',
1281 #'mac_greek', 'mac_iceland','mac_roman', 'mac_turkish',
Fred Drake004d5e62000-10-23 17:22:08 +00001282
Walter Dörwald28256f22003-01-19 16:59:20 +00001283 ### These fail the round-trip:
1284 #'cp1006', 'cp875', 'iso8859_8',
Tim Peters2f228e72001-05-13 00:19:31 +00001285
Walter Dörwald28256f22003-01-19 16:59:20 +00001286 ):
Guido van Rossumef87d6e2007-05-02 19:09:54 +00001287 self.assertEqual(str(s, encoding).encode(encoding), s)
Guido van Rossum9e896b32000-04-05 20:11:21 +00001288
Walter Dörwald28256f22003-01-19 16:59:20 +00001289 def test_concatenation(self):
Guido van Rossumef87d6e2007-05-02 19:09:54 +00001290 self.assertEqual(("abc" "def"), "abcdef")
1291 self.assertEqual(("abc" "def"), "abcdef")
1292 self.assertEqual(("abc" "def"), "abcdef")
1293 self.assertEqual(("abc" "def" "ghi"), "abcdefghi")
1294 self.assertEqual(("abc" "def" "ghi"), "abcdefghi")
Fred Drake004d5e62000-10-23 17:22:08 +00001295
Walter Dörwald28256f22003-01-19 16:59:20 +00001296 def test_printing(self):
1297 class BitBucket:
1298 def write(self, text):
1299 pass
Fred Drake004d5e62000-10-23 17:22:08 +00001300
Walter Dörwald28256f22003-01-19 16:59:20 +00001301 out = BitBucket()
Guido van Rossumef87d6e2007-05-02 19:09:54 +00001302 print('abc', file=out)
1303 print('abc', 'def', file=out)
1304 print('abc', 'def', file=out)
1305 print('abc', 'def', file=out)
1306 print('abc\n', file=out)
1307 print('abc\n', end=' ', file=out)
1308 print('abc\n', end=' ', file=out)
1309 print('def\n', file=out)
1310 print('def\n', file=out)
Fred Drake004d5e62000-10-23 17:22:08 +00001311
Martin v. Löwis9a3a9f72003-05-18 12:31:09 +00001312 def test_ucs4(self):
Guido van Rossumef87d6e2007-05-02 19:09:54 +00001313 x = '\U00100000'
Martin v. Löwis9a3a9f72003-05-18 12:31:09 +00001314 y = x.encode("raw-unicode-escape").decode("raw-unicode-escape")
1315 self.assertEqual(x, y)
1316
Florent Xiclunaa87b3832010-09-13 02:28:18 +00001317 y = br'\U00100000'
1318 x = y.decode("raw-unicode-escape").encode("raw-unicode-escape")
1319 self.assertEqual(x, y)
1320 y = br'\U00010000'
1321 x = y.decode("raw-unicode-escape").encode("raw-unicode-escape")
1322 self.assertEqual(x, y)
Christian Heimesfe337bf2008-03-23 21:54:12 +00001323
Florent Xiclunaa87b3832010-09-13 02:28:18 +00001324 try:
1325 br'\U11111111'.decode("raw-unicode-escape")
1326 except UnicodeDecodeError as e:
1327 self.assertEqual(e.start, 0)
1328 self.assertEqual(e.end, 10)
1329 else:
1330 self.fail("Should have raised UnicodeDecodeError")
Christian Heimesfe337bf2008-03-23 21:54:12 +00001331
Brett Cannonc3647ac2005-04-26 03:45:26 +00001332 def test_conversion(self):
1333 # Make sure __unicode__() works properly
1334 class Foo0:
1335 def __str__(self):
1336 return "foo"
1337
1338 class Foo1:
Guido van Rossum98297ee2007-11-06 21:34:58 +00001339 def __str__(self):
Guido van Rossumef87d6e2007-05-02 19:09:54 +00001340 return "foo"
Brett Cannonc3647ac2005-04-26 03:45:26 +00001341
1342 class Foo2(object):
Guido van Rossum98297ee2007-11-06 21:34:58 +00001343 def __str__(self):
Guido van Rossumef87d6e2007-05-02 19:09:54 +00001344 return "foo"
Brett Cannonc3647ac2005-04-26 03:45:26 +00001345
1346 class Foo3(object):
Guido van Rossum98297ee2007-11-06 21:34:58 +00001347 def __str__(self):
Brett Cannonc3647ac2005-04-26 03:45:26 +00001348 return "foo"
1349
1350 class Foo4(str):
Guido van Rossum98297ee2007-11-06 21:34:58 +00001351 def __str__(self):
Brett Cannonc3647ac2005-04-26 03:45:26 +00001352 return "foo"
1353
Guido van Rossumef87d6e2007-05-02 19:09:54 +00001354 class Foo5(str):
Guido van Rossum98297ee2007-11-06 21:34:58 +00001355 def __str__(self):
Brett Cannonc3647ac2005-04-26 03:45:26 +00001356 return "foo"
1357
1358 class Foo6(str):
1359 def __str__(self):
1360 return "foos"
1361
Guido van Rossum98297ee2007-11-06 21:34:58 +00001362 def __str__(self):
Guido van Rossumef87d6e2007-05-02 19:09:54 +00001363 return "foou"
Brett Cannonc3647ac2005-04-26 03:45:26 +00001364
Guido van Rossumef87d6e2007-05-02 19:09:54 +00001365 class Foo7(str):
Brett Cannonc3647ac2005-04-26 03:45:26 +00001366 def __str__(self):
1367 return "foos"
Guido van Rossum98297ee2007-11-06 21:34:58 +00001368 def __str__(self):
Guido van Rossumef87d6e2007-05-02 19:09:54 +00001369 return "foou"
Brett Cannonc3647ac2005-04-26 03:45:26 +00001370
Guido van Rossumef87d6e2007-05-02 19:09:54 +00001371 class Foo8(str):
Brett Cannonc3647ac2005-04-26 03:45:26 +00001372 def __new__(cls, content=""):
Guido van Rossumef87d6e2007-05-02 19:09:54 +00001373 return str.__new__(cls, 2*content)
Guido van Rossum98297ee2007-11-06 21:34:58 +00001374 def __str__(self):
Brett Cannonc3647ac2005-04-26 03:45:26 +00001375 return self
1376
Guido van Rossumef87d6e2007-05-02 19:09:54 +00001377 class Foo9(str):
Brett Cannonc3647ac2005-04-26 03:45:26 +00001378 def __str__(self):
Brett Cannonc3647ac2005-04-26 03:45:26 +00001379 return "not unicode"
1380
Guido van Rossumef87d6e2007-05-02 19:09:54 +00001381 self.assertEqual(str(Foo0()), "foo")
1382 self.assertEqual(str(Foo1()), "foo")
1383 self.assertEqual(str(Foo2()), "foo")
1384 self.assertEqual(str(Foo3()), "foo")
1385 self.assertEqual(str(Foo4("bar")), "foo")
1386 self.assertEqual(str(Foo5("bar")), "foo")
1387 self.assertEqual(str(Foo6("bar")), "foou")
1388 self.assertEqual(str(Foo7("bar")), "foou")
1389 self.assertEqual(str(Foo8("foo")), "foofoo")
Guido van Rossumef87d6e2007-05-02 19:09:54 +00001390 self.assertEqual(str(Foo9("foo")), "not unicode")
Brett Cannonc3647ac2005-04-26 03:45:26 +00001391
Thomas Wouters49fd7fa2006-04-21 10:40:58 +00001392 def test_unicode_repr(self):
1393 class s1:
1394 def __repr__(self):
1395 return '\\n'
1396
1397 class s2:
1398 def __repr__(self):
Guido van Rossumef87d6e2007-05-02 19:09:54 +00001399 return '\\n'
Thomas Wouters49fd7fa2006-04-21 10:40:58 +00001400
1401 self.assertEqual(repr(s1()), '\\n')
1402 self.assertEqual(repr(s2()), '\\n')
1403
Amaury Forgeot d'Arc324ac652010-08-18 20:44:58 +00001404 def test_printable_repr(self):
1405 self.assertEqual(repr('\U00010000'), "'%c'" % (0x10000,)) # printable
Martin v. Löwisbaecd722010-10-11 22:42:28 +00001406 self.assertEqual(repr('\U00014000'), "'\\U00014000'") # nonprintable
Amaury Forgeot d'Arc324ac652010-08-18 20:44:58 +00001407
Guido van Rossumcd16bf62007-06-13 18:07:49 +00001408 def test_expandtabs_overflows_gracefully(self):
1409 # This test only affects 32-bit platforms because expandtabs can only take
1410 # an int as the max value, not a 64-bit C long. If expandtabs is changed
1411 # to take a 64-bit long, this test should apply to all platforms.
Christian Heimesa37d4c62007-12-04 23:02:19 +00001412 if sys.maxsize > (1 << 32) or struct.calcsize('P') != 4:
Guido van Rossumcd16bf62007-06-13 18:07:49 +00001413 return
Christian Heimesa37d4c62007-12-04 23:02:19 +00001414 self.assertRaises(OverflowError, 't\tt\t'.expandtabs, sys.maxsize)
Thomas Wouters49fd7fa2006-04-21 10:40:58 +00001415
Amaury Forgeot d'Arc7888d082008-08-01 01:06:32 +00001416 def test_raiseMemError(self):
1417 # Ensure that the freelist contains a consistent object, even
1418 # when a string allocation fails with a MemoryError.
1419 # This used to crash the interpreter,
1420 # or leak references when the number was smaller.
Antoine Pitroub305aeb2008-09-05 22:13:06 +00001421 charwidth = 4 if sys.maxunicode >= 0x10000 else 2
1422 # Note: sys.maxsize is half of the actual max allocation because of
1423 # the signedness of Py_ssize_t.
1424 alloc = lambda: "a" * (sys.maxsize // charwidth * 2)
Antoine Pitrou3db3e872008-08-17 17:06:51 +00001425 self.assertRaises(MemoryError, alloc)
1426 self.assertRaises(MemoryError, alloc)
1427
Victor Stinner808fc0a2010-03-22 12:50:40 +00001428 def test_format_subclass(self):
1429 class S(str):
1430 def __str__(self):
1431 return '__str__ overridden'
1432 s = S('xxx')
Florent Xiclunaa87b3832010-09-13 02:28:18 +00001433 self.assertEqual("%s" % s, '__str__ overridden')
1434 self.assertEqual("{}".format(s), '__str__ overridden')
Victor Stinner808fc0a2010-03-22 12:50:40 +00001435
Victor Stinnerca1e7ec2011-01-05 00:19:28 +00001436 # Test PyUnicode_FromFormat()
Victor Stinner1205f272010-09-11 00:54:47 +00001437 def test_from_format(self):
Victor Stinnerca1e7ec2011-01-05 00:19:28 +00001438 support.import_module('ctypes')
Victor Stinner659eb842011-02-23 12:14:22 +00001439 from ctypes import pythonapi, py_object, c_int
Victor Stinnerca1e7ec2011-01-05 00:19:28 +00001440 if sys.maxunicode == 65535:
1441 name = "PyUnicodeUCS2_FromFormat"
1442 else:
1443 name = "PyUnicodeUCS4_FromFormat"
1444 _PyUnicode_FromFormat = getattr(pythonapi, name)
1445 _PyUnicode_FromFormat.restype = py_object
1446
1447 def PyUnicode_FromFormat(format, *args):
1448 cargs = tuple(
1449 py_object(arg) if isinstance(arg, str) else arg
1450 for arg in args)
1451 return _PyUnicode_FromFormat(format, *cargs)
Victor Stinner1205f272010-09-11 00:54:47 +00001452
1453 # ascii format, non-ascii argument
Victor Stinnerca1e7ec2011-01-05 00:19:28 +00001454 text = PyUnicode_FromFormat(b'ascii\x7f=%U', 'unicode\xe9')
Victor Stinner1205f272010-09-11 00:54:47 +00001455 self.assertEqual(text, 'ascii\x7f=unicode\xe9')
1456
Victor Stinnerca1e7ec2011-01-05 00:19:28 +00001457 # non-ascii format, ascii argument: ensure that PyUnicode_FromFormatV()
1458 # raises an error
Ezio Melottied3a7d22010-12-01 02:32:32 +00001459 self.assertRaisesRegex(ValueError,
Victor Stinner1205f272010-09-11 00:54:47 +00001460 '^PyUnicode_FromFormatV\(\) expects an ASCII-encoded format '
Victor Stinner4c7db312010-09-12 07:51:18 +00001461 'string, got a non-ASCII byte: 0xe9$',
Victor Stinnerca1e7ec2011-01-05 00:19:28 +00001462 PyUnicode_FromFormat, b'unicode\xe9=%s', 'ascii')
Amaury Forgeot d'Arc7888d082008-08-01 01:06:32 +00001463
Victor Stinner659eb842011-02-23 12:14:22 +00001464 self.assertEqual(PyUnicode_FromFormat(b'%c', c_int(0xabcd)), '\uabcd')
1465 self.assertEqual(PyUnicode_FromFormat(b'%c', c_int(0x10ffff)), '\U0010ffff')
1466
Victor Stinner9a909002010-10-18 20:59:24 +00001467 # other tests
Victor Stinnerca1e7ec2011-01-05 00:19:28 +00001468 text = PyUnicode_FromFormat(b'%%A:%A', 'abc\xe9\uabcd\U0010ffff')
Victor Stinner9a909002010-10-18 20:59:24 +00001469 self.assertEqual(text, r"%A:'abc\xe9\uabcd\U0010ffff'")
1470
Victor Stinner2b574a22011-03-01 22:48:49 +00001471 text = PyUnicode_FromFormat(b'repr=%V', 'abc', b'xyz')
1472 self.assertEqual(text, 'repr=abc')
1473
1474 # Test string decode from parameter of %s using utf-8.
1475 # b'\xe4\xba\xba\xe6\xb0\x91' is utf-8 encoded byte sequence of
1476 # '\u4eba\u6c11'
1477 text = PyUnicode_FromFormat(b'repr=%V', None, b'\xe4\xba\xba\xe6\xb0\x91')
1478 self.assertEqual(text, 'repr=\u4eba\u6c11')
1479
1480 #Test replace error handler.
1481 text = PyUnicode_FromFormat(b'repr=%V', None, b'abc\xff')
1482 self.assertEqual(text, 'repr=abc\ufffd')
1483
Victor Stinner1c24bd02010-10-02 11:03:13 +00001484 # Test PyUnicode_AsWideChar()
1485 def test_aswidechar(self):
Victor Stinner46c7b3b2010-10-02 11:49:31 +00001486 from _testcapi import unicode_aswidechar
Antoine Pitrou0662bc22010-11-22 16:19:04 +00001487 support.import_module('ctypes')
Victor Stinner1c24bd02010-10-02 11:03:13 +00001488 from ctypes import c_wchar, sizeof
1489
Victor Stinner46c7b3b2010-10-02 11:49:31 +00001490 wchar, size = unicode_aswidechar('abcdef', 2)
Ezio Melottib3aedd42010-11-20 19:04:17 +00001491 self.assertEqual(size, 2)
1492 self.assertEqual(wchar, 'ab')
Victor Stinner1c24bd02010-10-02 11:03:13 +00001493
Victor Stinner46c7b3b2010-10-02 11:49:31 +00001494 wchar, size = unicode_aswidechar('abc', 3)
Ezio Melottib3aedd42010-11-20 19:04:17 +00001495 self.assertEqual(size, 3)
1496 self.assertEqual(wchar, 'abc')
Victor Stinner1c24bd02010-10-02 11:03:13 +00001497
Victor Stinner46c7b3b2010-10-02 11:49:31 +00001498 wchar, size = unicode_aswidechar('abc', 4)
Ezio Melottib3aedd42010-11-20 19:04:17 +00001499 self.assertEqual(size, 3)
1500 self.assertEqual(wchar, 'abc\0')
Victor Stinner1c24bd02010-10-02 11:03:13 +00001501
Victor Stinner46c7b3b2010-10-02 11:49:31 +00001502 wchar, size = unicode_aswidechar('abc', 10)
Ezio Melottib3aedd42010-11-20 19:04:17 +00001503 self.assertEqual(size, 3)
1504 self.assertEqual(wchar, 'abc\0')
Victor Stinner1c24bd02010-10-02 11:03:13 +00001505
Victor Stinner46c7b3b2010-10-02 11:49:31 +00001506 wchar, size = unicode_aswidechar('abc\0def', 20)
Ezio Melottib3aedd42010-11-20 19:04:17 +00001507 self.assertEqual(size, 7)
1508 self.assertEqual(wchar, 'abc\0def\0')
Victor Stinner1c24bd02010-10-02 11:03:13 +00001509
Victor Stinner5593d8a2010-10-02 11:11:27 +00001510 nonbmp = chr(0x10ffff)
1511 if sizeof(c_wchar) == 2:
1512 buflen = 3
1513 nchar = 2
1514 else: # sizeof(c_wchar) == 4
1515 buflen = 2
1516 nchar = 1
Victor Stinner46c7b3b2010-10-02 11:49:31 +00001517 wchar, size = unicode_aswidechar(nonbmp, buflen)
Ezio Melottib3aedd42010-11-20 19:04:17 +00001518 self.assertEqual(size, nchar)
1519 self.assertEqual(wchar, nonbmp + '\0')
Victor Stinner5593d8a2010-10-02 11:11:27 +00001520
Victor Stinner1c24bd02010-10-02 11:03:13 +00001521 # Test PyUnicode_AsWideCharString()
1522 def test_aswidecharstring(self):
Victor Stinner46c7b3b2010-10-02 11:49:31 +00001523 from _testcapi import unicode_aswidecharstring
Antoine Pitrou0662bc22010-11-22 16:19:04 +00001524 support.import_module('ctypes')
Victor Stinner1c24bd02010-10-02 11:03:13 +00001525 from ctypes import c_wchar, sizeof
1526
Victor Stinner46c7b3b2010-10-02 11:49:31 +00001527 wchar, size = unicode_aswidecharstring('abc')
Ezio Melottib3aedd42010-11-20 19:04:17 +00001528 self.assertEqual(size, 3)
1529 self.assertEqual(wchar, 'abc\0')
Victor Stinner1c24bd02010-10-02 11:03:13 +00001530
Victor Stinner46c7b3b2010-10-02 11:49:31 +00001531 wchar, size = unicode_aswidecharstring('abc\0def')
Ezio Melottib3aedd42010-11-20 19:04:17 +00001532 self.assertEqual(size, 7)
1533 self.assertEqual(wchar, 'abc\0def\0')
Victor Stinner1c24bd02010-10-02 11:03:13 +00001534
Victor Stinner5593d8a2010-10-02 11:11:27 +00001535 nonbmp = chr(0x10ffff)
1536 if sizeof(c_wchar) == 2:
1537 nchar = 2
1538 else: # sizeof(c_wchar) == 4
1539 nchar = 1
Victor Stinner46c7b3b2010-10-02 11:49:31 +00001540 wchar, size = unicode_aswidecharstring(nonbmp)
Ezio Melottib3aedd42010-11-20 19:04:17 +00001541 self.assertEqual(size, nchar)
1542 self.assertEqual(wchar, nonbmp + '\0')
Victor Stinner5593d8a2010-10-02 11:11:27 +00001543
Victor Stinner1c24bd02010-10-02 11:03:13 +00001544
Eric Smitha1eac722011-01-29 11:15:35 +00001545class StringModuleTest(unittest.TestCase):
1546 def test_formatter_parser(self):
1547 def parse(format):
1548 return list(_string.formatter_parser(format))
1549
1550 formatter = parse("prefix {2!s}xxx{0:^+10.3f}{obj.attr!s} {z[0]!s:10}")
1551 self.assertEqual(formatter, [
1552 ('prefix ', '2', '', 's'),
1553 ('xxx', '0', '^+10.3f', None),
1554 ('', 'obj.attr', '', 's'),
1555 (' ', 'z[0]', '10', 's'),
1556 ])
1557
1558 formatter = parse("prefix {} suffix")
1559 self.assertEqual(formatter, [
1560 ('prefix ', '', '', None),
1561 (' suffix', None, None, None),
1562 ])
1563
1564 formatter = parse("str")
1565 self.assertEqual(formatter, [
1566 ('str', None, None, None),
1567 ])
1568
1569 formatter = parse("")
1570 self.assertEqual(formatter, [])
1571
1572 formatter = parse("{0}")
1573 self.assertEqual(formatter, [
1574 ('', '0', '', None),
1575 ])
1576
1577 self.assertRaises(TypeError, _string.formatter_parser, 1)
1578
1579 def test_formatter_field_name_split(self):
1580 def split(name):
1581 items = list(_string.formatter_field_name_split(name))
1582 items[1] = list(items[1])
1583 return items
1584 self.assertEqual(split("obj"), ["obj", []])
1585 self.assertEqual(split("obj.arg"), ["obj", [(True, 'arg')]])
1586 self.assertEqual(split("obj[key]"), ["obj", [(False, 'key')]])
1587 self.assertEqual(split("obj.arg[key1][key2]"), [
1588 "obj",
1589 [(True, 'arg'),
1590 (False, 'key1'),
1591 (False, 'key2'),
1592 ]])
1593 self.assertRaises(TypeError, _string.formatter_field_name_split, 1)
1594
1595
Walter Dörwald28256f22003-01-19 16:59:20 +00001596def test_main():
Benjamin Petersonee8712c2008-05-20 21:35:26 +00001597 support.run_unittest(__name__)
Barry Warsaw817918c2002-08-06 16:58:21 +00001598
Walter Dörwald28256f22003-01-19 16:59:20 +00001599if __name__ == "__main__":
1600 test_main()