blob: 99aa0033cda0235ff0b68c6056b86e6574884aa0 [file] [log] [blame]
Guido van Rossum24bdb042000-03-28 20:29:59 +00001""" Test script for the unicodedata module.
2
Marc-André Lemburg6a20ee72000-09-26 16:18:58 +00003 Written by Marc-Andre Lemburg (mal@lemburg.com).
Guido van Rossum24bdb042000-03-28 20:29:59 +00004
Marc-André Lemburg6a20ee72000-09-26 16:18:58 +00005 (c) Copyright CNRI, All Rights Reserved. NO WARRANTY.
Guido van Rossum24bdb042000-03-28 20:29:59 +00006
Benjamin Peterson6f7fad12008-11-21 22:58:57 +00007"""
8
9import sys
10import unittest
Thomas Wouters477c8d52006-05-27 19:21:47 +000011import hashlib
Benjamin Peterson6f7fad12008-11-21 22:58:57 +000012import subprocess
13import test.support
Guido van Rossum24bdb042000-03-28 20:29:59 +000014
Marc-André Lemburg67ceca72000-09-27 12:24:34 +000015encoding = 'utf-8'
Martin v. Löwis54d9d072009-05-10 09:33:21 +000016errors = 'surrogatepass'
Marc-André Lemburg67ceca72000-09-27 12:24:34 +000017
Marc-André Lemburg6a20ee72000-09-26 16:18:58 +000018
19### Run tests
20
Walter Dörwald37c47282003-02-26 14:49:41 +000021class UnicodeMethodsTest(unittest.TestCase):
Marc-André Lemburg6a20ee72000-09-26 16:18:58 +000022
Walter Dörwald37c47282003-02-26 14:49:41 +000023 # update this, if the database changes
Benjamin Peterson71f660e2012-02-20 22:24:29 -050024 expectedchecksum = 'bf7a78f1a532421b5033600102e23a92044dbba9'
Guido van Rossum24bdb042000-03-28 20:29:59 +000025
Walter Dörwald37c47282003-02-26 14:49:41 +000026 def test_method_checksum(self):
Thomas Wouters477c8d52006-05-27 19:21:47 +000027 h = hashlib.sha1()
Florent Xicluna806d8cf2010-03-30 19:34:18 +000028 for i in range(0x10000):
Guido van Rossum84fc66d2007-05-03 17:18:26 +000029 char = chr(i)
Walter Dörwald37c47282003-02-26 14:49:41 +000030 data = [
31 # Predicates (single char)
Guido van Rossumef87d6e2007-05-02 19:09:54 +000032 "01"[char.isalnum()],
33 "01"[char.isalpha()],
34 "01"[char.isdecimal()],
35 "01"[char.isdigit()],
36 "01"[char.islower()],
37 "01"[char.isnumeric()],
38 "01"[char.isspace()],
39 "01"[char.istitle()],
40 "01"[char.isupper()],
Guido van Rossum24bdb042000-03-28 20:29:59 +000041
Walter Dörwald37c47282003-02-26 14:49:41 +000042 # Predicates (multiple chars)
Guido van Rossumef87d6e2007-05-02 19:09:54 +000043 "01"[(char + 'abc').isalnum()],
44 "01"[(char + 'abc').isalpha()],
45 "01"[(char + '123').isdecimal()],
46 "01"[(char + '123').isdigit()],
47 "01"[(char + 'abc').islower()],
48 "01"[(char + '123').isnumeric()],
49 "01"[(char + ' \t').isspace()],
50 "01"[(char + 'abc').istitle()],
51 "01"[(char + 'ABC').isupper()],
Guido van Rossum24bdb042000-03-28 20:29:59 +000052
Walter Dörwald37c47282003-02-26 14:49:41 +000053 # Mappings (single char)
54 char.lower(),
55 char.upper(),
56 char.title(),
Guido van Rossum24bdb042000-03-28 20:29:59 +000057
Walter Dörwald37c47282003-02-26 14:49:41 +000058 # Mappings (multiple chars)
Guido van Rossumef87d6e2007-05-02 19:09:54 +000059 (char + 'abc').lower(),
60 (char + 'ABC').upper(),
61 (char + 'abc').title(),
62 (char + 'ABC').title(),
Guido van Rossum24bdb042000-03-28 20:29:59 +000063
Walter Dörwald37c47282003-02-26 14:49:41 +000064 ]
Martin v. Löwisdb12d452009-05-02 18:52:14 +000065 h.update(''.join(data).encode(encoding, errors))
Walter Dörwald37c47282003-02-26 14:49:41 +000066 result = h.hexdigest()
67 self.assertEqual(result, self.expectedchecksum)
Guido van Rossum24bdb042000-03-28 20:29:59 +000068
Walter Dörwald37c47282003-02-26 14:49:41 +000069class UnicodeDatabaseTest(unittest.TestCase):
Guido van Rossum24bdb042000-03-28 20:29:59 +000070
Walter Dörwald37c47282003-02-26 14:49:41 +000071 def setUp(self):
72 # In case unicodedata is not available, this will raise an ImportError,
73 # but the other test cases will still be run
74 import unicodedata
75 self.db = unicodedata
Guido van Rossum24bdb042000-03-28 20:29:59 +000076
Walter Dörwald37c47282003-02-26 14:49:41 +000077 def tearDown(self):
78 del self.db
Guido van Rossum24bdb042000-03-28 20:29:59 +000079
Walter Dörwald37c47282003-02-26 14:49:41 +000080class UnicodeFunctionsTest(UnicodeDatabaseTest):
Guido van Rossum24bdb042000-03-28 20:29:59 +000081
Walter Dörwald37c47282003-02-26 14:49:41 +000082 # update this, if the database changes
Benjamin Peterson71f660e2012-02-20 22:24:29 -050083 expectedchecksum = '17fe2f12b788e4fff5479b469c4404bb6ecf841f'
Walter Dörwald37c47282003-02-26 14:49:41 +000084 def test_function_checksum(self):
85 data = []
Thomas Wouters477c8d52006-05-27 19:21:47 +000086 h = hashlib.sha1()
Walter Dörwald37c47282003-02-26 14:49:41 +000087
88 for i in range(0x10000):
Guido van Rossum84fc66d2007-05-03 17:18:26 +000089 char = chr(i)
Walter Dörwald37c47282003-02-26 14:49:41 +000090 data = [
91 # Properties
Mark Dickinson388122d2010-08-04 20:56:28 +000092 format(self.db.digit(char, -1), '.12g'),
93 format(self.db.numeric(char, -1), '.12g'),
94 format(self.db.decimal(char, -1), '.12g'),
Walter Dörwald37c47282003-02-26 14:49:41 +000095 self.db.category(char),
96 self.db.bidirectional(char),
97 self.db.decomposition(char),
98 str(self.db.mirrored(char)),
99 str(self.db.combining(char)),
100 ]
Guido van Rossum9c627722007-08-27 18:31:48 +0000101 h.update(''.join(data).encode("ascii"))
Walter Dörwald37c47282003-02-26 14:49:41 +0000102 result = h.hexdigest()
103 self.assertEqual(result, self.expectedchecksum)
104
105 def test_digit(self):
Guido van Rossumef87d6e2007-05-02 19:09:54 +0000106 self.assertEqual(self.db.digit('A', None), None)
107 self.assertEqual(self.db.digit('9'), 9)
108 self.assertEqual(self.db.digit('\u215b', None), None)
109 self.assertEqual(self.db.digit('\u2468'), 9)
Walter Dörwaldf342bfc2008-06-03 11:45:02 +0000110 self.assertEqual(self.db.digit('\U00020000', None), None)
Ezio Melottif5036732011-09-29 03:14:56 +0300111 self.assertEqual(self.db.digit('\U0001D7FD'), 7)
Walter Dörwald37c47282003-02-26 14:49:41 +0000112
113 self.assertRaises(TypeError, self.db.digit)
Guido van Rossumef87d6e2007-05-02 19:09:54 +0000114 self.assertRaises(TypeError, self.db.digit, 'xx')
115 self.assertRaises(ValueError, self.db.digit, 'x')
Walter Dörwald37c47282003-02-26 14:49:41 +0000116
117 def test_numeric(self):
Guido van Rossumef87d6e2007-05-02 19:09:54 +0000118 self.assertEqual(self.db.numeric('A',None), None)
119 self.assertEqual(self.db.numeric('9'), 9)
120 self.assertEqual(self.db.numeric('\u215b'), 0.125)
121 self.assertEqual(self.db.numeric('\u2468'), 9.0)
Amaury Forgeot d'Arc7d520792009-10-06 21:03:20 +0000122 self.assertEqual(self.db.numeric('\ua627'), 7.0)
Walter Dörwaldf342bfc2008-06-03 11:45:02 +0000123 self.assertEqual(self.db.numeric('\U00020000', None), None)
Ezio Melottif5036732011-09-29 03:14:56 +0300124 self.assertEqual(self.db.numeric('\U0001012A'), 9000)
Walter Dörwald37c47282003-02-26 14:49:41 +0000125
126 self.assertRaises(TypeError, self.db.numeric)
Guido van Rossumef87d6e2007-05-02 19:09:54 +0000127 self.assertRaises(TypeError, self.db.numeric, 'xx')
128 self.assertRaises(ValueError, self.db.numeric, 'x')
Walter Dörwald37c47282003-02-26 14:49:41 +0000129
130 def test_decimal(self):
Guido van Rossumef87d6e2007-05-02 19:09:54 +0000131 self.assertEqual(self.db.decimal('A',None), None)
132 self.assertEqual(self.db.decimal('9'), 9)
133 self.assertEqual(self.db.decimal('\u215b', None), None)
134 self.assertEqual(self.db.decimal('\u2468', None), None)
Walter Dörwaldf342bfc2008-06-03 11:45:02 +0000135 self.assertEqual(self.db.decimal('\U00020000', None), None)
Ezio Melottif5036732011-09-29 03:14:56 +0300136 self.assertEqual(self.db.decimal('\U0001D7FD'), 7)
Walter Dörwald37c47282003-02-26 14:49:41 +0000137
138 self.assertRaises(TypeError, self.db.decimal)
Guido van Rossumef87d6e2007-05-02 19:09:54 +0000139 self.assertRaises(TypeError, self.db.decimal, 'xx')
140 self.assertRaises(ValueError, self.db.decimal, 'x')
Walter Dörwald37c47282003-02-26 14:49:41 +0000141
142 def test_category(self):
Guido van Rossumef87d6e2007-05-02 19:09:54 +0000143 self.assertEqual(self.db.category('\uFFFE'), 'Cn')
144 self.assertEqual(self.db.category('a'), 'Ll')
145 self.assertEqual(self.db.category('A'), 'Lu')
Walter Dörwaldf342bfc2008-06-03 11:45:02 +0000146 self.assertEqual(self.db.category('\U00020000'), 'Lo')
Ezio Melottif5036732011-09-29 03:14:56 +0300147 self.assertEqual(self.db.category('\U0001012A'), 'No')
Walter Dörwald37c47282003-02-26 14:49:41 +0000148
149 self.assertRaises(TypeError, self.db.category)
Guido van Rossumef87d6e2007-05-02 19:09:54 +0000150 self.assertRaises(TypeError, self.db.category, 'xx')
Walter Dörwald37c47282003-02-26 14:49:41 +0000151
152 def test_bidirectional(self):
Guido van Rossumef87d6e2007-05-02 19:09:54 +0000153 self.assertEqual(self.db.bidirectional('\uFFFE'), '')
154 self.assertEqual(self.db.bidirectional(' '), 'WS')
155 self.assertEqual(self.db.bidirectional('A'), 'L')
Walter Dörwaldf342bfc2008-06-03 11:45:02 +0000156 self.assertEqual(self.db.bidirectional('\U00020000'), 'L')
Walter Dörwald37c47282003-02-26 14:49:41 +0000157
158 self.assertRaises(TypeError, self.db.bidirectional)
Guido van Rossumef87d6e2007-05-02 19:09:54 +0000159 self.assertRaises(TypeError, self.db.bidirectional, 'xx')
Walter Dörwald37c47282003-02-26 14:49:41 +0000160
161 def test_decomposition(self):
Guido van Rossumef87d6e2007-05-02 19:09:54 +0000162 self.assertEqual(self.db.decomposition('\uFFFE'),'')
163 self.assertEqual(self.db.decomposition('\u00bc'), '<fraction> 0031 2044 0034')
Walter Dörwald37c47282003-02-26 14:49:41 +0000164
165 self.assertRaises(TypeError, self.db.decomposition)
Guido van Rossumef87d6e2007-05-02 19:09:54 +0000166 self.assertRaises(TypeError, self.db.decomposition, 'xx')
Walter Dörwald37c47282003-02-26 14:49:41 +0000167
168 def test_mirrored(self):
Guido van Rossumef87d6e2007-05-02 19:09:54 +0000169 self.assertEqual(self.db.mirrored('\uFFFE'), 0)
170 self.assertEqual(self.db.mirrored('a'), 0)
171 self.assertEqual(self.db.mirrored('\u2201'), 1)
Walter Dörwaldf342bfc2008-06-03 11:45:02 +0000172 self.assertEqual(self.db.mirrored('\U00020000'), 0)
Walter Dörwald37c47282003-02-26 14:49:41 +0000173
174 self.assertRaises(TypeError, self.db.mirrored)
Guido van Rossumef87d6e2007-05-02 19:09:54 +0000175 self.assertRaises(TypeError, self.db.mirrored, 'xx')
Walter Dörwald37c47282003-02-26 14:49:41 +0000176
177 def test_combining(self):
Guido van Rossumef87d6e2007-05-02 19:09:54 +0000178 self.assertEqual(self.db.combining('\uFFFE'), 0)
179 self.assertEqual(self.db.combining('a'), 0)
180 self.assertEqual(self.db.combining('\u20e1'), 230)
Walter Dörwaldf342bfc2008-06-03 11:45:02 +0000181 self.assertEqual(self.db.combining('\U00020000'), 0)
Walter Dörwald37c47282003-02-26 14:49:41 +0000182
183 self.assertRaises(TypeError, self.db.combining)
Guido van Rossumef87d6e2007-05-02 19:09:54 +0000184 self.assertRaises(TypeError, self.db.combining, 'xx')
Walter Dörwald37c47282003-02-26 14:49:41 +0000185
186 def test_normalize(self):
187 self.assertRaises(TypeError, self.db.normalize)
Guido van Rossumef87d6e2007-05-02 19:09:54 +0000188 self.assertRaises(ValueError, self.db.normalize, 'unknown', 'xx')
189 self.assertEqual(self.db.normalize('NFKC', ''), '')
Walter Dörwald37c47282003-02-26 14:49:41 +0000190 # The rest can be found in test_normalization.py
191 # which requires an external file.
192
Victor Stinner7ed9c4c2010-03-04 12:14:57 +0000193 def test_pr29(self):
194 # http://www.unicode.org/review/pr-29.html
Alexander Belopolsky86f65d52010-12-23 02:27:37 +0000195 # See issues #1054943 and #10254.
196 composed = ("\u0b47\u0300\u0b3e", "\u1100\u0300\u1161",
197 'Li\u030dt-s\u1e73\u0301',
198 '\u092e\u093e\u0930\u094d\u0915 \u091c\u093c'
199 + '\u0941\u0915\u0947\u0930\u092c\u0930\u094d\u0917',
200 '\u0915\u093f\u0930\u094d\u0917\u093f\u091c\u093c'
201 + '\u0938\u094d\u0924\u093e\u0928')
202 for text in composed:
Victor Stinner7ed9c4c2010-03-04 12:14:57 +0000203 self.assertEqual(self.db.normalize('NFC', text), text)
204
Alexander Belopolsky86f65d52010-12-23 02:27:37 +0000205 def test_issue10254(self):
206 # Crash reported in #10254
207 a = 'C\u0338' * 20 + 'C\u0327'
208 b = 'C\u0338' * 20 + '\xC7'
209 self.assertEqual(self.db.normalize('NFC', a), b)
210
Hye-Shik Change9ddfbb2004-08-04 07:38:35 +0000211 def test_east_asian_width(self):
212 eaw = self.db.east_asian_width
Guido van Rossum98297ee2007-11-06 21:34:58 +0000213 self.assertRaises(TypeError, eaw, b'a')
Guido van Rossum254348e2007-11-21 19:29:53 +0000214 self.assertRaises(TypeError, eaw, bytearray())
Guido van Rossumef87d6e2007-05-02 19:09:54 +0000215 self.assertRaises(TypeError, eaw, '')
216 self.assertRaises(TypeError, eaw, 'ra')
217 self.assertEqual(eaw('\x1e'), 'N')
218 self.assertEqual(eaw('\x20'), 'Na')
219 self.assertEqual(eaw('\uC894'), 'W')
220 self.assertEqual(eaw('\uFF66'), 'H')
221 self.assertEqual(eaw('\uFF1F'), 'F')
222 self.assertEqual(eaw('\u2010'), 'A')
Walter Dörwaldf342bfc2008-06-03 11:45:02 +0000223 self.assertEqual(eaw('\U00020000'), 'W')
Walter Dörwald37c47282003-02-26 14:49:41 +0000224
225class UnicodeMiscTest(UnicodeDatabaseTest):
226
Benjamin Peterson6f7fad12008-11-21 22:58:57 +0000227 def test_failed_import_during_compiling(self):
228 # Issue 4367
229 # Decoding \N escapes requires the unicodedata module. If it can't be
230 # imported, we shouldn't segfault.
231
232 # This program should raise a SyntaxError in the eval.
233 code = "import sys;" \
234 "sys.modules['unicodedata'] = None;" \
235 """eval("'\\\\N{SOFT HYPHEN}'")"""
236 args = [sys.executable, "-c", code]
237 # We use a subprocess because the unicodedata module may already have
238 # been loaded in this process.
239 popen = subprocess.Popen(args, stderr=subprocess.PIPE)
240 popen.wait()
241 self.assertEqual(popen.returncode, 1)
242 error = "SyntaxError: (unicode error) \\N escapes not supported " \
243 "(can't load unicodedata module)"
Benjamin Peterson577473f2010-01-19 00:09:57 +0000244 self.assertIn(error, popen.stderr.read().decode("ascii"))
Antoine Pitrou849e12b2010-10-30 14:24:33 +0000245 popen.stderr.close()
Benjamin Peterson6f7fad12008-11-21 22:58:57 +0000246
Walter Dörwald37c47282003-02-26 14:49:41 +0000247 def test_decimal_numeric_consistent(self):
248 # Test that decimal and numeric are consistent,
249 # i.e. if a character has a decimal value,
Georg Brandl7eb4b7d2005-07-22 21:49:32 +0000250 # its numeric value should be the same.
Walter Dörwald37c47282003-02-26 14:49:41 +0000251 count = 0
Guido van Rossum805365e2007-05-07 22:24:25 +0000252 for i in range(0x10000):
Guido van Rossum84fc66d2007-05-03 17:18:26 +0000253 c = chr(i)
Walter Dörwald37c47282003-02-26 14:49:41 +0000254 dec = self.db.decimal(c, -1)
255 if dec != -1:
256 self.assertEqual(dec, self.db.numeric(c))
257 count += 1
Benjamin Petersonc9c0f202009-06-30 23:06:06 +0000258 self.assertTrue(count >= 10) # should have tested at least the ASCII digits
Walter Dörwald37c47282003-02-26 14:49:41 +0000259
260 def test_digit_numeric_consistent(self):
261 # Test that digit and numeric are consistent,
Tim Peters669454e2003-03-07 17:30:48 +0000262 # i.e. if a character has a digit value,
Georg Brandl7eb4b7d2005-07-22 21:49:32 +0000263 # its numeric value should be the same.
Walter Dörwald37c47282003-02-26 14:49:41 +0000264 count = 0
Guido van Rossum805365e2007-05-07 22:24:25 +0000265 for i in range(0x10000):
Guido van Rossum84fc66d2007-05-03 17:18:26 +0000266 c = chr(i)
Walter Dörwald37c47282003-02-26 14:49:41 +0000267 dec = self.db.digit(c, -1)
268 if dec != -1:
269 self.assertEqual(dec, self.db.numeric(c))
270 count += 1
Benjamin Petersonc9c0f202009-06-30 23:06:06 +0000271 self.assertTrue(count >= 10) # should have tested at least the ASCII digits
Walter Dörwald37c47282003-02-26 14:49:41 +0000272
Guido van Rossum806c2462007-08-06 23:33:07 +0000273 def test_bug_1704793(self):
Ezio Melottib3aedd42010-11-20 19:04:17 +0000274 self.assertEqual(self.db.lookup("GOTHIC LETTER FAIHU"), '\U00010346')
Guido van Rossum806c2462007-08-06 23:33:07 +0000275
Martin v. Löwis93cbca32008-09-10 14:08:48 +0000276 def test_ucd_510(self):
277 import unicodedata
278 # In UCD 5.1.0, a mirrored property changed wrt. UCD 3.2.0
Benjamin Petersonc9c0f202009-06-30 23:06:06 +0000279 self.assertTrue(unicodedata.mirrored("\u0f3a"))
280 self.assertTrue(not unicodedata.ucd_3_2_0.mirrored("\u0f3a"))
Martin v. Löwis93cbca32008-09-10 14:08:48 +0000281 # Also, we now have two ways of representing
282 # the upper-case mapping: as delta, or as absolute value
Benjamin Petersonc9c0f202009-06-30 23:06:06 +0000283 self.assertTrue("a".upper()=='A')
284 self.assertTrue("\u1d79".upper()=='\ua77d')
285 self.assertTrue(".".upper()=='.')
Walter Dörwald1b08b302009-04-25 14:13:56 +0000286
287 def test_bug_5828(self):
288 self.assertEqual("\u1d79".lower(), "\u1d79")
289 # Only U+0000 should have U+0000 as its upper/lower/titlecase variant
290 self.assertEqual(
291 [
292 c for c in range(sys.maxunicode+1)
293 if "\x00" in chr(c).lower()+chr(c).upper()+chr(c).title()
294 ],
295 [0]
296 )
297
Walter Dörwalde2507752009-04-26 19:16:11 +0000298 def test_bug_4971(self):
Martin v. Löwis71efeb72009-04-26 01:02:07 +0000299 # LETTER DZ WITH CARON: DZ, Dz, dz
300 self.assertEqual("\u01c4".title(), "\u01c5")
301 self.assertEqual("\u01c5".title(), "\u01c5")
302 self.assertEqual("\u01c6".title(), "\u01c5")
Martin v. Löwis93cbca32008-09-10 14:08:48 +0000303
Florent Xicluna806d8cf2010-03-30 19:34:18 +0000304 def test_linebreak_7643(self):
305 for i in range(0x10000):
306 lines = (chr(i) + 'A').splitlines()
307 if i in (0x0a, 0x0b, 0x0c, 0x0d, 0x85,
308 0x1c, 0x1d, 0x1e, 0x2028, 0x2029):
309 self.assertEqual(len(lines), 2,
310 r"\u%.4x should be a linebreak" % i)
311 else:
312 self.assertEqual(len(lines), 1,
313 r"\u%.4x should not be a linebreak" % i)
314
Walter Dörwald37c47282003-02-26 14:49:41 +0000315def test_main():
Benjamin Petersonee8712c2008-05-20 21:35:26 +0000316 test.support.run_unittest(
Walter Dörwald21d3a322003-05-01 17:45:56 +0000317 UnicodeMiscTest,
318 UnicodeMethodsTest,
319 UnicodeFunctionsTest
320 )
Walter Dörwald37c47282003-02-26 14:49:41 +0000321
322if __name__ == "__main__":
323 test_main()