Guido van Rossum | 24bdb04 | 2000-03-28 20:29:59 +0000 | [diff] [blame] | 1 | """ Test script for the unicodedata module. |
| 2 | |
Marc-André Lemburg | 6a20ee7 | 2000-09-26 16:18:58 +0000 | [diff] [blame] | 3 | Written by Marc-Andre Lemburg (mal@lemburg.com). |
Guido van Rossum | 24bdb04 | 2000-03-28 20:29:59 +0000 | [diff] [blame] | 4 | |
Marc-André Lemburg | 6a20ee7 | 2000-09-26 16:18:58 +0000 | [diff] [blame] | 5 | (c) Copyright CNRI, All Rights Reserved. NO WARRANTY. |
Guido van Rossum | 24bdb04 | 2000-03-28 20:29:59 +0000 | [diff] [blame] | 6 | |
| 7 | """#" |
Walter Dörwald | 37c4728 | 2003-02-26 14:49:41 +0000 | [diff] [blame] | 8 | import unittest, test.test_support |
Thomas Wouters | 477c8d5 | 2006-05-27 19:21:47 +0000 | [diff] [blame] | 9 | import hashlib |
Guido van Rossum | 24bdb04 | 2000-03-28 20:29:59 +0000 | [diff] [blame] | 10 | |
Marc-André Lemburg | 67ceca7 | 2000-09-27 12:24:34 +0000 | [diff] [blame] | 11 | encoding = 'utf-8' |
| 12 | |
Marc-André Lemburg | 6a20ee7 | 2000-09-26 16:18:58 +0000 | [diff] [blame] | 13 | |
| 14 | ### Run tests |
| 15 | |
Walter Dörwald | 37c4728 | 2003-02-26 14:49:41 +0000 | [diff] [blame] | 16 | class UnicodeMethodsTest(unittest.TestCase): |
Marc-André Lemburg | 6a20ee7 | 2000-09-26 16:18:58 +0000 | [diff] [blame] | 17 | |
Walter Dörwald | 37c4728 | 2003-02-26 14:49:41 +0000 | [diff] [blame] | 18 | # update this, if the database changes |
Thomas Wouters | 477c8d5 | 2006-05-27 19:21:47 +0000 | [diff] [blame] | 19 | expectedchecksum = 'c198ed264497f108434b3f576d4107237221cc8a' |
Guido van Rossum | 24bdb04 | 2000-03-28 20:29:59 +0000 | [diff] [blame] | 20 | |
Walter Dörwald | 37c4728 | 2003-02-26 14:49:41 +0000 | [diff] [blame] | 21 | def test_method_checksum(self): |
Thomas Wouters | 477c8d5 | 2006-05-27 19:21:47 +0000 | [diff] [blame] | 22 | h = hashlib.sha1() |
Walter Dörwald | 37c4728 | 2003-02-26 14:49:41 +0000 | [diff] [blame] | 23 | for i in range(65536): |
Guido van Rossum | 84fc66d | 2007-05-03 17:18:26 +0000 | [diff] [blame] | 24 | char = chr(i) |
Walter Dörwald | 37c4728 | 2003-02-26 14:49:41 +0000 | [diff] [blame] | 25 | data = [ |
| 26 | # Predicates (single char) |
Guido van Rossum | ef87d6e | 2007-05-02 19:09:54 +0000 | [diff] [blame] | 27 | "01"[char.isalnum()], |
| 28 | "01"[char.isalpha()], |
| 29 | "01"[char.isdecimal()], |
| 30 | "01"[char.isdigit()], |
| 31 | "01"[char.islower()], |
| 32 | "01"[char.isnumeric()], |
| 33 | "01"[char.isspace()], |
| 34 | "01"[char.istitle()], |
| 35 | "01"[char.isupper()], |
Guido van Rossum | 24bdb04 | 2000-03-28 20:29:59 +0000 | [diff] [blame] | 36 | |
Walter Dörwald | 37c4728 | 2003-02-26 14:49:41 +0000 | [diff] [blame] | 37 | # Predicates (multiple chars) |
Guido van Rossum | ef87d6e | 2007-05-02 19:09:54 +0000 | [diff] [blame] | 38 | "01"[(char + 'abc').isalnum()], |
| 39 | "01"[(char + 'abc').isalpha()], |
| 40 | "01"[(char + '123').isdecimal()], |
| 41 | "01"[(char + '123').isdigit()], |
| 42 | "01"[(char + 'abc').islower()], |
| 43 | "01"[(char + '123').isnumeric()], |
| 44 | "01"[(char + ' \t').isspace()], |
| 45 | "01"[(char + 'abc').istitle()], |
| 46 | "01"[(char + 'ABC').isupper()], |
Guido van Rossum | 24bdb04 | 2000-03-28 20:29:59 +0000 | [diff] [blame] | 47 | |
Walter Dörwald | 37c4728 | 2003-02-26 14:49:41 +0000 | [diff] [blame] | 48 | # Mappings (single char) |
| 49 | char.lower(), |
| 50 | char.upper(), |
| 51 | char.title(), |
Guido van Rossum | 24bdb04 | 2000-03-28 20:29:59 +0000 | [diff] [blame] | 52 | |
Walter Dörwald | 37c4728 | 2003-02-26 14:49:41 +0000 | [diff] [blame] | 53 | # Mappings (multiple chars) |
Guido van Rossum | ef87d6e | 2007-05-02 19:09:54 +0000 | [diff] [blame] | 54 | (char + 'abc').lower(), |
| 55 | (char + 'ABC').upper(), |
| 56 | (char + 'abc').title(), |
| 57 | (char + 'ABC').title(), |
Guido van Rossum | 24bdb04 | 2000-03-28 20:29:59 +0000 | [diff] [blame] | 58 | |
Walter Dörwald | 37c4728 | 2003-02-26 14:49:41 +0000 | [diff] [blame] | 59 | ] |
Guido van Rossum | ef87d6e | 2007-05-02 19:09:54 +0000 | [diff] [blame] | 60 | h.update(''.join(data).encode(encoding)) |
Walter Dörwald | 37c4728 | 2003-02-26 14:49:41 +0000 | [diff] [blame] | 61 | result = h.hexdigest() |
| 62 | self.assertEqual(result, self.expectedchecksum) |
Guido van Rossum | 24bdb04 | 2000-03-28 20:29:59 +0000 | [diff] [blame] | 63 | |
Walter Dörwald | 37c4728 | 2003-02-26 14:49:41 +0000 | [diff] [blame] | 64 | class UnicodeDatabaseTest(unittest.TestCase): |
Guido van Rossum | 24bdb04 | 2000-03-28 20:29:59 +0000 | [diff] [blame] | 65 | |
Walter Dörwald | 37c4728 | 2003-02-26 14:49:41 +0000 | [diff] [blame] | 66 | def setUp(self): |
| 67 | # In case unicodedata is not available, this will raise an ImportError, |
| 68 | # but the other test cases will still be run |
| 69 | import unicodedata |
| 70 | self.db = unicodedata |
Guido van Rossum | 24bdb04 | 2000-03-28 20:29:59 +0000 | [diff] [blame] | 71 | |
Walter Dörwald | 37c4728 | 2003-02-26 14:49:41 +0000 | [diff] [blame] | 72 | def tearDown(self): |
| 73 | del self.db |
Guido van Rossum | 24bdb04 | 2000-03-28 20:29:59 +0000 | [diff] [blame] | 74 | |
Walter Dörwald | 37c4728 | 2003-02-26 14:49:41 +0000 | [diff] [blame] | 75 | class UnicodeFunctionsTest(UnicodeDatabaseTest): |
Guido van Rossum | 24bdb04 | 2000-03-28 20:29:59 +0000 | [diff] [blame] | 76 | |
Walter Dörwald | 37c4728 | 2003-02-26 14:49:41 +0000 | [diff] [blame] | 77 | # update this, if the database changes |
Thomas Wouters | 477c8d5 | 2006-05-27 19:21:47 +0000 | [diff] [blame] | 78 | expectedchecksum = '4e389f97e9f88b8b7ab743121fd643089116f9f2' |
Walter Dörwald | 37c4728 | 2003-02-26 14:49:41 +0000 | [diff] [blame] | 79 | |
| 80 | def test_function_checksum(self): |
| 81 | data = [] |
Thomas Wouters | 477c8d5 | 2006-05-27 19:21:47 +0000 | [diff] [blame] | 82 | h = hashlib.sha1() |
Walter Dörwald | 37c4728 | 2003-02-26 14:49:41 +0000 | [diff] [blame] | 83 | |
| 84 | for i in range(0x10000): |
Guido van Rossum | 84fc66d | 2007-05-03 17:18:26 +0000 | [diff] [blame] | 85 | char = chr(i) |
Walter Dörwald | 37c4728 | 2003-02-26 14:49:41 +0000 | [diff] [blame] | 86 | data = [ |
| 87 | # Properties |
| 88 | str(self.db.digit(char, -1)), |
| 89 | str(self.db.numeric(char, -1)), |
| 90 | str(self.db.decimal(char, -1)), |
| 91 | self.db.category(char), |
| 92 | self.db.bidirectional(char), |
| 93 | self.db.decomposition(char), |
| 94 | str(self.db.mirrored(char)), |
| 95 | str(self.db.combining(char)), |
| 96 | ] |
| 97 | h.update(''.join(data)) |
| 98 | result = h.hexdigest() |
| 99 | self.assertEqual(result, self.expectedchecksum) |
| 100 | |
| 101 | def test_digit(self): |
Guido van Rossum | ef87d6e | 2007-05-02 19:09:54 +0000 | [diff] [blame] | 102 | self.assertEqual(self.db.digit('A', None), None) |
| 103 | self.assertEqual(self.db.digit('9'), 9) |
| 104 | self.assertEqual(self.db.digit('\u215b', None), None) |
| 105 | self.assertEqual(self.db.digit('\u2468'), 9) |
Walter Dörwald | 37c4728 | 2003-02-26 14:49:41 +0000 | [diff] [blame] | 106 | |
| 107 | self.assertRaises(TypeError, self.db.digit) |
Guido van Rossum | ef87d6e | 2007-05-02 19:09:54 +0000 | [diff] [blame] | 108 | self.assertRaises(TypeError, self.db.digit, 'xx') |
| 109 | self.assertRaises(ValueError, self.db.digit, 'x') |
Walter Dörwald | 37c4728 | 2003-02-26 14:49:41 +0000 | [diff] [blame] | 110 | |
| 111 | def test_numeric(self): |
Guido van Rossum | ef87d6e | 2007-05-02 19:09:54 +0000 | [diff] [blame] | 112 | self.assertEqual(self.db.numeric('A',None), None) |
| 113 | self.assertEqual(self.db.numeric('9'), 9) |
| 114 | self.assertEqual(self.db.numeric('\u215b'), 0.125) |
| 115 | self.assertEqual(self.db.numeric('\u2468'), 9.0) |
Walter Dörwald | 37c4728 | 2003-02-26 14:49:41 +0000 | [diff] [blame] | 116 | |
| 117 | self.assertRaises(TypeError, self.db.numeric) |
Guido van Rossum | ef87d6e | 2007-05-02 19:09:54 +0000 | [diff] [blame] | 118 | self.assertRaises(TypeError, self.db.numeric, 'xx') |
| 119 | self.assertRaises(ValueError, self.db.numeric, 'x') |
Walter Dörwald | 37c4728 | 2003-02-26 14:49:41 +0000 | [diff] [blame] | 120 | |
| 121 | def test_decimal(self): |
Guido van Rossum | ef87d6e | 2007-05-02 19:09:54 +0000 | [diff] [blame] | 122 | self.assertEqual(self.db.decimal('A',None), None) |
| 123 | self.assertEqual(self.db.decimal('9'), 9) |
| 124 | self.assertEqual(self.db.decimal('\u215b', None), None) |
| 125 | self.assertEqual(self.db.decimal('\u2468', None), None) |
Walter Dörwald | 37c4728 | 2003-02-26 14:49:41 +0000 | [diff] [blame] | 126 | |
| 127 | self.assertRaises(TypeError, self.db.decimal) |
Guido van Rossum | ef87d6e | 2007-05-02 19:09:54 +0000 | [diff] [blame] | 128 | self.assertRaises(TypeError, self.db.decimal, 'xx') |
| 129 | self.assertRaises(ValueError, self.db.decimal, 'x') |
Walter Dörwald | 37c4728 | 2003-02-26 14:49:41 +0000 | [diff] [blame] | 130 | |
| 131 | def test_category(self): |
Guido van Rossum | ef87d6e | 2007-05-02 19:09:54 +0000 | [diff] [blame] | 132 | self.assertEqual(self.db.category('\uFFFE'), 'Cn') |
| 133 | self.assertEqual(self.db.category('a'), 'Ll') |
| 134 | self.assertEqual(self.db.category('A'), 'Lu') |
Walter Dörwald | 37c4728 | 2003-02-26 14:49:41 +0000 | [diff] [blame] | 135 | |
| 136 | self.assertRaises(TypeError, self.db.category) |
Guido van Rossum | ef87d6e | 2007-05-02 19:09:54 +0000 | [diff] [blame] | 137 | self.assertRaises(TypeError, self.db.category, 'xx') |
Walter Dörwald | 37c4728 | 2003-02-26 14:49:41 +0000 | [diff] [blame] | 138 | |
| 139 | def test_bidirectional(self): |
Guido van Rossum | ef87d6e | 2007-05-02 19:09:54 +0000 | [diff] [blame] | 140 | self.assertEqual(self.db.bidirectional('\uFFFE'), '') |
| 141 | self.assertEqual(self.db.bidirectional(' '), 'WS') |
| 142 | self.assertEqual(self.db.bidirectional('A'), 'L') |
Walter Dörwald | 37c4728 | 2003-02-26 14:49:41 +0000 | [diff] [blame] | 143 | |
| 144 | self.assertRaises(TypeError, self.db.bidirectional) |
Guido van Rossum | ef87d6e | 2007-05-02 19:09:54 +0000 | [diff] [blame] | 145 | self.assertRaises(TypeError, self.db.bidirectional, 'xx') |
Walter Dörwald | 37c4728 | 2003-02-26 14:49:41 +0000 | [diff] [blame] | 146 | |
| 147 | def test_decomposition(self): |
Guido van Rossum | ef87d6e | 2007-05-02 19:09:54 +0000 | [diff] [blame] | 148 | self.assertEqual(self.db.decomposition('\uFFFE'),'') |
| 149 | self.assertEqual(self.db.decomposition('\u00bc'), '<fraction> 0031 2044 0034') |
Walter Dörwald | 37c4728 | 2003-02-26 14:49:41 +0000 | [diff] [blame] | 150 | |
| 151 | self.assertRaises(TypeError, self.db.decomposition) |
Guido van Rossum | ef87d6e | 2007-05-02 19:09:54 +0000 | [diff] [blame] | 152 | self.assertRaises(TypeError, self.db.decomposition, 'xx') |
Walter Dörwald | 37c4728 | 2003-02-26 14:49:41 +0000 | [diff] [blame] | 153 | |
| 154 | def test_mirrored(self): |
Guido van Rossum | ef87d6e | 2007-05-02 19:09:54 +0000 | [diff] [blame] | 155 | self.assertEqual(self.db.mirrored('\uFFFE'), 0) |
| 156 | self.assertEqual(self.db.mirrored('a'), 0) |
| 157 | self.assertEqual(self.db.mirrored('\u2201'), 1) |
Walter Dörwald | 37c4728 | 2003-02-26 14:49:41 +0000 | [diff] [blame] | 158 | |
| 159 | self.assertRaises(TypeError, self.db.mirrored) |
Guido van Rossum | ef87d6e | 2007-05-02 19:09:54 +0000 | [diff] [blame] | 160 | self.assertRaises(TypeError, self.db.mirrored, 'xx') |
Walter Dörwald | 37c4728 | 2003-02-26 14:49:41 +0000 | [diff] [blame] | 161 | |
| 162 | def test_combining(self): |
Guido van Rossum | ef87d6e | 2007-05-02 19:09:54 +0000 | [diff] [blame] | 163 | self.assertEqual(self.db.combining('\uFFFE'), 0) |
| 164 | self.assertEqual(self.db.combining('a'), 0) |
| 165 | self.assertEqual(self.db.combining('\u20e1'), 230) |
Walter Dörwald | 37c4728 | 2003-02-26 14:49:41 +0000 | [diff] [blame] | 166 | |
| 167 | self.assertRaises(TypeError, self.db.combining) |
Guido van Rossum | ef87d6e | 2007-05-02 19:09:54 +0000 | [diff] [blame] | 168 | self.assertRaises(TypeError, self.db.combining, 'xx') |
Walter Dörwald | 37c4728 | 2003-02-26 14:49:41 +0000 | [diff] [blame] | 169 | |
| 170 | def test_normalize(self): |
| 171 | self.assertRaises(TypeError, self.db.normalize) |
Guido van Rossum | ef87d6e | 2007-05-02 19:09:54 +0000 | [diff] [blame] | 172 | self.assertRaises(ValueError, self.db.normalize, 'unknown', 'xx') |
| 173 | self.assertEqual(self.db.normalize('NFKC', ''), '') |
Walter Dörwald | 37c4728 | 2003-02-26 14:49:41 +0000 | [diff] [blame] | 174 | # The rest can be found in test_normalization.py |
| 175 | # which requires an external file. |
| 176 | |
Hye-Shik Chang | e9ddfbb | 2004-08-04 07:38:35 +0000 | [diff] [blame] | 177 | def test_east_asian_width(self): |
| 178 | eaw = self.db.east_asian_width |
| 179 | self.assertRaises(TypeError, eaw, 'a') |
Guido van Rossum | ef87d6e | 2007-05-02 19:09:54 +0000 | [diff] [blame] | 180 | self.assertRaises(TypeError, eaw, '') |
| 181 | self.assertRaises(TypeError, eaw, 'ra') |
| 182 | self.assertEqual(eaw('\x1e'), 'N') |
| 183 | self.assertEqual(eaw('\x20'), 'Na') |
| 184 | self.assertEqual(eaw('\uC894'), 'W') |
| 185 | self.assertEqual(eaw('\uFF66'), 'H') |
| 186 | self.assertEqual(eaw('\uFF1F'), 'F') |
| 187 | self.assertEqual(eaw('\u2010'), 'A') |
Walter Dörwald | 37c4728 | 2003-02-26 14:49:41 +0000 | [diff] [blame] | 188 | |
| 189 | class UnicodeMiscTest(UnicodeDatabaseTest): |
| 190 | |
| 191 | def test_decimal_numeric_consistent(self): |
| 192 | # Test that decimal and numeric are consistent, |
| 193 | # i.e. if a character has a decimal value, |
Georg Brandl | 7eb4b7d | 2005-07-22 21:49:32 +0000 | [diff] [blame] | 194 | # its numeric value should be the same. |
Walter Dörwald | 37c4728 | 2003-02-26 14:49:41 +0000 | [diff] [blame] | 195 | count = 0 |
Guido van Rossum | 805365e | 2007-05-07 22:24:25 +0000 | [diff] [blame^] | 196 | for i in range(0x10000): |
Guido van Rossum | 84fc66d | 2007-05-03 17:18:26 +0000 | [diff] [blame] | 197 | c = chr(i) |
Walter Dörwald | 37c4728 | 2003-02-26 14:49:41 +0000 | [diff] [blame] | 198 | dec = self.db.decimal(c, -1) |
| 199 | if dec != -1: |
| 200 | self.assertEqual(dec, self.db.numeric(c)) |
| 201 | count += 1 |
| 202 | self.assert_(count >= 10) # should have tested at least the ASCII digits |
| 203 | |
| 204 | def test_digit_numeric_consistent(self): |
| 205 | # Test that digit and numeric are consistent, |
Tim Peters | 669454e | 2003-03-07 17:30:48 +0000 | [diff] [blame] | 206 | # i.e. if a character has a digit value, |
Georg Brandl | 7eb4b7d | 2005-07-22 21:49:32 +0000 | [diff] [blame] | 207 | # its numeric value should be the same. |
Walter Dörwald | 37c4728 | 2003-02-26 14:49:41 +0000 | [diff] [blame] | 208 | count = 0 |
Guido van Rossum | 805365e | 2007-05-07 22:24:25 +0000 | [diff] [blame^] | 209 | for i in range(0x10000): |
Guido van Rossum | 84fc66d | 2007-05-03 17:18:26 +0000 | [diff] [blame] | 210 | c = chr(i) |
Walter Dörwald | 37c4728 | 2003-02-26 14:49:41 +0000 | [diff] [blame] | 211 | dec = self.db.digit(c, -1) |
| 212 | if dec != -1: |
| 213 | self.assertEqual(dec, self.db.numeric(c)) |
| 214 | count += 1 |
| 215 | self.assert_(count >= 10) # should have tested at least the ASCII digits |
| 216 | |
| 217 | def test_main(): |
Walter Dörwald | 21d3a32 | 2003-05-01 17:45:56 +0000 | [diff] [blame] | 218 | test.test_support.run_unittest( |
| 219 | UnicodeMiscTest, |
| 220 | UnicodeMethodsTest, |
| 221 | UnicodeFunctionsTest |
| 222 | ) |
Walter Dörwald | 37c4728 | 2003-02-26 14:49:41 +0000 | [diff] [blame] | 223 | |
| 224 | if __name__ == "__main__": |
| 225 | test_main() |