Guido van Rossum | 24bdb04 | 2000-03-28 20:29:59 +0000 | [diff] [blame] | 1 | """ Test script for the unicodedata module. |
| 2 | |
Marc-André Lemburg | 6a20ee7 | 2000-09-26 16:18:58 +0000 | [diff] [blame] | 3 | Written by Marc-Andre Lemburg (mal@lemburg.com). |
Guido van Rossum | 24bdb04 | 2000-03-28 20:29:59 +0000 | [diff] [blame] | 4 | |
Marc-André Lemburg | 6a20ee7 | 2000-09-26 16:18:58 +0000 | [diff] [blame] | 5 | (c) Copyright CNRI, All Rights Reserved. NO WARRANTY. |
Guido van Rossum | 24bdb04 | 2000-03-28 20:29:59 +0000 | [diff] [blame] | 6 | |
| 7 | """#" |
Walter Dörwald | 37c4728 | 2003-02-26 14:49:41 +0000 | [diff] [blame] | 8 | import unittest, test.test_support |
Thomas Wouters | 477c8d5 | 2006-05-27 19:21:47 +0000 | [diff] [blame^] | 9 | import hashlib |
Guido van Rossum | 24bdb04 | 2000-03-28 20:29:59 +0000 | [diff] [blame] | 10 | |
Marc-André Lemburg | 67ceca7 | 2000-09-27 12:24:34 +0000 | [diff] [blame] | 11 | encoding = 'utf-8' |
| 12 | |
Marc-André Lemburg | 6a20ee7 | 2000-09-26 16:18:58 +0000 | [diff] [blame] | 13 | |
| 14 | ### Run tests |
| 15 | |
Walter Dörwald | 37c4728 | 2003-02-26 14:49:41 +0000 | [diff] [blame] | 16 | class UnicodeMethodsTest(unittest.TestCase): |
Marc-André Lemburg | 6a20ee7 | 2000-09-26 16:18:58 +0000 | [diff] [blame] | 17 | |
Walter Dörwald | 37c4728 | 2003-02-26 14:49:41 +0000 | [diff] [blame] | 18 | # update this, if the database changes |
Thomas Wouters | 477c8d5 | 2006-05-27 19:21:47 +0000 | [diff] [blame^] | 19 | expectedchecksum = 'c198ed264497f108434b3f576d4107237221cc8a' |
Guido van Rossum | 24bdb04 | 2000-03-28 20:29:59 +0000 | [diff] [blame] | 20 | |
Walter Dörwald | 37c4728 | 2003-02-26 14:49:41 +0000 | [diff] [blame] | 21 | def test_method_checksum(self): |
Thomas Wouters | 477c8d5 | 2006-05-27 19:21:47 +0000 | [diff] [blame^] | 22 | h = hashlib.sha1() |
Walter Dörwald | 37c4728 | 2003-02-26 14:49:41 +0000 | [diff] [blame] | 23 | for i in range(65536): |
| 24 | char = unichr(i) |
| 25 | data = [ |
| 26 | # Predicates (single char) |
| 27 | u"01"[char.isalnum()], |
| 28 | u"01"[char.isalpha()], |
| 29 | u"01"[char.isdecimal()], |
| 30 | u"01"[char.isdigit()], |
| 31 | u"01"[char.islower()], |
| 32 | u"01"[char.isnumeric()], |
| 33 | u"01"[char.isspace()], |
| 34 | u"01"[char.istitle()], |
| 35 | u"01"[char.isupper()], |
Guido van Rossum | 24bdb04 | 2000-03-28 20:29:59 +0000 | [diff] [blame] | 36 | |
Walter Dörwald | 37c4728 | 2003-02-26 14:49:41 +0000 | [diff] [blame] | 37 | # Predicates (multiple chars) |
| 38 | u"01"[(char + u'abc').isalnum()], |
| 39 | u"01"[(char + u'abc').isalpha()], |
| 40 | u"01"[(char + u'123').isdecimal()], |
| 41 | u"01"[(char + u'123').isdigit()], |
| 42 | u"01"[(char + u'abc').islower()], |
| 43 | u"01"[(char + u'123').isnumeric()], |
| 44 | u"01"[(char + u' \t').isspace()], |
| 45 | u"01"[(char + u'abc').istitle()], |
| 46 | u"01"[(char + u'ABC').isupper()], |
Guido van Rossum | 24bdb04 | 2000-03-28 20:29:59 +0000 | [diff] [blame] | 47 | |
Walter Dörwald | 37c4728 | 2003-02-26 14:49:41 +0000 | [diff] [blame] | 48 | # Mappings (single char) |
| 49 | char.lower(), |
| 50 | char.upper(), |
| 51 | char.title(), |
Guido van Rossum | 24bdb04 | 2000-03-28 20:29:59 +0000 | [diff] [blame] | 52 | |
Walter Dörwald | 37c4728 | 2003-02-26 14:49:41 +0000 | [diff] [blame] | 53 | # Mappings (multiple chars) |
| 54 | (char + u'abc').lower(), |
| 55 | (char + u'ABC').upper(), |
| 56 | (char + u'abc').title(), |
| 57 | (char + u'ABC').title(), |
Guido van Rossum | 24bdb04 | 2000-03-28 20:29:59 +0000 | [diff] [blame] | 58 | |
Walter Dörwald | 37c4728 | 2003-02-26 14:49:41 +0000 | [diff] [blame] | 59 | ] |
| 60 | h.update(u''.join(data).encode(encoding)) |
| 61 | result = h.hexdigest() |
| 62 | self.assertEqual(result, self.expectedchecksum) |
Guido van Rossum | 24bdb04 | 2000-03-28 20:29:59 +0000 | [diff] [blame] | 63 | |
Walter Dörwald | 37c4728 | 2003-02-26 14:49:41 +0000 | [diff] [blame] | 64 | class UnicodeDatabaseTest(unittest.TestCase): |
Guido van Rossum | 24bdb04 | 2000-03-28 20:29:59 +0000 | [diff] [blame] | 65 | |
Walter Dörwald | 37c4728 | 2003-02-26 14:49:41 +0000 | [diff] [blame] | 66 | def setUp(self): |
| 67 | # In case unicodedata is not available, this will raise an ImportError, |
| 68 | # but the other test cases will still be run |
| 69 | import unicodedata |
| 70 | self.db = unicodedata |
Guido van Rossum | 24bdb04 | 2000-03-28 20:29:59 +0000 | [diff] [blame] | 71 | |
Walter Dörwald | 37c4728 | 2003-02-26 14:49:41 +0000 | [diff] [blame] | 72 | def tearDown(self): |
| 73 | del self.db |
Guido van Rossum | 24bdb04 | 2000-03-28 20:29:59 +0000 | [diff] [blame] | 74 | |
Walter Dörwald | 37c4728 | 2003-02-26 14:49:41 +0000 | [diff] [blame] | 75 | class UnicodeFunctionsTest(UnicodeDatabaseTest): |
Guido van Rossum | 24bdb04 | 2000-03-28 20:29:59 +0000 | [diff] [blame] | 76 | |
Walter Dörwald | 37c4728 | 2003-02-26 14:49:41 +0000 | [diff] [blame] | 77 | # update this, if the database changes |
Thomas Wouters | 477c8d5 | 2006-05-27 19:21:47 +0000 | [diff] [blame^] | 78 | expectedchecksum = '4e389f97e9f88b8b7ab743121fd643089116f9f2' |
Walter Dörwald | 37c4728 | 2003-02-26 14:49:41 +0000 | [diff] [blame] | 79 | |
| 80 | def test_function_checksum(self): |
| 81 | data = [] |
Thomas Wouters | 477c8d5 | 2006-05-27 19:21:47 +0000 | [diff] [blame^] | 82 | h = hashlib.sha1() |
Walter Dörwald | 37c4728 | 2003-02-26 14:49:41 +0000 | [diff] [blame] | 83 | |
| 84 | for i in range(0x10000): |
| 85 | char = unichr(i) |
| 86 | data = [ |
| 87 | # Properties |
| 88 | str(self.db.digit(char, -1)), |
| 89 | str(self.db.numeric(char, -1)), |
| 90 | str(self.db.decimal(char, -1)), |
| 91 | self.db.category(char), |
| 92 | self.db.bidirectional(char), |
| 93 | self.db.decomposition(char), |
| 94 | str(self.db.mirrored(char)), |
| 95 | str(self.db.combining(char)), |
| 96 | ] |
| 97 | h.update(''.join(data)) |
| 98 | result = h.hexdigest() |
| 99 | self.assertEqual(result, self.expectedchecksum) |
| 100 | |
| 101 | def test_digit(self): |
| 102 | self.assertEqual(self.db.digit(u'A', None), None) |
| 103 | self.assertEqual(self.db.digit(u'9'), 9) |
| 104 | self.assertEqual(self.db.digit(u'\u215b', None), None) |
| 105 | self.assertEqual(self.db.digit(u'\u2468'), 9) |
| 106 | |
| 107 | self.assertRaises(TypeError, self.db.digit) |
| 108 | self.assertRaises(TypeError, self.db.digit, u'xx') |
| 109 | self.assertRaises(ValueError, self.db.digit, u'x') |
| 110 | |
| 111 | def test_numeric(self): |
| 112 | self.assertEqual(self.db.numeric(u'A',None), None) |
| 113 | self.assertEqual(self.db.numeric(u'9'), 9) |
| 114 | self.assertEqual(self.db.numeric(u'\u215b'), 0.125) |
| 115 | self.assertEqual(self.db.numeric(u'\u2468'), 9.0) |
| 116 | |
| 117 | self.assertRaises(TypeError, self.db.numeric) |
| 118 | self.assertRaises(TypeError, self.db.numeric, u'xx') |
| 119 | self.assertRaises(ValueError, self.db.numeric, u'x') |
| 120 | |
| 121 | def test_decimal(self): |
| 122 | self.assertEqual(self.db.decimal(u'A',None), None) |
| 123 | self.assertEqual(self.db.decimal(u'9'), 9) |
| 124 | self.assertEqual(self.db.decimal(u'\u215b', None), None) |
| 125 | self.assertEqual(self.db.decimal(u'\u2468', None), None) |
| 126 | |
| 127 | self.assertRaises(TypeError, self.db.decimal) |
| 128 | self.assertRaises(TypeError, self.db.decimal, u'xx') |
| 129 | self.assertRaises(ValueError, self.db.decimal, u'x') |
| 130 | |
| 131 | def test_category(self): |
| 132 | self.assertEqual(self.db.category(u'\uFFFE'), 'Cn') |
| 133 | self.assertEqual(self.db.category(u'a'), 'Ll') |
| 134 | self.assertEqual(self.db.category(u'A'), 'Lu') |
| 135 | |
| 136 | self.assertRaises(TypeError, self.db.category) |
| 137 | self.assertRaises(TypeError, self.db.category, u'xx') |
| 138 | |
| 139 | def test_bidirectional(self): |
| 140 | self.assertEqual(self.db.bidirectional(u'\uFFFE'), '') |
| 141 | self.assertEqual(self.db.bidirectional(u' '), 'WS') |
| 142 | self.assertEqual(self.db.bidirectional(u'A'), 'L') |
| 143 | |
| 144 | self.assertRaises(TypeError, self.db.bidirectional) |
| 145 | self.assertRaises(TypeError, self.db.bidirectional, u'xx') |
| 146 | |
| 147 | def test_decomposition(self): |
| 148 | self.assertEqual(self.db.decomposition(u'\uFFFE'),'') |
| 149 | self.assertEqual(self.db.decomposition(u'\u00bc'), '<fraction> 0031 2044 0034') |
| 150 | |
| 151 | self.assertRaises(TypeError, self.db.decomposition) |
| 152 | self.assertRaises(TypeError, self.db.decomposition, u'xx') |
| 153 | |
| 154 | def test_mirrored(self): |
| 155 | self.assertEqual(self.db.mirrored(u'\uFFFE'), 0) |
| 156 | self.assertEqual(self.db.mirrored(u'a'), 0) |
| 157 | self.assertEqual(self.db.mirrored(u'\u2201'), 1) |
| 158 | |
| 159 | self.assertRaises(TypeError, self.db.mirrored) |
| 160 | self.assertRaises(TypeError, self.db.mirrored, u'xx') |
| 161 | |
| 162 | def test_combining(self): |
| 163 | self.assertEqual(self.db.combining(u'\uFFFE'), 0) |
| 164 | self.assertEqual(self.db.combining(u'a'), 0) |
| 165 | self.assertEqual(self.db.combining(u'\u20e1'), 230) |
| 166 | |
| 167 | self.assertRaises(TypeError, self.db.combining) |
| 168 | self.assertRaises(TypeError, self.db.combining, u'xx') |
| 169 | |
| 170 | def test_normalize(self): |
| 171 | self.assertRaises(TypeError, self.db.normalize) |
| 172 | self.assertRaises(ValueError, self.db.normalize, 'unknown', u'xx') |
Martin v. Löwis | 61e40bd | 2004-04-17 19:36:48 +0000 | [diff] [blame] | 173 | self.assertEqual(self.db.normalize('NFKC', u''), u'') |
Walter Dörwald | 37c4728 | 2003-02-26 14:49:41 +0000 | [diff] [blame] | 174 | # The rest can be found in test_normalization.py |
| 175 | # which requires an external file. |
| 176 | |
Hye-Shik Chang | e9ddfbb | 2004-08-04 07:38:35 +0000 | [diff] [blame] | 177 | def test_east_asian_width(self): |
| 178 | eaw = self.db.east_asian_width |
| 179 | self.assertRaises(TypeError, eaw, 'a') |
| 180 | self.assertRaises(TypeError, eaw, u'') |
| 181 | self.assertRaises(TypeError, eaw, u'ra') |
| 182 | self.assertEqual(eaw(u'\x1e'), 'N') |
| 183 | self.assertEqual(eaw(u'\x20'), 'Na') |
| 184 | self.assertEqual(eaw(u'\uC894'), 'W') |
| 185 | self.assertEqual(eaw(u'\uFF66'), 'H') |
| 186 | self.assertEqual(eaw(u'\uFF1F'), 'F') |
| 187 | self.assertEqual(eaw(u'\u2010'), 'A') |
Walter Dörwald | 37c4728 | 2003-02-26 14:49:41 +0000 | [diff] [blame] | 188 | |
| 189 | class UnicodeMiscTest(UnicodeDatabaseTest): |
| 190 | |
| 191 | def test_decimal_numeric_consistent(self): |
| 192 | # Test that decimal and numeric are consistent, |
| 193 | # i.e. if a character has a decimal value, |
Georg Brandl | 7eb4b7d | 2005-07-22 21:49:32 +0000 | [diff] [blame] | 194 | # its numeric value should be the same. |
Walter Dörwald | 37c4728 | 2003-02-26 14:49:41 +0000 | [diff] [blame] | 195 | count = 0 |
| 196 | for i in xrange(0x10000): |
| 197 | c = unichr(i) |
| 198 | dec = self.db.decimal(c, -1) |
| 199 | if dec != -1: |
| 200 | self.assertEqual(dec, self.db.numeric(c)) |
| 201 | count += 1 |
| 202 | self.assert_(count >= 10) # should have tested at least the ASCII digits |
| 203 | |
| 204 | def test_digit_numeric_consistent(self): |
| 205 | # Test that digit and numeric are consistent, |
Tim Peters | 669454e | 2003-03-07 17:30:48 +0000 | [diff] [blame] | 206 | # i.e. if a character has a digit value, |
Georg Brandl | 7eb4b7d | 2005-07-22 21:49:32 +0000 | [diff] [blame] | 207 | # its numeric value should be the same. |
Walter Dörwald | 37c4728 | 2003-02-26 14:49:41 +0000 | [diff] [blame] | 208 | count = 0 |
| 209 | for i in xrange(0x10000): |
| 210 | c = unichr(i) |
| 211 | dec = self.db.digit(c, -1) |
| 212 | if dec != -1: |
| 213 | self.assertEqual(dec, self.db.numeric(c)) |
| 214 | count += 1 |
| 215 | self.assert_(count >= 10) # should have tested at least the ASCII digits |
| 216 | |
| 217 | def test_main(): |
Walter Dörwald | 21d3a32 | 2003-05-01 17:45:56 +0000 | [diff] [blame] | 218 | test.test_support.run_unittest( |
| 219 | UnicodeMiscTest, |
| 220 | UnicodeMethodsTest, |
| 221 | UnicodeFunctionsTest |
| 222 | ) |
Walter Dörwald | 37c4728 | 2003-02-26 14:49:41 +0000 | [diff] [blame] | 223 | |
| 224 | if __name__ == "__main__": |
| 225 | test_main() |