Benjamin Peterson | ee8712c | 2008-05-20 21:35:26 +0000 | [diff] [blame] | 1 | from test.support import run_unittest, open_urlresource |
Guido van Rossum | d8faa36 | 2007-04-27 19:54:29 +0000 | [diff] [blame] | 2 | import unittest |
| 3 | |
Martin v. Löwis | 677bde2 | 2002-11-23 22:08:15 +0000 | [diff] [blame] | 4 | import sys |
Tim Peters | 1b445d3 | 2002-11-24 18:53:11 +0000 | [diff] [blame] | 5 | import os |
Martin v. Löwis | 677bde2 | 2002-11-23 22:08:15 +0000 | [diff] [blame] | 6 | from unicodedata import normalize |
Tim Peters | 1b445d3 | 2002-11-24 18:53:11 +0000 | [diff] [blame] | 7 | |
Skip Montanaro | 7a98be2 | 2007-08-16 14:35:24 +0000 | [diff] [blame] | 8 | TESTDATAFILE = "NormalizationTest.txt" |
Martin v. Löwis | 4196296 | 2006-03-10 11:59:47 +0000 | [diff] [blame] | 9 | TESTDATAURL = "http://www.unicode.org/Public/4.1.0/ucd/" + TESTDATAFILE |
Martin v. Löwis | 677bde2 | 2002-11-23 22:08:15 +0000 | [diff] [blame] | 10 | |
Neal Norwitz | 1e32b69 | 2006-03-24 08:02:35 +0000 | [diff] [blame] | 11 | class RangeError(Exception): |
Martin v. Löwis | 677bde2 | 2002-11-23 22:08:15 +0000 | [diff] [blame] | 12 | pass |
| 13 | |
| 14 | def NFC(str): |
| 15 | return normalize("NFC", str) |
| 16 | |
| 17 | def NFKC(str): |
| 18 | return normalize("NFKC", str) |
| 19 | |
| 20 | def NFD(str): |
| 21 | return normalize("NFD", str) |
| 22 | |
| 23 | def NFKD(str): |
| 24 | return normalize("NFKD", str) |
| 25 | |
| 26 | def unistr(data): |
| 27 | data = [int(x, 16) for x in data.split(" ")] |
| 28 | for x in data: |
| 29 | if x > sys.maxunicode: |
| 30 | raise RangeError |
Guido van Rossum | 84fc66d | 2007-05-03 17:18:26 +0000 | [diff] [blame] | 31 | return "".join([chr(x) for x in data]) |
Martin v. Löwis | 677bde2 | 2002-11-23 22:08:15 +0000 | [diff] [blame] | 32 | |
Guido van Rossum | d8faa36 | 2007-04-27 19:54:29 +0000 | [diff] [blame] | 33 | class NormalizationTest(unittest.TestCase): |
| 34 | def test_main(self): |
| 35 | part1_data = {} |
Martin v. Löwis | 234a34a | 2007-08-30 20:58:02 +0000 | [diff] [blame] | 36 | for line in open_urlresource(TESTDATAURL, encoding="utf-8"): |
Guido van Rossum | d8faa36 | 2007-04-27 19:54:29 +0000 | [diff] [blame] | 37 | if '#' in line: |
| 38 | line = line.split('#')[0] |
| 39 | line = line.strip() |
| 40 | if not line: |
| 41 | continue |
| 42 | if line.startswith("@Part"): |
| 43 | part = line.split()[0] |
| 44 | continue |
| 45 | if part == "@Part3": |
| 46 | # XXX we don't support PRI #29 yet, so skip these tests for now |
| 47 | continue |
| 48 | try: |
| 49 | c1,c2,c3,c4,c5 = [unistr(x) for x in line.split(';')[:-1]] |
| 50 | except RangeError: |
| 51 | # Skip unsupported characters; |
| 52 | # try atleast adding c1 if we are in part1 |
| 53 | if part == "@Part1": |
| 54 | try: |
| 55 | c1 = unistr(line.split(';')[0]) |
| 56 | except RangeError: |
| 57 | pass |
| 58 | else: |
| 59 | part1_data[c1] = 1 |
| 60 | continue |
| 61 | |
| 62 | # Perform tests |
| 63 | self.failUnless(c2 == NFC(c1) == NFC(c2) == NFC(c3), line) |
| 64 | self.failUnless(c4 == NFC(c4) == NFC(c5), line) |
| 65 | self.failUnless(c3 == NFD(c1) == NFD(c2) == NFD(c3), line) |
| 66 | self.failUnless(c5 == NFD(c4) == NFD(c5), line) |
| 67 | self.failUnless(c4 == NFKC(c1) == NFKC(c2) == \ |
| 68 | NFKC(c3) == NFKC(c4) == NFKC(c5), |
| 69 | line) |
| 70 | self.failUnless(c5 == NFKD(c1) == NFKD(c2) == \ |
| 71 | NFKD(c3) == NFKD(c4) == NFKD(c5), |
| 72 | line) |
| 73 | |
| 74 | # Record part 1 data |
Martin v. Löwis | 4196296 | 2006-03-10 11:59:47 +0000 | [diff] [blame] | 75 | if part == "@Part1": |
Guido van Rossum | d8faa36 | 2007-04-27 19:54:29 +0000 | [diff] [blame] | 76 | part1_data[c1] = 1 |
Martin v. Löwis | 677bde2 | 2002-11-23 22:08:15 +0000 | [diff] [blame] | 77 | |
Guido van Rossum | d8faa36 | 2007-04-27 19:54:29 +0000 | [diff] [blame] | 78 | # Perform tests for all other data |
| 79 | for c in range(sys.maxunicode+1): |
Guido van Rossum | 84fc66d | 2007-05-03 17:18:26 +0000 | [diff] [blame] | 80 | X = chr(c) |
Guido van Rossum | d8faa36 | 2007-04-27 19:54:29 +0000 | [diff] [blame] | 81 | if X in part1_data: |
| 82 | continue |
| 83 | self.failUnless(X == NFC(X) == NFD(X) == NFKC(X) == NFKD(X), c) |
Martin v. Löwis | 677bde2 | 2002-11-23 22:08:15 +0000 | [diff] [blame] | 84 | |
Guido van Rossum | d8faa36 | 2007-04-27 19:54:29 +0000 | [diff] [blame] | 85 | def test_bug_834676(self): |
| 86 | # Check for bug 834676 |
Guido van Rossum | ef87d6e | 2007-05-02 19:09:54 +0000 | [diff] [blame] | 87 | normalize('NFC', '\ud55c\uae00') |
Tim Peters | 1b445d3 | 2002-11-24 18:53:11 +0000 | [diff] [blame] | 88 | |
Tim Peters | 1b445d3 | 2002-11-24 18:53:11 +0000 | [diff] [blame] | 89 | |
Guido van Rossum | d8faa36 | 2007-04-27 19:54:29 +0000 | [diff] [blame] | 90 | def test_main(): |
| 91 | # Hit the exception early |
| 92 | open_urlresource(TESTDATAURL) |
| 93 | run_unittest(NormalizationTest) |
Martin v. Löwis | d2171d2 | 2003-11-06 20:47:57 +0000 | [diff] [blame] | 94 | |
Tim Peters | 1b445d3 | 2002-11-24 18:53:11 +0000 | [diff] [blame] | 95 | if __name__ == "__main__": |
| 96 | test_main() |