Benjamin Peterson | ee8712c | 2008-05-20 21:35:26 +0000 | [diff] [blame] | 1 | from test.support import run_unittest, open_urlresource |
Guido van Rossum | d8faa36 | 2007-04-27 19:54:29 +0000 | [diff] [blame] | 2 | import unittest |
| 3 | |
Antoine Pitrou | 1a305fd | 2009-11-26 12:38:23 +0000 | [diff] [blame] | 4 | from http.client import HTTPException |
Martin v. Löwis | 677bde2 | 2002-11-23 22:08:15 +0000 | [diff] [blame] | 5 | import sys |
Tim Peters | 1b445d3 | 2002-11-24 18:53:11 +0000 | [diff] [blame] | 6 | import os |
Martin v. Löwis | bb417dc | 2008-09-10 21:15:32 +0000 | [diff] [blame] | 7 | from unicodedata import normalize, unidata_version |
Tim Peters | 1b445d3 | 2002-11-24 18:53:11 +0000 | [diff] [blame] | 8 | |
Skip Montanaro | 7a98be2 | 2007-08-16 14:35:24 +0000 | [diff] [blame] | 9 | TESTDATAFILE = "NormalizationTest.txt" |
Martin v. Löwis | bb417dc | 2008-09-10 21:15:32 +0000 | [diff] [blame] | 10 | TESTDATAURL = "http://www.unicode.org/Public/" + unidata_version + "/ucd/" + TESTDATAFILE |
| 11 | |
Florent Xicluna | f089fd6 | 2010-03-19 14:25:03 +0000 | [diff] [blame] | 12 | def check_version(testfile): |
| 13 | hdr = testfile.readline() |
| 14 | return unidata_version in hdr |
Martin v. Löwis | 677bde2 | 2002-11-23 22:08:15 +0000 | [diff] [blame] | 15 | |
Neal Norwitz | 1e32b69 | 2006-03-24 08:02:35 +0000 | [diff] [blame] | 16 | class RangeError(Exception): |
Martin v. Löwis | 677bde2 | 2002-11-23 22:08:15 +0000 | [diff] [blame] | 17 | pass |
| 18 | |
| 19 | def NFC(str): |
| 20 | return normalize("NFC", str) |
| 21 | |
| 22 | def NFKC(str): |
| 23 | return normalize("NFKC", str) |
| 24 | |
| 25 | def NFD(str): |
| 26 | return normalize("NFD", str) |
| 27 | |
| 28 | def NFKD(str): |
| 29 | return normalize("NFKD", str) |
| 30 | |
| 31 | def unistr(data): |
| 32 | data = [int(x, 16) for x in data.split(" ")] |
| 33 | for x in data: |
| 34 | if x > sys.maxunicode: |
| 35 | raise RangeError |
Guido van Rossum | 84fc66d | 2007-05-03 17:18:26 +0000 | [diff] [blame] | 36 | return "".join([chr(x) for x in data]) |
Martin v. Löwis | 677bde2 | 2002-11-23 22:08:15 +0000 | [diff] [blame] | 37 | |
Guido van Rossum | d8faa36 | 2007-04-27 19:54:29 +0000 | [diff] [blame] | 38 | class NormalizationTest(unittest.TestCase): |
| 39 | def test_main(self): |
Florent Xicluna | f089fd6 | 2010-03-19 14:25:03 +0000 | [diff] [blame] | 40 | part = None |
Guido van Rossum | d8faa36 | 2007-04-27 19:54:29 +0000 | [diff] [blame] | 41 | part1_data = {} |
Antoine Pitrou | 7e05e7d | 2009-11-01 21:43:20 +0000 | [diff] [blame] | 42 | # Hit the exception early |
| 43 | try: |
Florent Xicluna | f089fd6 | 2010-03-19 14:25:03 +0000 | [diff] [blame] | 44 | testdata = open_urlresource(TESTDATAURL, encoding="utf-8", |
| 45 | check=check_version) |
Antoine Pitrou | 1a305fd | 2009-11-26 12:38:23 +0000 | [diff] [blame] | 46 | except (IOError, HTTPException): |
Antoine Pitrou | 7e05e7d | 2009-11-01 21:43:20 +0000 | [diff] [blame] | 47 | self.skipTest("Could not retrieve " + TESTDATAURL) |
Florent Xicluna | f089fd6 | 2010-03-19 14:25:03 +0000 | [diff] [blame] | 48 | for line in testdata: |
Guido van Rossum | d8faa36 | 2007-04-27 19:54:29 +0000 | [diff] [blame] | 49 | if '#' in line: |
| 50 | line = line.split('#')[0] |
| 51 | line = line.strip() |
| 52 | if not line: |
| 53 | continue |
| 54 | if line.startswith("@Part"): |
| 55 | part = line.split()[0] |
| 56 | continue |
| 57 | if part == "@Part3": |
| 58 | # XXX we don't support PRI #29 yet, so skip these tests for now |
| 59 | continue |
| 60 | try: |
| 61 | c1,c2,c3,c4,c5 = [unistr(x) for x in line.split(';')[:-1]] |
| 62 | except RangeError: |
| 63 | # Skip unsupported characters; |
| 64 | # try atleast adding c1 if we are in part1 |
| 65 | if part == "@Part1": |
| 66 | try: |
| 67 | c1 = unistr(line.split(';')[0]) |
| 68 | except RangeError: |
| 69 | pass |
| 70 | else: |
| 71 | part1_data[c1] = 1 |
| 72 | continue |
| 73 | |
| 74 | # Perform tests |
Benjamin Peterson | c9c0f20 | 2009-06-30 23:06:06 +0000 | [diff] [blame] | 75 | self.assertTrue(c2 == NFC(c1) == NFC(c2) == NFC(c3), line) |
| 76 | self.assertTrue(c4 == NFC(c4) == NFC(c5), line) |
| 77 | self.assertTrue(c3 == NFD(c1) == NFD(c2) == NFD(c3), line) |
| 78 | self.assertTrue(c5 == NFD(c4) == NFD(c5), line) |
| 79 | self.assertTrue(c4 == NFKC(c1) == NFKC(c2) == \ |
Guido van Rossum | d8faa36 | 2007-04-27 19:54:29 +0000 | [diff] [blame] | 80 | NFKC(c3) == NFKC(c4) == NFKC(c5), |
| 81 | line) |
Benjamin Peterson | c9c0f20 | 2009-06-30 23:06:06 +0000 | [diff] [blame] | 82 | self.assertTrue(c5 == NFKD(c1) == NFKD(c2) == \ |
Guido van Rossum | d8faa36 | 2007-04-27 19:54:29 +0000 | [diff] [blame] | 83 | NFKD(c3) == NFKD(c4) == NFKD(c5), |
| 84 | line) |
| 85 | |
| 86 | # Record part 1 data |
Martin v. Löwis | 4196296 | 2006-03-10 11:59:47 +0000 | [diff] [blame] | 87 | if part == "@Part1": |
Guido van Rossum | d8faa36 | 2007-04-27 19:54:29 +0000 | [diff] [blame] | 88 | part1_data[c1] = 1 |
Martin v. Löwis | 677bde2 | 2002-11-23 22:08:15 +0000 | [diff] [blame] | 89 | |
Guido van Rossum | d8faa36 | 2007-04-27 19:54:29 +0000 | [diff] [blame] | 90 | # Perform tests for all other data |
| 91 | for c in range(sys.maxunicode+1): |
Guido van Rossum | 84fc66d | 2007-05-03 17:18:26 +0000 | [diff] [blame] | 92 | X = chr(c) |
Guido van Rossum | d8faa36 | 2007-04-27 19:54:29 +0000 | [diff] [blame] | 93 | if X in part1_data: |
| 94 | continue |
Benjamin Peterson | c9c0f20 | 2009-06-30 23:06:06 +0000 | [diff] [blame] | 95 | self.assertTrue(X == NFC(X) == NFD(X) == NFKC(X) == NFKD(X), c) |
Martin v. Löwis | 677bde2 | 2002-11-23 22:08:15 +0000 | [diff] [blame] | 96 | |
Guido van Rossum | d8faa36 | 2007-04-27 19:54:29 +0000 | [diff] [blame] | 97 | def test_bug_834676(self): |
| 98 | # Check for bug 834676 |
Guido van Rossum | ef87d6e | 2007-05-02 19:09:54 +0000 | [diff] [blame] | 99 | normalize('NFC', '\ud55c\uae00') |
Tim Peters | 1b445d3 | 2002-11-24 18:53:11 +0000 | [diff] [blame] | 100 | |
Tim Peters | 1b445d3 | 2002-11-24 18:53:11 +0000 | [diff] [blame] | 101 | |
Guido van Rossum | d8faa36 | 2007-04-27 19:54:29 +0000 | [diff] [blame] | 102 | def test_main(): |
Guido van Rossum | d8faa36 | 2007-04-27 19:54:29 +0000 | [diff] [blame] | 103 | run_unittest(NormalizationTest) |
Martin v. Löwis | d2171d2 | 2003-11-06 20:47:57 +0000 | [diff] [blame] | 104 | |
Tim Peters | 1b445d3 | 2002-11-24 18:53:11 +0000 | [diff] [blame] | 105 | if __name__ == "__main__": |
| 106 | test_main() |