Mark Hammond | 7995eb2 | 2002-10-03 23:14:10 +0000 | [diff] [blame] | 1 | # Test the Unicode versions of normal file functions |
| 2 | # open, os.open, os.stat. os.listdir, os.rename, os.remove, os.mkdir, os.chdir, os.rmdir |
Victor Stinner | 1ab6c2d | 2011-11-15 22:27:41 +0100 | [diff] [blame] | 3 | import os |
| 4 | import sys |
| 5 | import unittest |
| 6 | import warnings |
Florent Xicluna | eb13618 | 2010-03-21 18:49:50 +0000 | [diff] [blame] | 7 | from unicodedata import normalize |
Benjamin Peterson | ee8712c | 2008-05-20 21:35:26 +0000 | [diff] [blame] | 8 | from test import support |
Mark Hammond | 7995eb2 | 2002-10-03 23:14:10 +0000 | [diff] [blame] | 9 | |
| 10 | filenames = [ |
Victor Stinner | 7362c4f | 2010-10-28 11:20:31 +0000 | [diff] [blame] | 11 | '1_abc', |
| 12 | '2_ascii', |
| 13 | '3_Gr\xfc\xdf-Gott', |
| 14 | '4_\u0393\u03b5\u03b9\u03ac-\u03c3\u03b1\u03c2', |
| 15 | '5_\u0417\u0434\u0440\u0430\u0432\u0441\u0442\u0432\u0443\u0439\u0442\u0435', |
| 16 | '6_\u306b\u307d\u3093', |
| 17 | '7_\u05d4\u05e9\u05e7\u05e6\u05e5\u05e1', |
| 18 | '8_\u66e8\u66e9\u66eb', |
| 19 | '9_\u66e8\u05e9\u3093\u0434\u0393\xdf', |
Florent Xicluna | eb13618 | 2010-03-21 18:49:50 +0000 | [diff] [blame] | 20 | # Specific code points: fn, NFC(fn) and NFKC(fn) all differents |
Victor Stinner | 7362c4f | 2010-10-28 11:20:31 +0000 | [diff] [blame] | 21 | '10_\u1fee\u1ffd', |
Mark Hammond | 7995eb2 | 2002-10-03 23:14:10 +0000 | [diff] [blame] | 22 | ] |
| 23 | |
Florent Xicluna | fd1b093 | 2010-03-28 00:25:02 +0000 | [diff] [blame] | 24 | # Mac OS X decomposes Unicode names, using Normal Form D. |
| 25 | # http://developer.apple.com/mac/library/qa/qa2001/qa1173.html |
| 26 | # "However, most volume formats do not follow the exact specification for |
| 27 | # these normal forms. For example, HFS Plus uses a variant of Normal Form D |
| 28 | # in which U+2000 through U+2FFF, U+F900 through U+FAFF, and U+2F800 through |
| 29 | # U+2FAFF are not decomposed." |
| 30 | if sys.platform != 'darwin': |
| 31 | filenames.extend([ |
Victor Stinner | fc6f5a4 | 2010-10-28 22:57:03 +0000 | [diff] [blame] | 32 | # Specific code points: NFC(fn), NFD(fn), NFKC(fn) and NFKD(fn) all differents |
| 33 | '11_\u0385\u03d3\u03d4', |
| 34 | '12_\u00a8\u0301\u03d2\u0301\u03d2\u0308', # == NFD('\u0385\u03d3\u03d4') |
| 35 | '13_\u0020\u0308\u0301\u038e\u03ab', # == NFKC('\u0385\u03d3\u03d4') |
| 36 | '14_\u1e9b\u1fc1\u1fcd\u1fce\u1fcf\u1fdd\u1fde\u1fdf\u1fed', |
| 37 | |
Florent Xicluna | fd1b093 | 2010-03-28 00:25:02 +0000 | [diff] [blame] | 38 | # Specific code points: fn, NFC(fn) and NFKC(fn) all differents |
Victor Stinner | 7362c4f | 2010-10-28 11:20:31 +0000 | [diff] [blame] | 39 | '15_\u1fee\u1ffd\ufad1', |
| 40 | '16_\u2000\u2000\u2000A', |
| 41 | '17_\u2001\u2001\u2001A', |
| 42 | '18_\u2003\u2003\u2003A', # == NFC('\u2001\u2001\u2001A') |
| 43 | '19_\u0020\u0020\u0020A', # '\u0020' == ' ' == NFKC('\u2000') == |
Florent Xicluna | 65eb453 | 2011-11-03 23:11:14 +0100 | [diff] [blame] | 44 | # NFKC('\u2001') == NFKC('\u2003') |
| 45 | ]) |
Florent Xicluna | eb13618 | 2010-03-21 18:49:50 +0000 | [diff] [blame] | 46 | |
Florent Xicluna | 87082ee | 2010-08-09 17:18:05 +0000 | [diff] [blame] | 47 | |
| 48 | # Is it Unicode-friendly? |
| 49 | if not os.path.supports_unicode_filenames: |
Victor Stinner | 94908bb | 2010-08-18 21:23:25 +0000 | [diff] [blame] | 50 | fsencoding = sys.getfilesystemencoding() |
Florent Xicluna | 87082ee | 2010-08-09 17:18:05 +0000 | [diff] [blame] | 51 | try: |
| 52 | for name in filenames: |
| 53 | name.encode(fsencoding) |
| 54 | except UnicodeEncodeError: |
| 55 | raise unittest.SkipTest("only NT+ and systems with " |
| 56 | "Unicode-friendly filesystem encoding") |
| 57 | |
| 58 | |
Mark Hammond | 7995eb2 | 2002-10-03 23:14:10 +0000 | [diff] [blame] | 59 | class UnicodeFileTests(unittest.TestCase): |
Florent Xicluna | eb13618 | 2010-03-21 18:49:50 +0000 | [diff] [blame] | 60 | files = set(filenames) |
| 61 | normal_form = None |
Tim Peters | 1ee401f | 2002-10-05 17:54:56 +0000 | [diff] [blame] | 62 | |
Mark Hammond | 7995eb2 | 2002-10-03 23:14:10 +0000 | [diff] [blame] | 63 | def setUp(self): |
Mark Hammond | 7995eb2 | 2002-10-03 23:14:10 +0000 | [diff] [blame] | 64 | try: |
Benjamin Peterson | ee8712c | 2008-05-20 21:35:26 +0000 | [diff] [blame] | 65 | os.mkdir(support.TESTFN) |
Florent Xicluna | 65eb453 | 2011-11-03 23:11:14 +0100 | [diff] [blame] | 66 | except FileExistsError: |
Mark Hammond | 7995eb2 | 2002-10-03 23:14:10 +0000 | [diff] [blame] | 67 | pass |
Victor Stinner | 0970657 | 2014-07-29 00:40:50 +0200 | [diff] [blame] | 68 | self.addCleanup(support.rmtree, support.TESTFN) |
| 69 | |
Florent Xicluna | eb13618 | 2010-03-21 18:49:50 +0000 | [diff] [blame] | 70 | files = set() |
Mark Hammond | 7995eb2 | 2002-10-03 23:14:10 +0000 | [diff] [blame] | 71 | for name in self.files: |
Florent Xicluna | eb13618 | 2010-03-21 18:49:50 +0000 | [diff] [blame] | 72 | name = os.path.join(support.TESTFN, self.norm(name)) |
Florent Xicluna | 87082ee | 2010-08-09 17:18:05 +0000 | [diff] [blame] | 73 | with open(name, 'wb') as f: |
| 74 | f.write((name+'\n').encode("utf-8")) |
Mark Hammond | 7995eb2 | 2002-10-03 23:14:10 +0000 | [diff] [blame] | 75 | os.stat(name) |
Florent Xicluna | eb13618 | 2010-03-21 18:49:50 +0000 | [diff] [blame] | 76 | files.add(name) |
| 77 | self.files = files |
Mark Hammond | 7995eb2 | 2002-10-03 23:14:10 +0000 | [diff] [blame] | 78 | |
Florent Xicluna | eb13618 | 2010-03-21 18:49:50 +0000 | [diff] [blame] | 79 | def norm(self, s): |
| 80 | if self.normal_form: |
| 81 | return normalize(self.normal_form, s) |
| 82 | return s |
| 83 | |
Florent Xicluna | 65eb453 | 2011-11-03 23:11:14 +0100 | [diff] [blame] | 84 | def _apply_failure(self, fn, filename, |
| 85 | expected_exception=FileNotFoundError, |
| 86 | check_filename=True): |
Florent Xicluna | eb13618 | 2010-03-21 18:49:50 +0000 | [diff] [blame] | 87 | with self.assertRaises(expected_exception) as c: |
Mark Hammond | 7995eb2 | 2002-10-03 23:14:10 +0000 | [diff] [blame] | 88 | fn(filename) |
Florent Xicluna | eb13618 | 2010-03-21 18:49:50 +0000 | [diff] [blame] | 89 | exc_filename = c.exception.filename |
Florent Xicluna | 65eb453 | 2011-11-03 23:11:14 +0100 | [diff] [blame] | 90 | if check_filename: |
Victor Stinner | 74ad754 | 2010-10-28 11:09:09 +0000 | [diff] [blame] | 91 | self.assertEqual(exc_filename, filename, "Function '%s(%a) failed " |
Victor Stinner | 7dae81b | 2010-10-28 11:11:24 +0000 | [diff] [blame] | 92 | "with bad filename in the exception: %a" % |
Florent Xicluna | eb13618 | 2010-03-21 18:49:50 +0000 | [diff] [blame] | 93 | (fn.__name__, filename, exc_filename)) |
Mark Hammond | 7995eb2 | 2002-10-03 23:14:10 +0000 | [diff] [blame] | 94 | |
| 95 | def test_failures(self): |
| 96 | # Pass non-existing Unicode filenames all over the place. |
| 97 | for name in self.files: |
| 98 | name = "not_" + name |
Florent Xicluna | 65eb453 | 2011-11-03 23:11:14 +0100 | [diff] [blame] | 99 | self._apply_failure(open, name) |
| 100 | self._apply_failure(os.stat, name) |
| 101 | self._apply_failure(os.chdir, name) |
| 102 | self._apply_failure(os.rmdir, name) |
| 103 | self._apply_failure(os.remove, name) |
Florent Xicluna | e0912db | 2011-11-07 21:56:17 +0100 | [diff] [blame] | 104 | self._apply_failure(os.listdir, name) |
| 105 | |
| 106 | if sys.platform == 'win32': |
Florent Xicluna | be90840 | 2011-11-07 23:24:08 +0100 | [diff] [blame] | 107 | # Windows is lunatic. Issue #13366. |
| 108 | _listdir_failure = NotADirectoryError, FileNotFoundError |
Florent Xicluna | e0912db | 2011-11-07 21:56:17 +0100 | [diff] [blame] | 109 | else: |
| 110 | _listdir_failure = NotADirectoryError |
Mark Hammond | 7995eb2 | 2002-10-03 23:14:10 +0000 | [diff] [blame] | 111 | |
| 112 | def test_open(self): |
| 113 | for name in self.files: |
Guido van Rossum | c12a813 | 2007-10-26 04:29:23 +0000 | [diff] [blame] | 114 | f = open(name, 'wb') |
Mark Hammond | 7995eb2 | 2002-10-03 23:14:10 +0000 | [diff] [blame] | 115 | f.write((name+'\n').encode("utf-8")) |
| 116 | f.close() |
| 117 | os.stat(name) |
Florent Xicluna | e0912db | 2011-11-07 21:56:17 +0100 | [diff] [blame] | 118 | self._apply_failure(os.listdir, name, self._listdir_failure) |
Mark Hammond | 7995eb2 | 2002-10-03 23:14:10 +0000 | [diff] [blame] | 119 | |
Victor Stinner | bfd7b26 | 2010-10-28 23:14:45 +0000 | [diff] [blame] | 120 | # Skip the test on darwin, because darwin does normalize the filename to |
| 121 | # NFD (a variant of Unicode NFD form). Normalize the filename to NFC, NFKC, |
| 122 | # NFKD in Python is useless, because darwin will normalize it later and so |
| 123 | # open(), os.stat(), etc. don't raise any exception. |
Florent Xicluna | 65eb453 | 2011-11-03 23:11:14 +0100 | [diff] [blame] | 124 | @unittest.skipIf(sys.platform == 'darwin', 'irrelevant test on Mac OS X') |
Florent Xicluna | eb13618 | 2010-03-21 18:49:50 +0000 | [diff] [blame] | 125 | def test_normalize(self): |
| 126 | files = set(self.files) |
| 127 | others = set() |
| 128 | for nf in set(['NFC', 'NFD', 'NFKC', 'NFKD']): |
| 129 | others |= set(normalize(nf, file) for file in files) |
| 130 | others -= files |
Florent Xicluna | eb13618 | 2010-03-21 18:49:50 +0000 | [diff] [blame] | 131 | for name in others: |
Florent Xicluna | 65eb453 | 2011-11-03 23:11:14 +0100 | [diff] [blame] | 132 | self._apply_failure(open, name) |
| 133 | self._apply_failure(os.stat, name) |
| 134 | self._apply_failure(os.chdir, name) |
| 135 | self._apply_failure(os.rmdir, name) |
| 136 | self._apply_failure(os.remove, name) |
Florent Xicluna | e0912db | 2011-11-07 21:56:17 +0100 | [diff] [blame] | 137 | self._apply_failure(os.listdir, name) |
Florent Xicluna | eb13618 | 2010-03-21 18:49:50 +0000 | [diff] [blame] | 138 | |
Victor Stinner | bfd7b26 | 2010-10-28 23:14:45 +0000 | [diff] [blame] | 139 | # Skip the test on darwin, because darwin uses a normalization different |
| 140 | # than Python NFD normalization: filenames are different even if we use |
| 141 | # Python NFD normalization. |
Florent Xicluna | 65eb453 | 2011-11-03 23:11:14 +0100 | [diff] [blame] | 142 | @unittest.skipIf(sys.platform == 'darwin', 'irrelevant test on Mac OS X') |
Mark Hammond | 7995eb2 | 2002-10-03 23:14:10 +0000 | [diff] [blame] | 143 | def test_listdir(self): |
Florent Xicluna | eb13618 | 2010-03-21 18:49:50 +0000 | [diff] [blame] | 144 | sf0 = set(self.files) |
Victor Stinner | 1ab6c2d | 2011-11-15 22:27:41 +0100 | [diff] [blame] | 145 | with warnings.catch_warnings(): |
| 146 | warnings.simplefilter("ignore", DeprecationWarning) |
| 147 | f1 = os.listdir(support.TESTFN.encode(sys.getfilesystemencoding())) |
Florent Xicluna | eb13618 | 2010-03-21 18:49:50 +0000 | [diff] [blame] | 148 | f2 = os.listdir(support.TESTFN) |
Florent Xicluna | eb13618 | 2010-03-21 18:49:50 +0000 | [diff] [blame] | 149 | sf2 = set(os.path.join(support.TESTFN, f) for f in f2) |
Victor Stinner | 2ebe697 | 2010-10-24 21:05:03 +0000 | [diff] [blame] | 150 | self.assertEqual(sf0, sf2, "%a != %a" % (sf0, sf2)) |
Florent Xicluna | eb13618 | 2010-03-21 18:49:50 +0000 | [diff] [blame] | 151 | self.assertEqual(len(f1), len(f2)) |
Mark Hammond | 7995eb2 | 2002-10-03 23:14:10 +0000 | [diff] [blame] | 152 | |
| 153 | def test_rename(self): |
| 154 | for name in self.files: |
Florent Xicluna | eb13618 | 2010-03-21 18:49:50 +0000 | [diff] [blame] | 155 | os.rename(name, "tmp") |
| 156 | os.rename("tmp", name) |
Mark Hammond | 7995eb2 | 2002-10-03 23:14:10 +0000 | [diff] [blame] | 157 | |
| 158 | def test_directory(self): |
Florent Xicluna | eb13618 | 2010-03-21 18:49:50 +0000 | [diff] [blame] | 159 | dirname = os.path.join(support.TESTFN, 'Gr\xfc\xdf-\u66e8\u66e9\u66eb') |
Guido van Rossum | ef87d6e | 2007-05-02 19:09:54 +0000 | [diff] [blame] | 160 | filename = '\xdf-\u66e8\u66e9\u66eb' |
Serhiy Storchaka | 2a23adf | 2015-09-06 14:13:25 +0300 | [diff] [blame] | 161 | with support.temp_cwd(dirname): |
Florent Xicluna | eb13618 | 2010-03-21 18:49:50 +0000 | [diff] [blame] | 162 | with open(filename, 'wb') as f: |
| 163 | f.write((filename + '\n').encode("utf-8")) |
| 164 | os.access(filename,os.R_OK) |
| 165 | os.remove(filename) |
Florent Xicluna | eb13618 | 2010-03-21 18:49:50 +0000 | [diff] [blame] | 166 | |
| 167 | |
| 168 | class UnicodeNFCFileTests(UnicodeFileTests): |
| 169 | normal_form = 'NFC' |
| 170 | |
| 171 | |
| 172 | class UnicodeNFDFileTests(UnicodeFileTests): |
| 173 | normal_form = 'NFD' |
| 174 | |
| 175 | |
| 176 | class UnicodeNFKCFileTests(UnicodeFileTests): |
| 177 | normal_form = 'NFKC' |
| 178 | |
| 179 | |
| 180 | class UnicodeNFKDFileTests(UnicodeFileTests): |
| 181 | normal_form = 'NFKD' |
| 182 | |
Mark Hammond | 7995eb2 | 2002-10-03 23:14:10 +0000 | [diff] [blame] | 183 | |
| 184 | def test_main(): |
Victor Stinner | 0970657 | 2014-07-29 00:40:50 +0200 | [diff] [blame] | 185 | support.run_unittest( |
| 186 | UnicodeFileTests, |
| 187 | UnicodeNFCFileTests, |
| 188 | UnicodeNFDFileTests, |
| 189 | UnicodeNFKCFileTests, |
| 190 | UnicodeNFKDFileTests, |
| 191 | ) |
Mark Hammond | 7995eb2 | 2002-10-03 23:14:10 +0000 | [diff] [blame] | 192 | |
Florent Xicluna | eb13618 | 2010-03-21 18:49:50 +0000 | [diff] [blame] | 193 | |
Mark Hammond | 7995eb2 | 2002-10-03 23:14:10 +0000 | [diff] [blame] | 194 | if __name__ == "__main__": |
| 195 | test_main() |