Benjamin Peterson | d75fcb4 | 2009-02-19 04:22:03 +0000 | [diff] [blame] | 1 | # -*- coding: koi8-r -*- |
Neal Norwitz | 40d3781 | 2005-10-02 01:48:49 +0000 | [diff] [blame] | 2 | |
Benjamin Peterson | 130786f | 2009-02-14 17:00:16 +0000 | [diff] [blame] | 3 | import unittest |
Victor Stinner | 047b7ae | 2014-10-05 17:37:41 +0200 | [diff] [blame] | 4 | from test.support import TESTFN, unlink, unload, rmtree |
Serhiy Storchaka | 9702a17 | 2013-06-13 10:08:00 +0300 | [diff] [blame] | 5 | import importlib |
| 6 | import os |
| 7 | import sys |
Martin v. Löwis | 78f1e4c | 2014-02-28 15:43:36 +0100 | [diff] [blame] | 8 | import subprocess |
Benjamin Peterson | 130786f | 2009-02-14 17:00:16 +0000 | [diff] [blame] | 9 | |
Serhiy Storchaka | 9702a17 | 2013-06-13 10:08:00 +0300 | [diff] [blame] | 10 | class SourceEncodingTest(unittest.TestCase): |
Benjamin Peterson | 130786f | 2009-02-14 17:00:16 +0000 | [diff] [blame] | 11 | |
| 12 | def test_pep263(self): |
| 13 | self.assertEqual( |
| 14 | "ðÉÔÏÎ".encode("utf-8"), |
| 15 | b'\xd0\x9f\xd0\xb8\xd1\x82\xd0\xbe\xd0\xbd' |
| 16 | ) |
| 17 | self.assertEqual( |
| 18 | "\ð".encode("utf-8"), |
| 19 | b'\\\xd0\x9f' |
| 20 | ) |
| 21 | |
| 22 | def test_compilestring(self): |
| 23 | # see #1882 |
| 24 | c = compile(b"\n# coding: utf-8\nu = '\xc3\xb3'\n", "dummy", "exec") |
| 25 | d = {} |
| 26 | exec(c, d) |
| 27 | self.assertEqual(d['u'], '\xf3') |
| 28 | |
| 29 | def test_issue2301(self): |
| 30 | try: |
| 31 | compile(b"# coding: cp932\nprint '\x94\x4e'", "dummy", "exec") |
| 32 | except SyntaxError as v: |
Ezio Melotti | b3aedd4 | 2010-11-20 19:04:17 +0000 | [diff] [blame] | 33 | self.assertEqual(v.text, "print '\u5e74'\n") |
Benjamin Peterson | 130786f | 2009-02-14 17:00:16 +0000 | [diff] [blame] | 34 | else: |
| 35 | self.fail() |
| 36 | |
Benjamin Peterson | f5b5224 | 2009-03-02 23:31:26 +0000 | [diff] [blame] | 37 | def test_issue4626(self): |
| 38 | c = compile("# coding=latin-1\n\u00c6 = '\u00c6'", "dummy", "exec") |
| 39 | d = {} |
| 40 | exec(c, d) |
Ezio Melotti | b3aedd4 | 2010-11-20 19:04:17 +0000 | [diff] [blame] | 41 | self.assertEqual(d['\xc6'], '\xc6') |
Benjamin Peterson | f5b5224 | 2009-03-02 23:31:26 +0000 | [diff] [blame] | 42 | |
Benjamin Peterson | b2e796a | 2009-10-28 21:59:39 +0000 | [diff] [blame] | 43 | def test_issue3297(self): |
| 44 | c = compile("a, b = '\U0001010F', '\\U0001010F'", "dummy", "exec") |
| 45 | d = {} |
| 46 | exec(c, d) |
| 47 | self.assertEqual(d['a'], d['b']) |
| 48 | self.assertEqual(len(d['a']), len(d['b'])) |
| 49 | self.assertEqual(ascii(d['a']), ascii(d['b'])) |
| 50 | |
Victor Stinner | 6aa278e | 2010-03-03 00:18:49 +0000 | [diff] [blame] | 51 | def test_issue7820(self): |
| 52 | # Ensure that check_bom() restores all bytes in the right order if |
| 53 | # check_bom() fails in pydebug mode: a buffer starts with the first |
| 54 | # byte of a valid BOM, but next bytes are different |
| 55 | |
| 56 | # one byte in common with the UTF-16-LE BOM |
| 57 | self.assertRaises(SyntaxError, eval, b'\xff\x20') |
| 58 | |
| 59 | # two bytes in common with the UTF-8 BOM |
| 60 | self.assertRaises(SyntaxError, eval, b'\xef\xbb\x20') |
| 61 | |
Martin v. Löwis | 78f1e4c | 2014-02-28 15:43:36 +0100 | [diff] [blame] | 62 | def test_20731(self): |
Martin v. Löwis | 80875fe | 2014-02-28 15:49:06 +0100 | [diff] [blame] | 63 | sub = subprocess.Popen([sys.executable, |
Martin v. Löwis | 78f1e4c | 2014-02-28 15:43:36 +0100 | [diff] [blame] | 64 | os.path.join(os.path.dirname(__file__), |
| 65 | 'coding20731.py')], |
| 66 | stderr=subprocess.PIPE) |
| 67 | err = sub.communicate()[1] |
Benjamin Peterson | 1260418 | 2014-03-01 10:34:22 -0500 | [diff] [blame] | 68 | self.assertEqual(sub.returncode, 0) |
| 69 | self.assertNotIn(b'SyntaxError', err) |
Martin v. Löwis | 78f1e4c | 2014-02-28 15:43:36 +0100 | [diff] [blame] | 70 | |
Serhiy Storchaka | 3af14aa | 2013-06-09 16:51:52 +0300 | [diff] [blame] | 71 | def test_error_message(self): |
| 72 | compile(b'# -*- coding: iso-8859-15 -*-\n', 'dummy', 'exec') |
| 73 | compile(b'\xef\xbb\xbf\n', 'dummy', 'exec') |
| 74 | compile(b'\xef\xbb\xbf# -*- coding: utf-8 -*-\n', 'dummy', 'exec') |
Brett Cannon | 39295e7 | 2013-06-16 11:37:57 -0400 | [diff] [blame] | 75 | with self.assertRaisesRegex(SyntaxError, 'fake'): |
Serhiy Storchaka | 3af14aa | 2013-06-09 16:51:52 +0300 | [diff] [blame] | 76 | compile(b'# -*- coding: fake -*-\n', 'dummy', 'exec') |
Brett Cannon | 39295e7 | 2013-06-16 11:37:57 -0400 | [diff] [blame] | 77 | with self.assertRaisesRegex(SyntaxError, 'iso-8859-15'): |
Serhiy Storchaka | 3af14aa | 2013-06-09 16:51:52 +0300 | [diff] [blame] | 78 | compile(b'\xef\xbb\xbf# -*- coding: iso-8859-15 -*-\n', |
| 79 | 'dummy', 'exec') |
Brett Cannon | 39295e7 | 2013-06-16 11:37:57 -0400 | [diff] [blame] | 80 | with self.assertRaisesRegex(SyntaxError, 'BOM'): |
Serhiy Storchaka | 3af14aa | 2013-06-09 16:51:52 +0300 | [diff] [blame] | 81 | compile(b'\xef\xbb\xbf# -*- coding: iso-8859-15 -*-\n', |
| 82 | 'dummy', 'exec') |
Brett Cannon | 39295e7 | 2013-06-16 11:37:57 -0400 | [diff] [blame] | 83 | with self.assertRaisesRegex(SyntaxError, 'fake'): |
Serhiy Storchaka | 3af14aa | 2013-06-09 16:51:52 +0300 | [diff] [blame] | 84 | compile(b'\xef\xbb\xbf# -*- coding: fake -*-\n', 'dummy', 'exec') |
Brett Cannon | 39295e7 | 2013-06-16 11:37:57 -0400 | [diff] [blame] | 85 | with self.assertRaisesRegex(SyntaxError, 'BOM'): |
Serhiy Storchaka | 3af14aa | 2013-06-09 16:51:52 +0300 | [diff] [blame] | 86 | compile(b'\xef\xbb\xbf# -*- coding: fake -*-\n', 'dummy', 'exec') |
| 87 | |
Neal Norwitz | 40d3781 | 2005-10-02 01:48:49 +0000 | [diff] [blame] | 88 | def test_bad_coding(self): |
| 89 | module_name = 'bad_coding' |
Neal Norwitz | db83eb3 | 2005-12-18 05:29:30 +0000 | [diff] [blame] | 90 | self.verify_bad_module(module_name) |
| 91 | |
| 92 | def test_bad_coding2(self): |
| 93 | module_name = 'bad_coding2' |
| 94 | self.verify_bad_module(module_name) |
| 95 | |
| 96 | def verify_bad_module(self, module_name): |
Neal Norwitz | 40d3781 | 2005-10-02 01:48:49 +0000 | [diff] [blame] | 97 | self.assertRaises(SyntaxError, __import__, 'test.' + module_name) |
| 98 | |
| 99 | path = os.path.dirname(__file__) |
| 100 | filename = os.path.join(path, module_name + '.py') |
Florent Xicluna | 8fbddf1 | 2010-03-17 20:29:51 +0000 | [diff] [blame] | 101 | with open(filename, "rb") as fp: |
| 102 | bytes = fp.read() |
Benjamin Peterson | f5b5224 | 2009-03-02 23:31:26 +0000 | [diff] [blame] | 103 | self.assertRaises(SyntaxError, compile, bytes, filename, 'exec') |
Neal Norwitz | 40d3781 | 2005-10-02 01:48:49 +0000 | [diff] [blame] | 104 | |
Neal Norwitz | f7f28fc | 2007-08-11 21:31:25 +0000 | [diff] [blame] | 105 | def test_exec_valid_coding(self): |
| 106 | d = {} |
Serhiy Storchaka | 10dc0eb | 2013-06-19 10:29:58 +0300 | [diff] [blame] | 107 | exec(b'# coding: cp949\na = "\xaa\xa7"\n', d) |
| 108 | self.assertEqual(d['a'], '\u3047') |
Neal Norwitz | f7f28fc | 2007-08-11 21:31:25 +0000 | [diff] [blame] | 109 | |
Amaury Forgeot d'Arc | 65f9ace | 2007-11-15 23:19:43 +0000 | [diff] [blame] | 110 | def test_file_parse(self): |
| 111 | # issue1134: all encodings outside latin-1 and utf-8 fail on |
| 112 | # multiline strings and long lines (>512 columns) |
Florent Xicluna | 8fbddf1 | 2010-03-17 20:29:51 +0000 | [diff] [blame] | 113 | unload(TESTFN) |
Amaury Forgeot d'Arc | dd9e3b8 | 2007-11-16 00:56:23 +0000 | [diff] [blame] | 114 | filename = TESTFN + ".py" |
Serhiy Storchaka | 10dc0eb | 2013-06-19 10:29:58 +0300 | [diff] [blame] | 115 | f = open(filename, "w", encoding="cp1252") |
| 116 | sys.path.insert(0, os.curdir) |
Amaury Forgeot d'Arc | 65f9ace | 2007-11-15 23:19:43 +0000 | [diff] [blame] | 117 | try: |
Serhiy Storchaka | 10dc0eb | 2013-06-19 10:29:58 +0300 | [diff] [blame] | 118 | with f: |
| 119 | f.write("# -*- coding: cp1252 -*-\n") |
| 120 | f.write("'''A short string\n") |
| 121 | f.write("'''\n") |
| 122 | f.write("'A very long string %s'\n" % ("X" * 1000)) |
Amaury Forgeot d'Arc | 65f9ace | 2007-11-15 23:19:43 +0000 | [diff] [blame] | 123 | |
Brett Cannon | afbdc13 | 2012-04-14 15:06:17 -0400 | [diff] [blame] | 124 | importlib.invalidate_caches() |
Amaury Forgeot d'Arc | dd9e3b8 | 2007-11-16 00:56:23 +0000 | [diff] [blame] | 125 | __import__(TESTFN) |
Amaury Forgeot d'Arc | 65f9ace | 2007-11-15 23:19:43 +0000 | [diff] [blame] | 126 | finally: |
Serhiy Storchaka | 10dc0eb | 2013-06-19 10:29:58 +0300 | [diff] [blame] | 127 | del sys.path[0] |
Florent Xicluna | 8fbddf1 | 2010-03-17 20:29:51 +0000 | [diff] [blame] | 128 | unlink(filename) |
| 129 | unlink(filename + "c") |
Serhiy Storchaka | 10dc0eb | 2013-06-19 10:29:58 +0300 | [diff] [blame] | 130 | unlink(filename + "o") |
Florent Xicluna | 8fbddf1 | 2010-03-17 20:29:51 +0000 | [diff] [blame] | 131 | unload(TESTFN) |
Victor Stinner | 047b7ae | 2014-10-05 17:37:41 +0200 | [diff] [blame] | 132 | rmtree('__pycache__') |
Amaury Forgeot d'Arc | 65f9ace | 2007-11-15 23:19:43 +0000 | [diff] [blame] | 133 | |
Benjamin Peterson | 0289b15 | 2009-06-28 17:22:03 +0000 | [diff] [blame] | 134 | def test_error_from_string(self): |
| 135 | # See http://bugs.python.org/issue6289 |
| 136 | input = "# coding: ascii\n\N{SNOWMAN}".encode('utf-8') |
Florent Xicluna | 8fbddf1 | 2010-03-17 20:29:51 +0000 | [diff] [blame] | 137 | with self.assertRaises(SyntaxError) as c: |
Benjamin Peterson | 0289b15 | 2009-06-28 17:22:03 +0000 | [diff] [blame] | 138 | compile(input, "<string>", "exec") |
Florent Xicluna | 8fbddf1 | 2010-03-17 20:29:51 +0000 | [diff] [blame] | 139 | expected = "'ascii' codec can't decode byte 0xe2 in position 16: " \ |
| 140 | "ordinal not in range(128)" |
Serhiy Storchaka | 10dc0eb | 2013-06-19 10:29:58 +0300 | [diff] [blame] | 141 | self.assertTrue(c.exception.args[0].startswith(expected), |
| 142 | msg=c.exception.args[0]) |
Florent Xicluna | 8fbddf1 | 2010-03-17 20:29:51 +0000 | [diff] [blame] | 143 | |
Benjamin Peterson | 0289b15 | 2009-06-28 17:22:03 +0000 | [diff] [blame] | 144 | |
Neal Norwitz | 40d3781 | 2005-10-02 01:48:49 +0000 | [diff] [blame] | 145 | if __name__ == "__main__": |
Serhiy Storchaka | 9702a17 | 2013-06-13 10:08:00 +0300 | [diff] [blame] | 146 | unittest.main() |