Guido van Rossum | 29fd712 | 2007-11-12 01:13:56 +0000 | [diff] [blame] | 1 | r"""Test correct treatment of various string literals by the parser. |
| 2 | |
| 3 | There are four types of string literals: |
| 4 | |
Antoine Pitrou | 3a5d4cb | 2012-01-12 22:46:19 +0100 | [diff] [blame] | 5 | 'abc' -- normal str |
| 6 | r'abc' -- raw str |
| 7 | b'xyz' -- normal bytes |
| 8 | br'xyz' | rb'xyz' -- raw bytes |
Guido van Rossum | 29fd712 | 2007-11-12 01:13:56 +0000 | [diff] [blame] | 9 | |
| 10 | The difference between normal and raw strings is of course that in a |
| 11 | raw string, \ escapes (while still used to determine the end of the |
| 12 | literal) are not interpreted, so that r'\x00' contains four |
| 13 | characters: a backslash, an x, and two zeros; while '\x00' contains a |
| 14 | single character (code point zero). |
| 15 | |
| 16 | The tricky thing is what should happen when non-ASCII bytes are used |
| 17 | inside literals. For bytes literals, this is considered illegal. But |
| 18 | for str literals, those bytes are supposed to be decoded using the |
| 19 | encoding declared for the file (UTF-8 by default). |
| 20 | |
| 21 | We have to test this with various file encodings. We also test it with |
| 22 | exec()/eval(), which uses a different code path. |
| 23 | |
| 24 | This file is really about correct treatment of encodings and |
Ezio Melotti | 1392500 | 2011-03-16 11:05:33 +0200 | [diff] [blame] | 25 | backslashes. It doesn't concern itself with issues like single |
Guido van Rossum | 29fd712 | 2007-11-12 01:13:56 +0000 | [diff] [blame] | 26 | vs. double quotes or singly- vs. triply-quoted strings: that's dealt |
| 27 | with elsewhere (I assume). |
| 28 | """ |
| 29 | |
| 30 | import os |
| 31 | import sys |
| 32 | import shutil |
| 33 | import tempfile |
Eric V. Smith | 5646648 | 2016-10-31 14:46:26 -0400 | [diff] [blame] | 34 | import warnings |
Guido van Rossum | 29fd712 | 2007-11-12 01:13:56 +0000 | [diff] [blame] | 35 | import unittest |
| 36 | |
| 37 | |
| 38 | TEMPLATE = r"""# coding: %s |
| 39 | a = 'x' |
| 40 | assert ord(a) == 120 |
| 41 | b = '\x01' |
| 42 | assert ord(b) == 1 |
| 43 | c = r'\x01' |
| 44 | assert list(map(ord, c)) == [92, 120, 48, 49] |
| 45 | d = '\x81' |
| 46 | assert ord(d) == 0x81 |
| 47 | e = r'\x81' |
| 48 | assert list(map(ord, e)) == [92, 120, 56, 49] |
| 49 | f = '\u1881' |
| 50 | assert ord(f) == 0x1881 |
| 51 | g = r'\u1881' |
| 52 | assert list(map(ord, g)) == [92, 117, 49, 56, 56, 49] |
Serhiy Storchaka | 5e61f14 | 2013-02-10 17:36:00 +0200 | [diff] [blame] | 53 | h = '\U0001d120' |
| 54 | assert ord(h) == 0x1d120 |
| 55 | i = r'\U0001d120' |
| 56 | assert list(map(ord, i)) == [92, 85, 48, 48, 48, 49, 100, 49, 50, 48] |
Guido van Rossum | 29fd712 | 2007-11-12 01:13:56 +0000 | [diff] [blame] | 57 | """ |
| 58 | |
| 59 | |
| 60 | def byte(i): |
| 61 | return bytes([i]) |
| 62 | |
| 63 | |
| 64 | class TestLiterals(unittest.TestCase): |
| 65 | |
| 66 | def setUp(self): |
| 67 | self.save_path = sys.path[:] |
| 68 | self.tmpdir = tempfile.mkdtemp() |
| 69 | sys.path.insert(0, self.tmpdir) |
| 70 | |
| 71 | def tearDown(self): |
Georg Brandl | 242631d | 2012-02-20 21:36:28 +0100 | [diff] [blame] | 72 | sys.path[:] = self.save_path |
Guido van Rossum | 29fd712 | 2007-11-12 01:13:56 +0000 | [diff] [blame] | 73 | shutil.rmtree(self.tmpdir, ignore_errors=True) |
| 74 | |
| 75 | def test_template(self): |
| 76 | # Check that the template doesn't contain any non-printables |
| 77 | # except for \n. |
| 78 | for c in TEMPLATE: |
| 79 | assert c == '\n' or ' ' <= c <= '~', repr(c) |
| 80 | |
| 81 | def test_eval_str_normal(self): |
| 82 | self.assertEqual(eval(""" 'x' """), 'x') |
| 83 | self.assertEqual(eval(r""" '\x01' """), chr(1)) |
| 84 | self.assertEqual(eval(""" '\x01' """), chr(1)) |
| 85 | self.assertEqual(eval(r""" '\x81' """), chr(0x81)) |
| 86 | self.assertEqual(eval(""" '\x81' """), chr(0x81)) |
| 87 | self.assertEqual(eval(r""" '\u1881' """), chr(0x1881)) |
| 88 | self.assertEqual(eval(""" '\u1881' """), chr(0x1881)) |
Serhiy Storchaka | 5e61f14 | 2013-02-10 17:36:00 +0200 | [diff] [blame] | 89 | self.assertEqual(eval(r""" '\U0001d120' """), chr(0x1d120)) |
| 90 | self.assertEqual(eval(""" '\U0001d120' """), chr(0x1d120)) |
| 91 | |
| 92 | def test_eval_str_incomplete(self): |
| 93 | self.assertRaises(SyntaxError, eval, r""" '\x' """) |
| 94 | self.assertRaises(SyntaxError, eval, r""" '\x0' """) |
| 95 | self.assertRaises(SyntaxError, eval, r""" '\u' """) |
| 96 | self.assertRaises(SyntaxError, eval, r""" '\u0' """) |
| 97 | self.assertRaises(SyntaxError, eval, r""" '\u00' """) |
| 98 | self.assertRaises(SyntaxError, eval, r""" '\u000' """) |
| 99 | self.assertRaises(SyntaxError, eval, r""" '\U' """) |
| 100 | self.assertRaises(SyntaxError, eval, r""" '\U0' """) |
| 101 | self.assertRaises(SyntaxError, eval, r""" '\U00' """) |
| 102 | self.assertRaises(SyntaxError, eval, r""" '\U000' """) |
| 103 | self.assertRaises(SyntaxError, eval, r""" '\U0000' """) |
| 104 | self.assertRaises(SyntaxError, eval, r""" '\U00000' """) |
| 105 | self.assertRaises(SyntaxError, eval, r""" '\U000000' """) |
| 106 | self.assertRaises(SyntaxError, eval, r""" '\U0000000' """) |
Guido van Rossum | 29fd712 | 2007-11-12 01:13:56 +0000 | [diff] [blame] | 107 | |
Eric V. Smith | 5646648 | 2016-10-31 14:46:26 -0400 | [diff] [blame] | 108 | def test_eval_str_invalid_escape(self): |
| 109 | for b in range(1, 128): |
| 110 | if b in b"""\n\r"'01234567NU\\abfnrtuvx""": |
| 111 | continue |
| 112 | with self.assertWarns(DeprecationWarning): |
| 113 | self.assertEqual(eval(r"'\%c'" % b), '\\' + chr(b)) |
Victor Stinner | f9cca36 | 2016-11-15 09:12:10 +0100 | [diff] [blame^] | 114 | |
Eric V. Smith | 5646648 | 2016-10-31 14:46:26 -0400 | [diff] [blame] | 115 | with warnings.catch_warnings(record=True) as w: |
| 116 | warnings.simplefilter('always', category=DeprecationWarning) |
| 117 | eval("'''\n\\z'''") |
| 118 | self.assertEqual(len(w), 1) |
| 119 | self.assertEqual(w[0].filename, '<string>') |
| 120 | self.assertEqual(w[0].lineno, 2) |
| 121 | |
Victor Stinner | f9cca36 | 2016-11-15 09:12:10 +0100 | [diff] [blame^] | 122 | with warnings.catch_warnings(record=True) as w: |
| 123 | warnings.simplefilter('error', category=DeprecationWarning) |
| 124 | with self.assertRaises(SyntaxError) as cm: |
| 125 | eval("'''\n\\z'''") |
| 126 | exc = cm.exception |
| 127 | self.assertEqual(w, []) |
| 128 | self.assertEqual(exc.filename, '<string>') |
| 129 | self.assertEqual(exc.lineno, 2) |
| 130 | |
Guido van Rossum | 29fd712 | 2007-11-12 01:13:56 +0000 | [diff] [blame] | 131 | def test_eval_str_raw(self): |
| 132 | self.assertEqual(eval(""" r'x' """), 'x') |
| 133 | self.assertEqual(eval(r""" r'\x01' """), '\\' + 'x01') |
| 134 | self.assertEqual(eval(""" r'\x01' """), chr(1)) |
| 135 | self.assertEqual(eval(r""" r'\x81' """), '\\' + 'x81') |
| 136 | self.assertEqual(eval(""" r'\x81' """), chr(0x81)) |
| 137 | self.assertEqual(eval(r""" r'\u1881' """), '\\' + 'u1881') |
| 138 | self.assertEqual(eval(""" r'\u1881' """), chr(0x1881)) |
Serhiy Storchaka | 5e61f14 | 2013-02-10 17:36:00 +0200 | [diff] [blame] | 139 | self.assertEqual(eval(r""" r'\U0001d120' """), '\\' + 'U0001d120') |
| 140 | self.assertEqual(eval(""" r'\U0001d120' """), chr(0x1d120)) |
Guido van Rossum | 29fd712 | 2007-11-12 01:13:56 +0000 | [diff] [blame] | 141 | |
| 142 | def test_eval_bytes_normal(self): |
| 143 | self.assertEqual(eval(""" b'x' """), b'x') |
| 144 | self.assertEqual(eval(r""" b'\x01' """), byte(1)) |
| 145 | self.assertEqual(eval(""" b'\x01' """), byte(1)) |
| 146 | self.assertEqual(eval(r""" b'\x81' """), byte(0x81)) |
| 147 | self.assertRaises(SyntaxError, eval, """ b'\x81' """) |
R David Murray | 44b548d | 2016-09-08 13:59:53 -0400 | [diff] [blame] | 148 | self.assertEqual(eval(r""" br'\u1881' """), b'\\' + b'u1881') |
Guido van Rossum | 29fd712 | 2007-11-12 01:13:56 +0000 | [diff] [blame] | 149 | self.assertRaises(SyntaxError, eval, """ b'\u1881' """) |
R David Murray | 44b548d | 2016-09-08 13:59:53 -0400 | [diff] [blame] | 150 | self.assertEqual(eval(r""" br'\U0001d120' """), b'\\' + b'U0001d120') |
Serhiy Storchaka | 5e61f14 | 2013-02-10 17:36:00 +0200 | [diff] [blame] | 151 | self.assertRaises(SyntaxError, eval, """ b'\U0001d120' """) |
| 152 | |
| 153 | def test_eval_bytes_incomplete(self): |
| 154 | self.assertRaises(SyntaxError, eval, r""" b'\x' """) |
| 155 | self.assertRaises(SyntaxError, eval, r""" b'\x0' """) |
Guido van Rossum | 29fd712 | 2007-11-12 01:13:56 +0000 | [diff] [blame] | 156 | |
Eric V. Smith | 5646648 | 2016-10-31 14:46:26 -0400 | [diff] [blame] | 157 | def test_eval_bytes_invalid_escape(self): |
| 158 | for b in range(1, 128): |
| 159 | if b in b"""\n\r"'01234567\\abfnrtvx""": |
| 160 | continue |
| 161 | with self.assertWarns(DeprecationWarning): |
| 162 | self.assertEqual(eval(r"b'\%c'" % b), b'\\' + bytes([b])) |
Victor Stinner | f9cca36 | 2016-11-15 09:12:10 +0100 | [diff] [blame^] | 163 | |
Eric V. Smith | 5646648 | 2016-10-31 14:46:26 -0400 | [diff] [blame] | 164 | with warnings.catch_warnings(record=True) as w: |
| 165 | warnings.simplefilter('always', category=DeprecationWarning) |
| 166 | eval("b'''\n\\z'''") |
| 167 | self.assertEqual(len(w), 1) |
| 168 | self.assertEqual(w[0].filename, '<string>') |
| 169 | self.assertEqual(w[0].lineno, 2) |
| 170 | |
Victor Stinner | f9cca36 | 2016-11-15 09:12:10 +0100 | [diff] [blame^] | 171 | with warnings.catch_warnings(record=True) as w: |
| 172 | warnings.simplefilter('error', category=DeprecationWarning) |
| 173 | with self.assertRaises(SyntaxError) as cm: |
| 174 | eval("b'''\n\\z'''") |
| 175 | exc = cm.exception |
| 176 | self.assertEqual(w, []) |
| 177 | self.assertEqual(exc.filename, '<string>') |
| 178 | self.assertEqual(exc.lineno, 2) |
| 179 | |
Guido van Rossum | 29fd712 | 2007-11-12 01:13:56 +0000 | [diff] [blame] | 180 | def test_eval_bytes_raw(self): |
| 181 | self.assertEqual(eval(""" br'x' """), b'x') |
Antoine Pitrou | 3a5d4cb | 2012-01-12 22:46:19 +0100 | [diff] [blame] | 182 | self.assertEqual(eval(""" rb'x' """), b'x') |
Guido van Rossum | 29fd712 | 2007-11-12 01:13:56 +0000 | [diff] [blame] | 183 | self.assertEqual(eval(r""" br'\x01' """), b'\\' + b'x01') |
Antoine Pitrou | 3a5d4cb | 2012-01-12 22:46:19 +0100 | [diff] [blame] | 184 | self.assertEqual(eval(r""" rb'\x01' """), b'\\' + b'x01') |
Guido van Rossum | 29fd712 | 2007-11-12 01:13:56 +0000 | [diff] [blame] | 185 | self.assertEqual(eval(""" br'\x01' """), byte(1)) |
Antoine Pitrou | 3a5d4cb | 2012-01-12 22:46:19 +0100 | [diff] [blame] | 186 | self.assertEqual(eval(""" rb'\x01' """), byte(1)) |
Guido van Rossum | 29fd712 | 2007-11-12 01:13:56 +0000 | [diff] [blame] | 187 | self.assertEqual(eval(r""" br'\x81' """), b"\\" + b"x81") |
Antoine Pitrou | 3a5d4cb | 2012-01-12 22:46:19 +0100 | [diff] [blame] | 188 | self.assertEqual(eval(r""" rb'\x81' """), b"\\" + b"x81") |
Guido van Rossum | 29fd712 | 2007-11-12 01:13:56 +0000 | [diff] [blame] | 189 | self.assertRaises(SyntaxError, eval, """ br'\x81' """) |
Antoine Pitrou | 3a5d4cb | 2012-01-12 22:46:19 +0100 | [diff] [blame] | 190 | self.assertRaises(SyntaxError, eval, """ rb'\x81' """) |
Guido van Rossum | 29fd712 | 2007-11-12 01:13:56 +0000 | [diff] [blame] | 191 | self.assertEqual(eval(r""" br'\u1881' """), b"\\" + b"u1881") |
Antoine Pitrou | 3a5d4cb | 2012-01-12 22:46:19 +0100 | [diff] [blame] | 192 | self.assertEqual(eval(r""" rb'\u1881' """), b"\\" + b"u1881") |
Guido van Rossum | 29fd712 | 2007-11-12 01:13:56 +0000 | [diff] [blame] | 193 | self.assertRaises(SyntaxError, eval, """ br'\u1881' """) |
Antoine Pitrou | 3a5d4cb | 2012-01-12 22:46:19 +0100 | [diff] [blame] | 194 | self.assertRaises(SyntaxError, eval, """ rb'\u1881' """) |
Serhiy Storchaka | 5e61f14 | 2013-02-10 17:36:00 +0200 | [diff] [blame] | 195 | self.assertEqual(eval(r""" br'\U0001d120' """), b"\\" + b"U0001d120") |
Serhiy Storchaka | 801d955 | 2013-02-10 17:42:01 +0200 | [diff] [blame] | 196 | self.assertEqual(eval(r""" rb'\U0001d120' """), b"\\" + b"U0001d120") |
Serhiy Storchaka | 5e61f14 | 2013-02-10 17:36:00 +0200 | [diff] [blame] | 197 | self.assertRaises(SyntaxError, eval, """ br'\U0001d120' """) |
Serhiy Storchaka | 801d955 | 2013-02-10 17:42:01 +0200 | [diff] [blame] | 198 | self.assertRaises(SyntaxError, eval, """ rb'\U0001d120' """) |
Antoine Pitrou | 3a5d4cb | 2012-01-12 22:46:19 +0100 | [diff] [blame] | 199 | self.assertRaises(SyntaxError, eval, """ bb'' """) |
| 200 | self.assertRaises(SyntaxError, eval, """ rr'' """) |
| 201 | self.assertRaises(SyntaxError, eval, """ brr'' """) |
| 202 | self.assertRaises(SyntaxError, eval, """ bbr'' """) |
| 203 | self.assertRaises(SyntaxError, eval, """ rrb'' """) |
| 204 | self.assertRaises(SyntaxError, eval, """ rbb'' """) |
Guido van Rossum | 29fd712 | 2007-11-12 01:13:56 +0000 | [diff] [blame] | 205 | |
Christian Heimes | 0b3847d | 2012-06-20 11:17:58 +0200 | [diff] [blame] | 206 | def test_eval_str_u(self): |
| 207 | self.assertEqual(eval(""" u'x' """), 'x') |
| 208 | self.assertEqual(eval(""" U'\u00e4' """), 'ä') |
| 209 | self.assertEqual(eval(""" u'\N{LATIN SMALL LETTER A WITH DIAERESIS}' """), 'ä') |
| 210 | self.assertRaises(SyntaxError, eval, """ ur'' """) |
| 211 | self.assertRaises(SyntaxError, eval, """ ru'' """) |
| 212 | self.assertRaises(SyntaxError, eval, """ bu'' """) |
| 213 | self.assertRaises(SyntaxError, eval, """ ub'' """) |
| 214 | |
Guido van Rossum | 29fd712 | 2007-11-12 01:13:56 +0000 | [diff] [blame] | 215 | def check_encoding(self, encoding, extra=""): |
| 216 | modname = "xx_" + encoding.replace("-", "_") |
| 217 | fn = os.path.join(self.tmpdir, modname + ".py") |
| 218 | f = open(fn, "w", encoding=encoding) |
| 219 | try: |
| 220 | f.write(TEMPLATE % encoding) |
| 221 | f.write(extra) |
| 222 | finally: |
| 223 | f.close() |
| 224 | __import__(modname) |
| 225 | del sys.modules[modname] |
| 226 | |
| 227 | def test_file_utf_8(self): |
| 228 | extra = "z = '\u1234'; assert ord(z) == 0x1234\n" |
| 229 | self.check_encoding("utf-8", extra) |
| 230 | |
| 231 | def test_file_utf_8_error(self): |
| 232 | extra = "b'\x80'\n" |
| 233 | self.assertRaises(SyntaxError, self.check_encoding, "utf-8", extra) |
| 234 | |
| 235 | def test_file_utf8(self): |
Marc-André Lemburg | 8f36af7 | 2011-02-25 15:42:01 +0000 | [diff] [blame] | 236 | self.check_encoding("utf-8") |
Guido van Rossum | 29fd712 | 2007-11-12 01:13:56 +0000 | [diff] [blame] | 237 | |
| 238 | def test_file_iso_8859_1(self): |
| 239 | self.check_encoding("iso-8859-1") |
| 240 | |
| 241 | def test_file_latin_1(self): |
| 242 | self.check_encoding("latin-1") |
| 243 | |
| 244 | def test_file_latin9(self): |
| 245 | self.check_encoding("latin9") |
| 246 | |
| 247 | |
| 248 | if __name__ == "__main__": |
Zachary Ware | 38c707e | 2015-04-13 15:00:43 -0500 | [diff] [blame] | 249 | unittest.main() |