blob: a4ae19803c9dbf5e690dcaa466c282dcf4dc283b [file] [log] [blame]
Guido van Rossum29fd7122007-11-12 01:13:56 +00001r"""Test correct treatment of various string literals by the parser.
2
3There are four types of string literals:
4
5 'abc' -- normal str
6 r'abc' -- raw str
7 b'xyz' -- normal bytes
8 br'xyz' -- raw bytes
9
10The difference between normal and raw strings is of course that in a
11raw string, \ escapes (while still used to determine the end of the
12literal) are not interpreted, so that r'\x00' contains four
13characters: a backslash, an x, and two zeros; while '\x00' contains a
14single character (code point zero).
15
16The tricky thing is what should happen when non-ASCII bytes are used
17inside literals. For bytes literals, this is considered illegal. But
18for str literals, those bytes are supposed to be decoded using the
19encoding declared for the file (UTF-8 by default).
20
21We have to test this with various file encodings. We also test it with
22exec()/eval(), which uses a different code path.
23
24This file is really about correct treatment of encodings and
Ezio Melotti13925002011-03-16 11:05:33 +020025backslashes. It doesn't concern itself with issues like single
Guido van Rossum29fd7122007-11-12 01:13:56 +000026vs. double quotes or singly- vs. triply-quoted strings: that's dealt
27with elsewhere (I assume).
28"""
29
30import os
31import sys
32import shutil
33import tempfile
34import unittest
Antoine Pitroubbf53612012-01-12 22:36:48 +010035import test.support
Guido van Rossum29fd7122007-11-12 01:13:56 +000036
37
38TEMPLATE = r"""# coding: %s
39a = 'x'
40assert ord(a) == 120
41b = '\x01'
42assert ord(b) == 1
43c = r'\x01'
44assert list(map(ord, c)) == [92, 120, 48, 49]
45d = '\x81'
46assert ord(d) == 0x81
47e = r'\x81'
48assert list(map(ord, e)) == [92, 120, 56, 49]
49f = '\u1881'
50assert ord(f) == 0x1881
51g = r'\u1881'
52assert list(map(ord, g)) == [92, 117, 49, 56, 56, 49]
Serhiy Storchaka5e61f142013-02-10 17:36:00 +020053h = '\U0001d120'
54assert ord(h) == 0x1d120
55i = r'\U0001d120'
56assert list(map(ord, i)) == [92, 85, 48, 48, 48, 49, 100, 49, 50, 48]
Guido van Rossum29fd7122007-11-12 01:13:56 +000057"""
58
59
60def byte(i):
61 return bytes([i])
62
63
64class TestLiterals(unittest.TestCase):
65
66 def setUp(self):
67 self.save_path = sys.path[:]
68 self.tmpdir = tempfile.mkdtemp()
69 sys.path.insert(0, self.tmpdir)
70
71 def tearDown(self):
Georg Brandl242631d2012-02-20 21:36:28 +010072 sys.path[:] = self.save_path
Guido van Rossum29fd7122007-11-12 01:13:56 +000073 shutil.rmtree(self.tmpdir, ignore_errors=True)
74
75 def test_template(self):
76 # Check that the template doesn't contain any non-printables
77 # except for \n.
78 for c in TEMPLATE:
79 assert c == '\n' or ' ' <= c <= '~', repr(c)
80
81 def test_eval_str_normal(self):
82 self.assertEqual(eval(""" 'x' """), 'x')
83 self.assertEqual(eval(r""" '\x01' """), chr(1))
84 self.assertEqual(eval(""" '\x01' """), chr(1))
85 self.assertEqual(eval(r""" '\x81' """), chr(0x81))
86 self.assertEqual(eval(""" '\x81' """), chr(0x81))
87 self.assertEqual(eval(r""" '\u1881' """), chr(0x1881))
88 self.assertEqual(eval(""" '\u1881' """), chr(0x1881))
Serhiy Storchaka5e61f142013-02-10 17:36:00 +020089 self.assertEqual(eval(r""" '\U0001d120' """), chr(0x1d120))
90 self.assertEqual(eval(""" '\U0001d120' """), chr(0x1d120))
91
92 def test_eval_str_incomplete(self):
93 self.assertRaises(SyntaxError, eval, r""" '\x' """)
94 self.assertRaises(SyntaxError, eval, r""" '\x0' """)
95 self.assertRaises(SyntaxError, eval, r""" '\u' """)
96 self.assertRaises(SyntaxError, eval, r""" '\u0' """)
97 self.assertRaises(SyntaxError, eval, r""" '\u00' """)
98 self.assertRaises(SyntaxError, eval, r""" '\u000' """)
99 self.assertRaises(SyntaxError, eval, r""" '\U' """)
100 self.assertRaises(SyntaxError, eval, r""" '\U0' """)
101 self.assertRaises(SyntaxError, eval, r""" '\U00' """)
102 self.assertRaises(SyntaxError, eval, r""" '\U000' """)
103 self.assertRaises(SyntaxError, eval, r""" '\U0000' """)
104 self.assertRaises(SyntaxError, eval, r""" '\U00000' """)
105 self.assertRaises(SyntaxError, eval, r""" '\U000000' """)
106 self.assertRaises(SyntaxError, eval, r""" '\U0000000' """)
Guido van Rossum29fd7122007-11-12 01:13:56 +0000107
108 def test_eval_str_raw(self):
109 self.assertEqual(eval(""" r'x' """), 'x')
110 self.assertEqual(eval(r""" r'\x01' """), '\\' + 'x01')
111 self.assertEqual(eval(""" r'\x01' """), chr(1))
112 self.assertEqual(eval(r""" r'\x81' """), '\\' + 'x81')
113 self.assertEqual(eval(""" r'\x81' """), chr(0x81))
114 self.assertEqual(eval(r""" r'\u1881' """), '\\' + 'u1881')
115 self.assertEqual(eval(""" r'\u1881' """), chr(0x1881))
Serhiy Storchaka5e61f142013-02-10 17:36:00 +0200116 self.assertEqual(eval(r""" r'\U0001d120' """), '\\' + 'U0001d120')
117 self.assertEqual(eval(""" r'\U0001d120' """), chr(0x1d120))
Guido van Rossum29fd7122007-11-12 01:13:56 +0000118
119 def test_eval_bytes_normal(self):
120 self.assertEqual(eval(""" b'x' """), b'x')
121 self.assertEqual(eval(r""" b'\x01' """), byte(1))
122 self.assertEqual(eval(""" b'\x01' """), byte(1))
123 self.assertEqual(eval(r""" b'\x81' """), byte(0x81))
124 self.assertRaises(SyntaxError, eval, """ b'\x81' """)
125 self.assertEqual(eval(r""" b'\u1881' """), b'\\' + b'u1881')
126 self.assertRaises(SyntaxError, eval, """ b'\u1881' """)
Serhiy Storchaka5e61f142013-02-10 17:36:00 +0200127 self.assertEqual(eval(r""" b'\U0001d120' """), b'\\' + b'U0001d120')
128 self.assertRaises(SyntaxError, eval, """ b'\U0001d120' """)
129
130 def test_eval_bytes_incomplete(self):
131 self.assertRaises(SyntaxError, eval, r""" b'\x' """)
132 self.assertRaises(SyntaxError, eval, r""" b'\x0' """)
Guido van Rossum29fd7122007-11-12 01:13:56 +0000133
134 def test_eval_bytes_raw(self):
135 self.assertEqual(eval(""" br'x' """), b'x')
136 self.assertEqual(eval(r""" br'\x01' """), b'\\' + b'x01')
137 self.assertEqual(eval(""" br'\x01' """), byte(1))
138 self.assertEqual(eval(r""" br'\x81' """), b"\\" + b"x81")
139 self.assertRaises(SyntaxError, eval, """ br'\x81' """)
140 self.assertEqual(eval(r""" br'\u1881' """), b"\\" + b"u1881")
141 self.assertRaises(SyntaxError, eval, """ br'\u1881' """)
Serhiy Storchaka5e61f142013-02-10 17:36:00 +0200142 self.assertEqual(eval(r""" br'\U0001d120' """), b"\\" + b"U0001d120")
143 self.assertRaises(SyntaxError, eval, """ br'\U0001d120' """)
144 self.assertRaises(SyntaxError, eval, """ rb'' """)
145 self.assertRaises(SyntaxError, eval, """ bb'' """)
146 self.assertRaises(SyntaxError, eval, """ rr'' """)
147 self.assertRaises(SyntaxError, eval, """ brr'' """)
Guido van Rossum29fd7122007-11-12 01:13:56 +0000148
149 def check_encoding(self, encoding, extra=""):
150 modname = "xx_" + encoding.replace("-", "_")
151 fn = os.path.join(self.tmpdir, modname + ".py")
152 f = open(fn, "w", encoding=encoding)
153 try:
154 f.write(TEMPLATE % encoding)
155 f.write(extra)
156 finally:
157 f.close()
158 __import__(modname)
159 del sys.modules[modname]
160
161 def test_file_utf_8(self):
162 extra = "z = '\u1234'; assert ord(z) == 0x1234\n"
163 self.check_encoding("utf-8", extra)
164
165 def test_file_utf_8_error(self):
166 extra = "b'\x80'\n"
167 self.assertRaises(SyntaxError, self.check_encoding, "utf-8", extra)
168
169 def test_file_utf8(self):
170 self.check_encoding("utf8")
171
172 def test_file_iso_8859_1(self):
173 self.check_encoding("iso-8859-1")
174
175 def test_file_latin_1(self):
176 self.check_encoding("latin-1")
177
178 def test_file_latin9(self):
179 self.check_encoding("latin9")
180
181
Antoine Pitroubbf53612012-01-12 22:36:48 +0100182def test_main():
183 test.support.run_unittest(__name__)
184
Guido van Rossum29fd7122007-11-12 01:13:56 +0000185if __name__ == "__main__":
Antoine Pitroubbf53612012-01-12 22:36:48 +0100186 test_main()