blob: ba9fe465f837b9da00d7d9b27b2ef8d7232aa950 [file] [log] [blame]
Martin v. Löwis4b003072010-03-16 13:19:21 +00001import sys
Brett Cannonbefb14f2009-02-10 02:10:16 +00002import compileall
3import imp
4import os
5import py_compile
6import shutil
7import struct
Barry Warsaw28a691b2010-04-17 00:19:56 +00008import subprocess
Brett Cannonbefb14f2009-02-10 02:10:16 +00009import tempfile
R. David Murray650f1472010-11-20 21:18:51 +000010import time
Brett Cannonbefb14f2009-02-10 02:10:16 +000011import unittest
Martin v. Löwis4b003072010-03-16 13:19:21 +000012import io
Brett Cannonbefb14f2009-02-10 02:10:16 +000013
R. David Murray95333e32010-12-14 22:32:50 +000014from test import support, script_helper
Brett Cannonbefb14f2009-02-10 02:10:16 +000015
16class CompileallTests(unittest.TestCase):
17
18 def setUp(self):
19 self.directory = tempfile.mkdtemp()
20 self.source_path = os.path.join(self.directory, '_test.py')
Barry Warsaw28a691b2010-04-17 00:19:56 +000021 self.bc_path = imp.cache_from_source(self.source_path)
Brett Cannonbefb14f2009-02-10 02:10:16 +000022 with open(self.source_path, 'w') as file:
23 file.write('x = 123\n')
Matthias Klosec33b9022010-03-16 00:36:26 +000024 self.source_path2 = os.path.join(self.directory, '_test2.py')
Barry Warsaw28a691b2010-04-17 00:19:56 +000025 self.bc_path2 = imp.cache_from_source(self.source_path2)
Matthias Klosec33b9022010-03-16 00:36:26 +000026 shutil.copyfile(self.source_path, self.source_path2)
Georg Brandl45438462011-02-07 12:36:54 +000027 self.subdirectory = os.path.join(self.directory, '_subdir')
28 os.mkdir(self.subdirectory)
29 self.source_path3 = os.path.join(self.subdirectory, '_test3.py')
30 shutil.copyfile(self.source_path, self.source_path3)
Brett Cannonbefb14f2009-02-10 02:10:16 +000031
32 def tearDown(self):
33 shutil.rmtree(self.directory)
34
35 def data(self):
36 with open(self.bc_path, 'rb') as file:
37 data = file.read(8)
38 mtime = int(os.stat(self.source_path).st_mtime)
39 compare = struct.pack('<4sl', imp.get_magic(), mtime)
40 return data, compare
41
42 def recreation_check(self, metadata):
43 """Check that compileall recreates bytecode when the new metadata is
44 used."""
45 if not hasattr(os, 'stat'):
46 return
47 py_compile.compile(self.source_path)
48 self.assertEqual(*self.data())
49 with open(self.bc_path, 'rb') as file:
50 bc = file.read()[len(metadata):]
51 with open(self.bc_path, 'wb') as file:
52 file.write(metadata)
53 file.write(bc)
54 self.assertNotEqual(*self.data())
55 compileall.compile_dir(self.directory, force=False, quiet=True)
56 self.assertTrue(*self.data())
57
58 def test_mtime(self):
59 # Test a change in mtime leads to a new .pyc.
60 self.recreation_check(struct.pack('<4sl', imp.get_magic(), 1))
61
62 def test_magic_number(self):
63 # Test a change in mtime leads to a new .pyc.
64 self.recreation_check(b'\0\0\0\0')
65
Matthias Klosec33b9022010-03-16 00:36:26 +000066 def test_compile_files(self):
67 # Test compiling a single file, and complete directory
68 for fn in (self.bc_path, self.bc_path2):
69 try:
70 os.unlink(fn)
71 except:
72 pass
73 compileall.compile_file(self.source_path, force=False, quiet=True)
Barry Warsaw28a691b2010-04-17 00:19:56 +000074 self.assertTrue(os.path.isfile(self.bc_path) and
75 not os.path.isfile(self.bc_path2))
Matthias Klosec33b9022010-03-16 00:36:26 +000076 os.unlink(self.bc_path)
77 compileall.compile_dir(self.directory, force=False, quiet=True)
Barry Warsaw28a691b2010-04-17 00:19:56 +000078 self.assertTrue(os.path.isfile(self.bc_path) and
79 os.path.isfile(self.bc_path2))
Matthias Klosec33b9022010-03-16 00:36:26 +000080 os.unlink(self.bc_path)
81 os.unlink(self.bc_path2)
Brett Cannonbefb14f2009-02-10 02:10:16 +000082
Barry Warsawc8a99de2010-04-29 18:43:10 +000083 def test_no_pycache_in_non_package(self):
84 # Bug 8563 reported that __pycache__ directories got created by
85 # compile_file() for non-.py files.
86 data_dir = os.path.join(self.directory, 'data')
87 data_file = os.path.join(data_dir, 'file')
88 os.mkdir(data_dir)
89 # touch data/file
90 with open(data_file, 'w'):
91 pass
92 compileall.compile_file(data_file)
93 self.assertFalse(os.path.exists(os.path.join(data_dir, '__pycache__')))
94
Georg Brandl8334fd92010-12-04 10:26:46 +000095 def test_optimize(self):
96 # make sure compiling with different optimization settings than the
97 # interpreter's creates the correct file names
98 optimize = 1 if __debug__ else 0
99 compileall.compile_dir(self.directory, quiet=True, optimize=optimize)
100 cached = imp.cache_from_source(self.source_path,
101 debug_override=not optimize)
102 self.assertTrue(os.path.isfile(cached))
Georg Brandl45438462011-02-07 12:36:54 +0000103 cached2 = imp.cache_from_source(self.source_path2,
104 debug_override=not optimize)
105 self.assertTrue(os.path.isfile(cached2))
106 cached3 = imp.cache_from_source(self.source_path3,
107 debug_override=not optimize)
108 self.assertTrue(os.path.isfile(cached3))
Georg Brandl8334fd92010-12-04 10:26:46 +0000109
Barry Warsaw28a691b2010-04-17 00:19:56 +0000110
Martin v. Löwis4b003072010-03-16 13:19:21 +0000111class EncodingTest(unittest.TestCase):
Barry Warsaw28a691b2010-04-17 00:19:56 +0000112 """Issue 6716: compileall should escape source code when printing errors
113 to stdout."""
Martin v. Löwis4b003072010-03-16 13:19:21 +0000114
115 def setUp(self):
116 self.directory = tempfile.mkdtemp()
117 self.source_path = os.path.join(self.directory, '_test.py')
118 with open(self.source_path, 'w', encoding='utf-8') as file:
119 file.write('# -*- coding: utf-8 -*-\n')
120 file.write('print u"\u20ac"\n')
121
122 def tearDown(self):
123 shutil.rmtree(self.directory)
124
125 def test_error(self):
126 try:
127 orig_stdout = sys.stdout
128 sys.stdout = io.TextIOWrapper(io.BytesIO(),encoding='ascii')
129 compileall.compile_dir(self.directory)
130 finally:
131 sys.stdout = orig_stdout
132
Barry Warsawc8a99de2010-04-29 18:43:10 +0000133
Barry Warsaw28a691b2010-04-17 00:19:56 +0000134class CommandLineTests(unittest.TestCase):
R. David Murray650f1472010-11-20 21:18:51 +0000135 """Test compileall's CLI."""
Barry Warsaw28a691b2010-04-17 00:19:56 +0000136
Benjamin Petersona820c7c2012-09-25 11:42:35 -0400137 def _get_run_args(self, args):
138 interp_args = ['-S']
139 if sys.flags.optimize:
140 interp_args.append({1 : '-O', 2 : '-OO'}[sys.flags.optimize])
141 return interp_args + ['-m', 'compileall'] + list(args)
142
R. David Murray5317e9c2010-12-16 19:08:51 +0000143 def assertRunOK(self, *args, **env_vars):
144 rc, out, err = script_helper.assert_python_ok(
Benjamin Petersona820c7c2012-09-25 11:42:35 -0400145 *self._get_run_args(args), **env_vars)
R. David Murray95333e32010-12-14 22:32:50 +0000146 self.assertEqual(b'', err)
147 return out
148
R. David Murray5317e9c2010-12-16 19:08:51 +0000149 def assertRunNotOK(self, *args, **env_vars):
R. David Murray95333e32010-12-14 22:32:50 +0000150 rc, out, err = script_helper.assert_python_failure(
Benjamin Petersona820c7c2012-09-25 11:42:35 -0400151 *self._get_run_args(args), **env_vars)
R. David Murray95333e32010-12-14 22:32:50 +0000152 return rc, out, err
153
154 def assertCompiled(self, fn):
155 self.assertTrue(os.path.exists(imp.cache_from_source(fn)))
156
157 def assertNotCompiled(self, fn):
158 self.assertFalse(os.path.exists(imp.cache_from_source(fn)))
159
Barry Warsaw28a691b2010-04-17 00:19:56 +0000160 def setUp(self):
161 self.addCleanup(self._cleanup)
162 self.directory = tempfile.mkdtemp()
163 self.pkgdir = os.path.join(self.directory, 'foo')
164 os.mkdir(self.pkgdir)
R. David Murray95333e32010-12-14 22:32:50 +0000165 self.pkgdir_cachedir = os.path.join(self.pkgdir, '__pycache__')
166 # Create the __init__.py and a package module.
167 self.initfn = script_helper.make_script(self.pkgdir, '__init__', '')
168 self.barfn = script_helper.make_script(self.pkgdir, 'bar', '')
Barry Warsaw28a691b2010-04-17 00:19:56 +0000169
170 def _cleanup(self):
171 support.rmtree(self.directory)
R. David Murray5317e9c2010-12-16 19:08:51 +0000172
173 def test_no_args_compiles_path(self):
174 # Note that -l is implied for the no args case.
175 bazfn = script_helper.make_script(self.directory, 'baz', '')
176 self.assertRunOK(PYTHONPATH=self.directory)
177 self.assertCompiled(bazfn)
178 self.assertNotCompiled(self.initfn)
179 self.assertNotCompiled(self.barfn)
Barry Warsaw28a691b2010-04-17 00:19:56 +0000180
Georg Brandl1463a3f2010-10-14 07:42:27 +0000181 # Ensure that the default behavior of compileall's CLI is to create
182 # PEP 3147 pyc/pyo files.
183 for name, ext, switch in [
184 ('normal', 'pyc', []),
185 ('optimize', 'pyo', ['-O']),
Éric Araujo31717e82010-11-26 00:39:59 +0000186 ('doubleoptimize', 'pyo', ['-OO']),
Georg Brandl1463a3f2010-10-14 07:42:27 +0000187 ]:
188 def f(self, ext=ext, switch=switch):
R. David Murray95333e32010-12-14 22:32:50 +0000189 script_helper.assert_python_ok(*(switch +
190 ['-m', 'compileall', '-q', self.pkgdir]))
Georg Brandl1463a3f2010-10-14 07:42:27 +0000191 # Verify the __pycache__ directory contents.
R. David Murray95333e32010-12-14 22:32:50 +0000192 self.assertTrue(os.path.exists(self.pkgdir_cachedir))
Georg Brandl1463a3f2010-10-14 07:42:27 +0000193 expected = sorted(base.format(imp.get_tag(), ext) for base in
194 ('__init__.{}.{}', 'bar.{}.{}'))
R. David Murray95333e32010-12-14 22:32:50 +0000195 self.assertEqual(sorted(os.listdir(self.pkgdir_cachedir)), expected)
Georg Brandl1463a3f2010-10-14 07:42:27 +0000196 # Make sure there are no .pyc files in the source directory.
R. David Murray95333e32010-12-14 22:32:50 +0000197 self.assertFalse([fn for fn in os.listdir(self.pkgdir)
198 if fn.endswith(ext)])
Georg Brandl1463a3f2010-10-14 07:42:27 +0000199 locals()['test_pep3147_paths_' + name] = f
Barry Warsaw28a691b2010-04-17 00:19:56 +0000200
201 def test_legacy_paths(self):
202 # Ensure that with the proper switch, compileall leaves legacy
203 # pyc/pyo files, and no __pycache__ directory.
R. David Murray95333e32010-12-14 22:32:50 +0000204 self.assertRunOK('-b', '-q', self.pkgdir)
Barry Warsaw28a691b2010-04-17 00:19:56 +0000205 # Verify the __pycache__ directory contents.
R. David Murray95333e32010-12-14 22:32:50 +0000206 self.assertFalse(os.path.exists(self.pkgdir_cachedir))
Benjamin Petersona820c7c2012-09-25 11:42:35 -0400207 opt = 'c' if __debug__ else 'o'
208 expected = sorted(['__init__.py', '__init__.py' + opt, 'bar.py',
209 'bar.py' + opt])
Barry Warsaw28a691b2010-04-17 00:19:56 +0000210 self.assertEqual(sorted(os.listdir(self.pkgdir)), expected)
211
Barry Warsawc04317f2010-04-26 15:59:03 +0000212 def test_multiple_runs(self):
213 # Bug 8527 reported that multiple calls produced empty
214 # __pycache__/__pycache__ directories.
R. David Murray95333e32010-12-14 22:32:50 +0000215 self.assertRunOK('-q', self.pkgdir)
Barry Warsawc04317f2010-04-26 15:59:03 +0000216 # Verify the __pycache__ directory contents.
R. David Murray95333e32010-12-14 22:32:50 +0000217 self.assertTrue(os.path.exists(self.pkgdir_cachedir))
218 cachecachedir = os.path.join(self.pkgdir_cachedir, '__pycache__')
Barry Warsawc04317f2010-04-26 15:59:03 +0000219 self.assertFalse(os.path.exists(cachecachedir))
220 # Call compileall again.
R. David Murray95333e32010-12-14 22:32:50 +0000221 self.assertRunOK('-q', self.pkgdir)
222 self.assertTrue(os.path.exists(self.pkgdir_cachedir))
Barry Warsawc04317f2010-04-26 15:59:03 +0000223 self.assertFalse(os.path.exists(cachecachedir))
224
R. David Murray650f1472010-11-20 21:18:51 +0000225 def test_force(self):
R. David Murray95333e32010-12-14 22:32:50 +0000226 self.assertRunOK('-q', self.pkgdir)
227 pycpath = imp.cache_from_source(self.barfn)
R. David Murray650f1472010-11-20 21:18:51 +0000228 # set atime/mtime backward to avoid file timestamp resolution issues
229 os.utime(pycpath, (time.time()-60,)*2)
R. David Murray95333e32010-12-14 22:32:50 +0000230 mtime = os.stat(pycpath).st_mtime
231 # without force, no recompilation
232 self.assertRunOK('-q', self.pkgdir)
233 mtime2 = os.stat(pycpath).st_mtime
234 self.assertEqual(mtime, mtime2)
235 # now force it.
236 self.assertRunOK('-q', '-f', self.pkgdir)
237 mtime2 = os.stat(pycpath).st_mtime
238 self.assertNotEqual(mtime, mtime2)
R. David Murray650f1472010-11-20 21:18:51 +0000239
R. David Murray95333e32010-12-14 22:32:50 +0000240 def test_recursion_control(self):
241 subpackage = os.path.join(self.pkgdir, 'spam')
242 os.mkdir(subpackage)
243 subinitfn = script_helper.make_script(subpackage, '__init__', '')
244 hamfn = script_helper.make_script(subpackage, 'ham', '')
245 self.assertRunOK('-q', '-l', self.pkgdir)
246 self.assertNotCompiled(subinitfn)
247 self.assertFalse(os.path.exists(os.path.join(subpackage, '__pycache__')))
248 self.assertRunOK('-q', self.pkgdir)
249 self.assertCompiled(subinitfn)
250 self.assertCompiled(hamfn)
R. David Murray650f1472010-11-20 21:18:51 +0000251
252 def test_quiet(self):
R. David Murray95333e32010-12-14 22:32:50 +0000253 noisy = self.assertRunOK(self.pkgdir)
254 quiet = self.assertRunOK('-q', self.pkgdir)
255 self.assertNotEqual(b'', noisy)
256 self.assertEqual(b'', quiet)
R. David Murray650f1472010-11-20 21:18:51 +0000257
258 def test_regexp(self):
R David Murrayee1a7cb2011-07-01 14:55:43 -0400259 self.assertRunOK('-q', '-x', r'ba[^\\/]*$', self.pkgdir)
R. David Murray95333e32010-12-14 22:32:50 +0000260 self.assertNotCompiled(self.barfn)
261 self.assertCompiled(self.initfn)
R. David Murray650f1472010-11-20 21:18:51 +0000262
R. David Murray95333e32010-12-14 22:32:50 +0000263 def test_multiple_dirs(self):
264 pkgdir2 = os.path.join(self.directory, 'foo2')
265 os.mkdir(pkgdir2)
266 init2fn = script_helper.make_script(pkgdir2, '__init__', '')
267 bar2fn = script_helper.make_script(pkgdir2, 'bar2', '')
268 self.assertRunOK('-q', self.pkgdir, pkgdir2)
269 self.assertCompiled(self.initfn)
270 self.assertCompiled(self.barfn)
271 self.assertCompiled(init2fn)
272 self.assertCompiled(bar2fn)
273
274 def test_d_takes_exactly_one_dir(self):
275 rc, out, err = self.assertRunNotOK('-d', 'foo')
276 self.assertEqual(out, b'')
277 self.assertRegex(err, b'-d')
278 rc, out, err = self.assertRunNotOK('-d', 'foo', 'bar')
279 self.assertEqual(out, b'')
280 self.assertRegex(err, b'-d')
281
282 def test_d_compile_error(self):
283 script_helper.make_script(self.pkgdir, 'crunchyfrog', 'bad(syntax')
284 rc, out, err = self.assertRunNotOK('-q', '-d', 'dinsdale', self.pkgdir)
285 self.assertRegex(out, b'File "dinsdale')
286
287 def test_d_runtime_error(self):
288 bazfn = script_helper.make_script(self.pkgdir, 'baz', 'raise Exception')
289 self.assertRunOK('-q', '-d', 'dinsdale', self.pkgdir)
290 fn = script_helper.make_script(self.pkgdir, 'bing', 'import baz')
291 pyc = imp.cache_from_source(bazfn)
292 os.rename(pyc, os.path.join(self.pkgdir, 'baz.pyc'))
293 os.remove(bazfn)
294 rc, out, err = script_helper.assert_python_failure(fn)
295 self.assertRegex(err, b'File "dinsdale')
296
297 def test_include_bad_file(self):
298 rc, out, err = self.assertRunNotOK(
299 '-i', os.path.join(self.directory, 'nosuchfile'), self.pkgdir)
300 self.assertRegex(out, b'rror.*nosuchfile')
301 self.assertNotRegex(err, b'Traceback')
302 self.assertFalse(os.path.exists(imp.cache_from_source(
303 self.pkgdir_cachedir)))
304
305 def test_include_file_with_arg(self):
306 f1 = script_helper.make_script(self.pkgdir, 'f1', '')
307 f2 = script_helper.make_script(self.pkgdir, 'f2', '')
308 f3 = script_helper.make_script(self.pkgdir, 'f3', '')
309 f4 = script_helper.make_script(self.pkgdir, 'f4', '')
310 with open(os.path.join(self.directory, 'l1'), 'w') as l1:
311 l1.write(os.path.join(self.pkgdir, 'f1.py')+os.linesep)
312 l1.write(os.path.join(self.pkgdir, 'f2.py')+os.linesep)
313 self.assertRunOK('-i', os.path.join(self.directory, 'l1'), f4)
314 self.assertCompiled(f1)
315 self.assertCompiled(f2)
316 self.assertNotCompiled(f3)
317 self.assertCompiled(f4)
318
319 def test_include_file_no_arg(self):
320 f1 = script_helper.make_script(self.pkgdir, 'f1', '')
321 f2 = script_helper.make_script(self.pkgdir, 'f2', '')
322 f3 = script_helper.make_script(self.pkgdir, 'f3', '')
323 f4 = script_helper.make_script(self.pkgdir, 'f4', '')
324 with open(os.path.join(self.directory, 'l1'), 'w') as l1:
325 l1.write(os.path.join(self.pkgdir, 'f2.py')+os.linesep)
326 self.assertRunOK('-i', os.path.join(self.directory, 'l1'))
327 self.assertNotCompiled(f1)
328 self.assertCompiled(f2)
329 self.assertNotCompiled(f3)
330 self.assertNotCompiled(f4)
331
332 def test_include_on_stdin(self):
333 f1 = script_helper.make_script(self.pkgdir, 'f1', '')
334 f2 = script_helper.make_script(self.pkgdir, 'f2', '')
335 f3 = script_helper.make_script(self.pkgdir, 'f3', '')
336 f4 = script_helper.make_script(self.pkgdir, 'f4', '')
Benjamin Petersona820c7c2012-09-25 11:42:35 -0400337 p = script_helper.spawn_python(*(self._get_run_args(()) + ['-i', '-']))
R. David Murray95333e32010-12-14 22:32:50 +0000338 p.stdin.write((f3+os.linesep).encode('ascii'))
339 script_helper.kill_python(p)
340 self.assertNotCompiled(f1)
341 self.assertNotCompiled(f2)
342 self.assertCompiled(f3)
343 self.assertNotCompiled(f4)
344
345 def test_compiles_as_much_as_possible(self):
346 bingfn = script_helper.make_script(self.pkgdir, 'bing', 'syntax(error')
347 rc, out, err = self.assertRunNotOK('nosuchfile', self.initfn,
348 bingfn, self.barfn)
R. David Murray5317e9c2010-12-16 19:08:51 +0000349 self.assertRegex(out, b'rror')
R. David Murray95333e32010-12-14 22:32:50 +0000350 self.assertNotCompiled(bingfn)
351 self.assertCompiled(self.initfn)
352 self.assertCompiled(self.barfn)
353
R. David Murray5317e9c2010-12-16 19:08:51 +0000354 def test_invalid_arg_produces_message(self):
355 out = self.assertRunOK('badfilename')
Victor Stinner53071262011-05-11 00:36:28 +0200356 self.assertRegex(out, b"Can't list 'badfilename'")
R. David Murray650f1472010-11-20 21:18:51 +0000357
Barry Warsaw28a691b2010-04-17 00:19:56 +0000358
Brett Cannonbefb14f2009-02-10 02:10:16 +0000359def test_main():
Barry Warsaw28a691b2010-04-17 00:19:56 +0000360 support.run_unittest(
361 CommandLineTests,
362 CompileallTests,
363 EncodingTest,
364 )
Brett Cannonbefb14f2009-02-10 02:10:16 +0000365
366
367if __name__ == "__main__":
368 test_main()