blob: 4012a389d750d048067b2f392fceea661cae241f [file] [log] [blame]
Nick Coghlan39f0bb52017-11-28 08:11:51 +10001# Run the tests in Programs/_testembed.c (tests for the CPython embedding APIs)
2from test import support
3import unittest
4
5from collections import namedtuple
Victor Stinner7ddd56f2018-11-14 00:24:28 +01006import json
Nick Coghlan39f0bb52017-11-28 08:11:51 +10007import os
Michael Feltd2067312018-09-15 11:28:31 +02008import platform
Nick Coghlan39f0bb52017-11-28 08:11:51 +10009import re
10import subprocess
11import sys
Victor Stinnera6537fb2018-11-26 11:54:12 +010012import textwrap
Nick Coghlan39f0bb52017-11-28 08:11:51 +100013
14
Victor Stinner01de89c2018-11-14 17:39:45 +010015MS_WINDOWS = (os.name == 'nt')
16
17
Victor Stinner56b29b62018-07-26 18:57:56 +020018class EmbeddingTestsMixin:
Nick Coghlan39f0bb52017-11-28 08:11:51 +100019 def setUp(self):
20 here = os.path.abspath(__file__)
21 basepath = os.path.dirname(os.path.dirname(os.path.dirname(here)))
22 exename = "_testembed"
Victor Stinner01de89c2018-11-14 17:39:45 +010023 if MS_WINDOWS:
Nick Coghlan39f0bb52017-11-28 08:11:51 +100024 ext = ("_d" if "_d" in sys.executable else "") + ".exe"
25 exename += ext
26 exepath = os.path.dirname(sys.executable)
27 else:
28 exepath = os.path.join(basepath, "Programs")
29 self.test_exe = exe = os.path.join(exepath, exename)
30 if not os.path.exists(exe):
31 self.skipTest("%r doesn't exist" % exe)
32 # This is needed otherwise we get a fatal error:
33 # "Py_Initialize: Unable to get the locale encoding
34 # LookupError: no codec search functions registered: can't find encoding"
35 self.oldcwd = os.getcwd()
36 os.chdir(basepath)
37
38 def tearDown(self):
39 os.chdir(self.oldcwd)
40
41 def run_embedded_interpreter(self, *args, env=None):
42 """Runs a test in the embedded interpreter"""
43 cmd = [self.test_exe]
44 cmd.extend(args)
Victor Stinner01de89c2018-11-14 17:39:45 +010045 if env is not None and MS_WINDOWS:
Nick Coghlan39f0bb52017-11-28 08:11:51 +100046 # Windows requires at least the SYSTEMROOT environment variable to
47 # start Python.
48 env = env.copy()
49 env['SYSTEMROOT'] = os.environ['SYSTEMROOT']
50
51 p = subprocess.Popen(cmd,
52 stdout=subprocess.PIPE,
53 stderr=subprocess.PIPE,
54 universal_newlines=True,
55 env=env)
Victor Stinner2f549082019-03-29 15:13:46 +010056 try:
57 (out, err) = p.communicate()
58 except:
59 p.terminate()
60 p.wait()
61 raise
Nick Coghlan39f0bb52017-11-28 08:11:51 +100062 if p.returncode != 0 and support.verbose:
63 print(f"--- {cmd} failed ---")
64 print(f"stdout:\n{out}")
Nick Coghlanbc77eff2018-03-25 20:44:30 +100065 print(f"stderr:\n{err}")
Nick Coghlan39f0bb52017-11-28 08:11:51 +100066 print(f"------")
67
68 self.assertEqual(p.returncode, 0,
69 "bad returncode %d, stderr is %r" %
70 (p.returncode, err))
71 return out, err
72
73 def run_repeated_init_and_subinterpreters(self):
74 out, err = self.run_embedded_interpreter("repeated_init_and_subinterpreters")
75 self.assertEqual(err, "")
76
77 # The output from _testembed looks like this:
78 # --- Pass 0 ---
79 # interp 0 <0x1cf9330>, thread state <0x1cf9700>: id(modules) = 139650431942728
80 # interp 1 <0x1d4f690>, thread state <0x1d35350>: id(modules) = 139650431165784
81 # interp 2 <0x1d5a690>, thread state <0x1d99ed0>: id(modules) = 139650413140368
82 # interp 3 <0x1d4f690>, thread state <0x1dc3340>: id(modules) = 139650412862200
83 # interp 0 <0x1cf9330>, thread state <0x1cf9700>: id(modules) = 139650431942728
84 # --- Pass 1 ---
85 # ...
86
87 interp_pat = (r"^interp (\d+) <(0x[\dA-F]+)>, "
88 r"thread state <(0x[\dA-F]+)>: "
89 r"id\(modules\) = ([\d]+)$")
90 Interp = namedtuple("Interp", "id interp tstate modules")
91
92 numloops = 0
93 current_run = []
94 for line in out.splitlines():
95 if line == "--- Pass {} ---".format(numloops):
96 self.assertEqual(len(current_run), 0)
Nick Coghlanbc77eff2018-03-25 20:44:30 +100097 if support.verbose > 1:
Nick Coghlan39f0bb52017-11-28 08:11:51 +100098 print(line)
99 numloops += 1
100 continue
101
102 self.assertLess(len(current_run), 5)
103 match = re.match(interp_pat, line)
104 if match is None:
105 self.assertRegex(line, interp_pat)
106
107 # Parse the line from the loop. The first line is the main
108 # interpreter and the 3 afterward are subinterpreters.
109 interp = Interp(*match.groups())
Nick Coghlanbc77eff2018-03-25 20:44:30 +1000110 if support.verbose > 1:
Nick Coghlan39f0bb52017-11-28 08:11:51 +1000111 print(interp)
112 self.assertTrue(interp.interp)
113 self.assertTrue(interp.tstate)
114 self.assertTrue(interp.modules)
115 current_run.append(interp)
116
117 # The last line in the loop should be the same as the first.
118 if len(current_run) == 5:
119 main = current_run[0]
120 self.assertEqual(interp, main)
121 yield current_run
122 current_run = []
123
Victor Stinner56b29b62018-07-26 18:57:56 +0200124
125class EmbeddingTests(EmbeddingTestsMixin, unittest.TestCase):
Nick Coghlan39f0bb52017-11-28 08:11:51 +1000126 def test_subinterps_main(self):
127 for run in self.run_repeated_init_and_subinterpreters():
128 main = run[0]
129
130 self.assertEqual(main.id, '0')
131
132 def test_subinterps_different_ids(self):
133 for run in self.run_repeated_init_and_subinterpreters():
134 main, *subs, _ = run
135
136 mainid = int(main.id)
137 for i, sub in enumerate(subs):
138 self.assertEqual(sub.id, str(mainid + i + 1))
139
140 def test_subinterps_distinct_state(self):
141 for run in self.run_repeated_init_and_subinterpreters():
142 main, *subs, _ = run
143
144 if '0x0' in main:
145 # XXX Fix on Windows (and other platforms): something
146 # is going on with the pointers in Programs/_testembed.c.
147 # interp.interp is 0x0 and interp.modules is the same
148 # between interpreters.
149 raise unittest.SkipTest('platform prints pointers as 0x0')
150
151 for sub in subs:
152 # A new subinterpreter may have the same
153 # PyInterpreterState pointer as a previous one if
154 # the earlier one has already been destroyed. So
155 # we compare with the main interpreter. The same
156 # applies to tstate.
157 self.assertNotEqual(sub.interp, main.interp)
158 self.assertNotEqual(sub.tstate, main.tstate)
159 self.assertNotEqual(sub.modules, main.modules)
160
161 def test_forced_io_encoding(self):
162 # Checks forced configuration of embedded interpreter IO streams
163 env = dict(os.environ, PYTHONIOENCODING="utf-8:surrogateescape")
164 out, err = self.run_embedded_interpreter("forced_io_encoding", env=env)
165 if support.verbose > 1:
166 print()
167 print(out)
168 print(err)
169 expected_stream_encoding = "utf-8"
170 expected_errors = "surrogateescape"
171 expected_output = '\n'.join([
172 "--- Use defaults ---",
173 "Expected encoding: default",
174 "Expected errors: default",
175 "stdin: {in_encoding}:{errors}",
176 "stdout: {out_encoding}:{errors}",
177 "stderr: {out_encoding}:backslashreplace",
178 "--- Set errors only ---",
179 "Expected encoding: default",
180 "Expected errors: ignore",
181 "stdin: {in_encoding}:ignore",
182 "stdout: {out_encoding}:ignore",
183 "stderr: {out_encoding}:backslashreplace",
184 "--- Set encoding only ---",
Victor Stinner9e4994d2018-08-28 23:26:33 +0200185 "Expected encoding: iso8859-1",
Nick Coghlan39f0bb52017-11-28 08:11:51 +1000186 "Expected errors: default",
Victor Stinner9e4994d2018-08-28 23:26:33 +0200187 "stdin: iso8859-1:{errors}",
188 "stdout: iso8859-1:{errors}",
189 "stderr: iso8859-1:backslashreplace",
Nick Coghlan39f0bb52017-11-28 08:11:51 +1000190 "--- Set encoding and errors ---",
Victor Stinner9e4994d2018-08-28 23:26:33 +0200191 "Expected encoding: iso8859-1",
Nick Coghlan39f0bb52017-11-28 08:11:51 +1000192 "Expected errors: replace",
Victor Stinner9e4994d2018-08-28 23:26:33 +0200193 "stdin: iso8859-1:replace",
194 "stdout: iso8859-1:replace",
195 "stderr: iso8859-1:backslashreplace"])
Nick Coghlan39f0bb52017-11-28 08:11:51 +1000196 expected_output = expected_output.format(
197 in_encoding=expected_stream_encoding,
198 out_encoding=expected_stream_encoding,
199 errors=expected_errors)
200 # This is useful if we ever trip over odd platform behaviour
201 self.maxDiff = None
202 self.assertEqual(out.strip(), expected_output)
203
204 def test_pre_initialization_api(self):
205 """
Nick Coghlanbc77eff2018-03-25 20:44:30 +1000206 Checks some key parts of the C-API that need to work before the runtine
Nick Coghlan39f0bb52017-11-28 08:11:51 +1000207 is initialized (via Py_Initialize()).
208 """
209 env = dict(os.environ, PYTHONPATH=os.pathsep.join(sys.path))
210 out, err = self.run_embedded_interpreter("pre_initialization_api", env=env)
Victor Stinner01de89c2018-11-14 17:39:45 +0100211 if MS_WINDOWS:
Nick Coghlanbc77eff2018-03-25 20:44:30 +1000212 expected_path = self.test_exe
213 else:
214 expected_path = os.path.join(os.getcwd(), "spam")
215 expected_output = f"sys.executable: {expected_path}\n"
216 self.assertIn(expected_output, out)
217 self.assertEqual(err, '')
218
219 def test_pre_initialization_sys_options(self):
220 """
221 Checks that sys.warnoptions and sys._xoptions can be set before the
222 runtime is initialized (otherwise they won't be effective).
223 """
Pablo Galindo41148462018-04-27 13:23:13 +0100224 env = dict(os.environ, PYTHONPATH=os.pathsep.join(sys.path))
Nick Coghlanbc77eff2018-03-25 20:44:30 +1000225 out, err = self.run_embedded_interpreter(
226 "pre_initialization_sys_options", env=env)
227 expected_output = (
228 "sys.warnoptions: ['once', 'module', 'default']\n"
229 "sys._xoptions: {'not_an_option': '1', 'also_not_an_option': '2'}\n"
230 "warnings.filters[:3]: ['default', 'module', 'once']\n"
231 )
232 self.assertIn(expected_output, out)
Nick Coghlan39f0bb52017-11-28 08:11:51 +1000233 self.assertEqual(err, '')
234
Victor Stinnerb4d1e1f2017-11-30 22:05:00 +0100235 def test_bpo20891(self):
236 """
237 bpo-20891: Calling PyGILState_Ensure in a non-Python thread before
238 calling PyEval_InitThreads() must not crash. PyGILState_Ensure() must
239 call PyEval_InitThreads() for us in this case.
240 """
241 out, err = self.run_embedded_interpreter("bpo20891")
242 self.assertEqual(out, '')
243 self.assertEqual(err, '')
244
Victor Stinner209abf72018-06-22 19:14:51 +0200245 def test_initialize_twice(self):
246 """
247 bpo-33932: Calling Py_Initialize() twice should do nothing (and not
248 crash!).
249 """
250 out, err = self.run_embedded_interpreter("initialize_twice")
251 self.assertEqual(out, '')
252 self.assertEqual(err, '')
253
Victor Stinnerfb47bca2018-07-20 17:34:23 +0200254 def test_initialize_pymain(self):
255 """
256 bpo-34008: Calling Py_Main() after Py_Initialize() must not fail.
257 """
258 out, err = self.run_embedded_interpreter("initialize_pymain")
259 self.assertEqual(out.rstrip(), "Py_Main() after Py_Initialize: sys.argv=['-c', 'arg2']")
260 self.assertEqual(err, '')
261
Victor Stinner2f549082019-03-29 15:13:46 +0100262 def test_run_main(self):
263 out, err = self.run_embedded_interpreter("run_main")
264 self.assertEqual(out.rstrip(), "_Py_RunMain(): sys.argv=['-c', 'arg2']")
265 self.assertEqual(err, '')
266
Nick Coghlan39f0bb52017-11-28 08:11:51 +1000267
Victor Stinner56b29b62018-07-26 18:57:56 +0200268class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
269 maxDiff = 4096
Victor Stinner01de89c2018-11-14 17:39:45 +0100270 UTF8_MODE_ERRORS = ('surrogatepass' if MS_WINDOWS else 'surrogateescape')
271
Victor Stinnera6537fb2018-11-26 11:54:12 +0100272 # Mark config which should be get by get_default_config()
273 GET_DEFAULT_CONFIG = object()
Victor Stinner1075d162019-03-25 23:19:57 +0100274 DEFAULT_PRE_CONFIG = {
275 'allocator': None,
276 'coerce_c_locale': 0,
277 'coerce_c_locale_warn': 0,
Victor Stinner1075d162019-03-25 23:19:57 +0100278 'utf8_mode': 0,
279 }
Victor Stinner20004952019-03-26 02:31:11 +0100280 COPY_PRE_CONFIG = [
281 'dev_mode',
282 'isolated',
283 'use_environment',
284 ]
285
Victor Stinner00b137c2018-11-13 19:59:26 +0100286 DEFAULT_CORE_CONFIG = {
Victor Stinner20004952019-03-26 02:31:11 +0100287 'isolated': 0,
288 'use_environment': 1,
289 'dev_mode': 0,
290
Victor Stinner56b29b62018-07-26 18:57:56 +0200291 'install_signal_handlers': 1,
Victor Stinner56b29b62018-07-26 18:57:56 +0200292 'use_hash_seed': 0,
293 'hash_seed': 0,
Victor Stinner56b29b62018-07-26 18:57:56 +0200294 'faulthandler': 0,
295 'tracemalloc': 0,
296 'import_time': 0,
297 'show_ref_count': 0,
298 'show_alloc_count': 0,
299 'dump_refs': 0,
300 'malloc_stats': 0,
Victor Stinner56b29b62018-07-26 18:57:56 +0200301
Victor Stinnera6537fb2018-11-26 11:54:12 +0100302 'filesystem_encoding': GET_DEFAULT_CONFIG,
303 'filesystem_errors': GET_DEFAULT_CONFIG,
Victor Stinnerc5989cd2018-08-29 19:32:47 +0200304
Victor Stinner7ddd56f2018-11-14 00:24:28 +0100305 'pycache_prefix': None,
Victor Stinner91c99872019-05-14 22:01:51 +0200306 'program_name': GET_DEFAULT_CONFIG,
Victor Stinnerae239f62019-05-16 17:02:56 +0200307 'parse_argv': 1,
Victor Stinner62599762019-03-15 16:03:23 +0100308 'argv': [""],
Victor Stinner7ddd56f2018-11-14 00:24:28 +0100309
310 'xoptions': [],
311 'warnoptions': [],
Victor Stinner56b29b62018-07-26 18:57:56 +0200312
Victor Stinner01de89c2018-11-14 17:39:45 +0100313 'module_search_path_env': None,
314 'home': None,
Victor Stinner91c99872019-05-14 22:01:51 +0200315 'executable': GET_DEFAULT_CONFIG,
Victor Stinnera6537fb2018-11-26 11:54:12 +0100316
317 'prefix': GET_DEFAULT_CONFIG,
318 'base_prefix': GET_DEFAULT_CONFIG,
319 'exec_prefix': GET_DEFAULT_CONFIG,
320 'base_exec_prefix': GET_DEFAULT_CONFIG,
Victor Stinner5eb8b072019-05-15 02:12:48 +0200321 'module_search_paths': GET_DEFAULT_CONFIG,
Victor Stinner01de89c2018-11-14 17:39:45 +0100322
Victor Stinner56b29b62018-07-26 18:57:56 +0200323 'site_import': 1,
324 'bytes_warning': 0,
325 'inspect': 0,
326 'interactive': 0,
327 'optimization_level': 0,
Victor Stinner98512272018-08-01 03:07:00 +0200328 'parser_debug': 0,
Victor Stinner56b29b62018-07-26 18:57:56 +0200329 'write_bytecode': 1,
330 'verbose': 0,
331 'quiet': 0,
332 'user_site_directory': 1,
Victor Stinner54b43bb2019-05-16 18:30:15 +0200333 'configure_c_stdio': 1,
Victor Stinner98512272018-08-01 03:07:00 +0200334 'buffered_stdio': 1,
Victor Stinnerc5989cd2018-08-29 19:32:47 +0200335
Victor Stinnera6537fb2018-11-26 11:54:12 +0100336 'stdio_encoding': GET_DEFAULT_CONFIG,
337 'stdio_errors': GET_DEFAULT_CONFIG,
Victor Stinner56b29b62018-07-26 18:57:56 +0200338
Victor Stinner62be7632019-03-01 13:10:14 +0100339 'skip_source_first_line': 0,
340 'run_command': None,
341 'run_module': None,
342 'run_filename': None,
343
Victor Stinner56b29b62018-07-26 18:57:56 +0200344 '_install_importlib': 1,
Victor Stinnercb9fbd32019-05-01 23:51:56 -0400345 'check_hash_pycs_mode': 'default',
Victor Stinner9ef5dca2019-05-16 17:38:16 +0200346 'pathconfig_warnings': 1,
347 '_init_main': 1,
Victor Stinner56b29b62018-07-26 18:57:56 +0200348 }
Victor Stinner01de89c2018-11-14 17:39:45 +0100349 if MS_WINDOWS:
Victor Stinner1075d162019-03-25 23:19:57 +0100350 DEFAULT_PRE_CONFIG.update({
Victor Stinner01de89c2018-11-14 17:39:45 +0100351 'legacy_windows_fs_encoding': 0,
Victor Stinner1075d162019-03-25 23:19:57 +0100352 })
353 DEFAULT_CORE_CONFIG.update({
Victor Stinner5eb8b072019-05-15 02:12:48 +0200354 'dll_path': GET_DEFAULT_CONFIG,
Victor Stinner01de89c2018-11-14 17:39:45 +0100355 'legacy_windows_stdio': 0,
356 })
357
Victor Stinner01de89c2018-11-14 17:39:45 +0100358 # global config
359 DEFAULT_GLOBAL_CONFIG = {
360 'Py_HasFileSystemDefaultEncoding': 0,
361 'Py_HashRandomizationFlag': 1,
362 '_Py_HasFileSystemDefaultEncodeErrors': 0,
363 }
Victor Stinner1075d162019-03-25 23:19:57 +0100364 COPY_GLOBAL_PRE_CONFIG = [
Victor Stinner1075d162019-03-25 23:19:57 +0100365 ('Py_UTF8Mode', 'utf8_mode'),
366 ]
Victor Stinner01de89c2018-11-14 17:39:45 +0100367 COPY_GLOBAL_CONFIG = [
368 # Copy core config to global config for expected values
369 # True means that the core config value is inverted (0 => 1 and 1 => 0)
370 ('Py_BytesWarningFlag', 'bytes_warning'),
371 ('Py_DebugFlag', 'parser_debug'),
372 ('Py_DontWriteBytecodeFlag', 'write_bytecode', True),
373 ('Py_FileSystemDefaultEncodeErrors', 'filesystem_errors'),
374 ('Py_FileSystemDefaultEncoding', 'filesystem_encoding'),
Victor Stinner9ef5dca2019-05-16 17:38:16 +0200375 ('Py_FrozenFlag', 'pathconfig_warnings', True),
Victor Stinner20004952019-03-26 02:31:11 +0100376 ('Py_IgnoreEnvironmentFlag', 'use_environment', True),
Victor Stinner01de89c2018-11-14 17:39:45 +0100377 ('Py_InspectFlag', 'inspect'),
378 ('Py_InteractiveFlag', 'interactive'),
Victor Stinner20004952019-03-26 02:31:11 +0100379 ('Py_IsolatedFlag', 'isolated'),
Victor Stinner01de89c2018-11-14 17:39:45 +0100380 ('Py_NoSiteFlag', 'site_import', True),
381 ('Py_NoUserSiteDirectory', 'user_site_directory', True),
382 ('Py_OptimizeFlag', 'optimization_level'),
383 ('Py_QuietFlag', 'quiet'),
Victor Stinner01de89c2018-11-14 17:39:45 +0100384 ('Py_UnbufferedStdioFlag', 'buffered_stdio', True),
385 ('Py_VerboseFlag', 'verbose'),
386 ]
387 if MS_WINDOWS:
Victor Stinner1075d162019-03-25 23:19:57 +0100388 COPY_GLOBAL_PRE_CONFIG.extend((
Victor Stinner01de89c2018-11-14 17:39:45 +0100389 ('Py_LegacyWindowsFSEncodingFlag', 'legacy_windows_fs_encoding'),
Victor Stinner1075d162019-03-25 23:19:57 +0100390 ))
391 COPY_GLOBAL_CONFIG.extend((
Victor Stinner01de89c2018-11-14 17:39:45 +0100392 ('Py_LegacyWindowsStdioFlag', 'legacy_windows_stdio'),
393 ))
Victor Stinner56b29b62018-07-26 18:57:56 +0200394
Victor Stinner01de89c2018-11-14 17:39:45 +0100395 def main_xoptions(self, xoptions_list):
396 xoptions = {}
397 for opt in xoptions_list:
398 if '=' in opt:
399 key, value = opt.split('=', 1)
400 xoptions[key] = value
401 else:
402 xoptions[opt] = True
403 return xoptions
Victor Stinnerdfe0dc72018-08-29 11:47:29 +0200404
Victor Stinner91c99872019-05-14 22:01:51 +0200405 def get_expected_config(self, expected, env, add_path=None):
Victor Stinnera6537fb2018-11-26 11:54:12 +0100406 expected = dict(self.DEFAULT_CORE_CONFIG, **expected)
407
408 code = textwrap.dedent('''
409 import json
Victor Stinnera6537fb2018-11-26 11:54:12 +0100410 import sys
Victor Stinner5eb8b072019-05-15 02:12:48 +0200411 import _testinternalcapi
Victor Stinnera6537fb2018-11-26 11:54:12 +0100412
Victor Stinner5eb8b072019-05-15 02:12:48 +0200413 configs = _testinternalcapi.get_configs()
414 core_config = configs['core_config']
Victor Stinnera6537fb2018-11-26 11:54:12 +0100415 data = {
416 'stdio_encoding': sys.stdout.encoding,
417 'stdio_errors': sys.stdout.errors,
418 'prefix': sys.prefix,
419 'base_prefix': sys.base_prefix,
420 'exec_prefix': sys.exec_prefix,
421 'base_exec_prefix': sys.base_exec_prefix,
422 'filesystem_encoding': sys.getfilesystemencoding(),
423 'filesystem_errors': sys.getfilesystemencodeerrors(),
Victor Stinner5eb8b072019-05-15 02:12:48 +0200424 'module_search_paths': core_config['module_search_paths'],
Victor Stinnera6537fb2018-11-26 11:54:12 +0100425 }
Victor Stinner5eb8b072019-05-15 02:12:48 +0200426 if sys.platform == 'win32':
427 data['dll_path'] = core_config['dll_path']
Victor Stinnera6537fb2018-11-26 11:54:12 +0100428
429 data = json.dumps(data)
430 data = data.encode('utf-8')
431 sys.stdout.buffer.write(data)
432 sys.stdout.buffer.flush()
433 ''')
434
435 # Use -S to not import the site module: get the proper configuration
436 # when test_embed is run from a venv (bpo-35313)
Victor Stinner5eb8b072019-05-15 02:12:48 +0200437 args = [sys.executable, '-S', '-c', code]
Victor Stinnera6537fb2018-11-26 11:54:12 +0100438 env = dict(env)
Victor Stinner20004952019-03-26 02:31:11 +0100439 if not expected['isolated']:
Victor Stinnera6537fb2018-11-26 11:54:12 +0100440 env['PYTHONCOERCECLOCALE'] = '0'
441 env['PYTHONUTF8'] = '0'
442 proc = subprocess.run(args, env=env,
443 stdout=subprocess.PIPE,
444 stderr=subprocess.STDOUT)
445 if proc.returncode:
446 raise Exception(f"failed to get the default config: "
447 f"stdout={proc.stdout!r} stderr={proc.stderr!r}")
448 stdout = proc.stdout.decode('utf-8')
Victor Stinner4631da12019-05-02 15:30:21 -0400449 try:
450 config = json.loads(stdout)
451 except json.JSONDecodeError:
452 self.fail(f"fail to decode stdout: {stdout!r}")
Victor Stinnera6537fb2018-11-26 11:54:12 +0100453
Victor Stinner91c99872019-05-14 22:01:51 +0200454 if expected['executable'] is self.GET_DEFAULT_CONFIG:
455 if sys.platform == 'win32':
456 expected['executable'] = self.test_exe
457 else:
458 if expected['program_name'] is not self.GET_DEFAULT_CONFIG:
459 expected['executable'] = os.path.abspath(expected['program_name'])
460 else:
461 expected['executable'] = os.path.join(os.getcwd(), '_testembed')
462 if expected['program_name'] is self.GET_DEFAULT_CONFIG:
463 expected['program_name'] = './_testembed'
464
Victor Stinnera6537fb2018-11-26 11:54:12 +0100465 for key, value in expected.items():
466 if value is self.GET_DEFAULT_CONFIG:
467 expected[key] = config[key]
Victor Stinner5eb8b072019-05-15 02:12:48 +0200468
469 if add_path is not None:
470 expected['module_search_paths'].append(add_path)
Victor Stinnera6537fb2018-11-26 11:54:12 +0100471 return expected
Victor Stinner01de89c2018-11-14 17:39:45 +0100472
Victor Stinner1075d162019-03-25 23:19:57 +0100473 def check_pre_config(self, config, expected):
474 pre_config = dict(config['pre_config'])
475 core_config = dict(config['core_config'])
476 self.assertEqual(pre_config, expected)
477
Victor Stinner5eb8b072019-05-15 02:12:48 +0200478 def check_core_config(self, config, expected):
Victor Stinner01de89c2018-11-14 17:39:45 +0100479 core_config = dict(config['core_config'])
Victor Stinner01de89c2018-11-14 17:39:45 +0100480 self.assertEqual(core_config, expected)
Victor Stinnerdfe0dc72018-08-29 11:47:29 +0200481
Victor Stinner01de89c2018-11-14 17:39:45 +0100482 def check_global_config(self, config):
Victor Stinner1075d162019-03-25 23:19:57 +0100483 pre_config = config['pre_config']
Victor Stinner7ddd56f2018-11-14 00:24:28 +0100484 core_config = config['core_config']
Victor Stinner00b137c2018-11-13 19:59:26 +0100485
Victor Stinnera6537fb2018-11-26 11:54:12 +0100486 expected = dict(self.DEFAULT_GLOBAL_CONFIG)
Victor Stinner01de89c2018-11-14 17:39:45 +0100487 for item in self.COPY_GLOBAL_CONFIG:
Victor Stinner7ddd56f2018-11-14 00:24:28 +0100488 if len(item) == 3:
489 global_key, core_key, opposite = item
Victor Stinnera6537fb2018-11-26 11:54:12 +0100490 expected[global_key] = 0 if core_config[core_key] else 1
Victor Stinner7ddd56f2018-11-14 00:24:28 +0100491 else:
492 global_key, core_key = item
Victor Stinnera6537fb2018-11-26 11:54:12 +0100493 expected[global_key] = core_config[core_key]
Victor Stinner1075d162019-03-25 23:19:57 +0100494 for item in self.COPY_GLOBAL_PRE_CONFIG:
495 if len(item) == 3:
496 global_key, core_key, opposite = item
497 expected[global_key] = 0 if pre_config[core_key] else 1
498 else:
499 global_key, core_key = item
500 expected[global_key] = pre_config[core_key]
Victor Stinner7ddd56f2018-11-14 00:24:28 +0100501
Victor Stinnera6537fb2018-11-26 11:54:12 +0100502 self.assertEqual(config['global_config'], expected)
Victor Stinner7ddd56f2018-11-14 00:24:28 +0100503
Victor Stinner9ef5dca2019-05-16 17:38:16 +0200504 def check_config(self, testname, expected_config, expected_preconfig,
505 add_path=None, stderr=None):
Victor Stinner01de89c2018-11-14 17:39:45 +0100506 env = dict(os.environ)
507 # Remove PYTHON* environment variables to get deterministic environment
508 for key in list(env):
509 if key.startswith('PYTHON'):
510 del env[key]
511 # Disable C locale coercion and UTF-8 mode to not depend
512 # on the current locale
513 env['PYTHONCOERCECLOCALE'] = '0'
514 env['PYTHONUTF8'] = '0'
515
Victor Stinner1075d162019-03-25 23:19:57 +0100516 expected_preconfig = dict(self.DEFAULT_PRE_CONFIG, **expected_preconfig)
Victor Stinner91c99872019-05-14 22:01:51 +0200517 expected_config = self.get_expected_config(expected_config, env, add_path)
Victor Stinner20004952019-03-26 02:31:11 +0100518 for key in self.COPY_PRE_CONFIG:
519 if key not in expected_preconfig:
520 expected_preconfig[key] = expected_config[key]
Victor Stinner1075d162019-03-25 23:19:57 +0100521
Victor Stinner9ef5dca2019-05-16 17:38:16 +0200522 out, err = self.run_embedded_interpreter(testname, env=env)
523 if stderr is None and not expected_config['verbose']:
524 stderr = ""
525 if stderr is not None:
526 self.assertEqual(err.rstrip(), stderr)
527 try:
528 config = json.loads(out)
529 except json.JSONDecodeError:
530 self.fail(f"fail to decode stdout: {out!r}")
531
Victor Stinner1075d162019-03-25 23:19:57 +0100532 self.check_pre_config(config, expected_preconfig)
Victor Stinner5eb8b072019-05-15 02:12:48 +0200533 self.check_core_config(config, expected_config)
Victor Stinner01de89c2018-11-14 17:39:45 +0100534 self.check_global_config(config)
Victor Stinner7ddd56f2018-11-14 00:24:28 +0100535
Victor Stinner56b29b62018-07-26 18:57:56 +0200536 def test_init_default_config(self):
Victor Stinner1075d162019-03-25 23:19:57 +0100537 self.check_config("init_default_config", {}, {})
Victor Stinner56b29b62018-07-26 18:57:56 +0200538
539 def test_init_global_config(self):
Victor Stinner1075d162019-03-25 23:19:57 +0100540 preconfig = {
541 'utf8_mode': 1,
542 }
Victor Stinner56b29b62018-07-26 18:57:56 +0200543 config = {
544 'program_name': './globalvar',
545 'site_import': 0,
546 'bytes_warning': 1,
Victor Stinnerf8ba6f52019-03-26 16:58:50 +0100547 'warnoptions': ['default::BytesWarning'],
Victor Stinner56b29b62018-07-26 18:57:56 +0200548 'inspect': 1,
549 'interactive': 1,
550 'optimization_level': 2,
551 'write_bytecode': 0,
552 'verbose': 1,
553 'quiet': 1,
Victor Stinner98512272018-08-01 03:07:00 +0200554 'buffered_stdio': 0,
Victor Stinnerdfe0dc72018-08-29 11:47:29 +0200555
Victor Stinnerdfe0dc72018-08-29 11:47:29 +0200556 'stdio_encoding': 'utf-8',
557 'stdio_errors': 'surrogateescape',
Victor Stinnerb2457ef2018-08-29 13:25:36 +0200558 'filesystem_encoding': 'utf-8',
559 'filesystem_errors': self.UTF8_MODE_ERRORS,
Victor Stinner56b29b62018-07-26 18:57:56 +0200560 'user_site_directory': 0,
Victor Stinner54b43bb2019-05-16 18:30:15 +0200561 'pathconfig_warnings': 0,
Victor Stinner56b29b62018-07-26 18:57:56 +0200562 }
Victor Stinner1075d162019-03-25 23:19:57 +0100563 self.check_config("init_global_config", config, preconfig)
Victor Stinner56b29b62018-07-26 18:57:56 +0200564
565 def test_init_from_config(self):
Victor Stinner1075d162019-03-25 23:19:57 +0100566 preconfig = {
567 'allocator': 'malloc',
568 'utf8_mode': 1,
569 }
Victor Stinner56b29b62018-07-26 18:57:56 +0200570 config = {
571 'install_signal_handlers': 0,
572 'use_hash_seed': 1,
573 'hash_seed': 123,
Victor Stinner56b29b62018-07-26 18:57:56 +0200574 'tracemalloc': 2,
575 'import_time': 1,
576 'show_ref_count': 1,
577 'show_alloc_count': 1,
578 'malloc_stats': 1,
579
Victor Stinnerdfe0dc72018-08-29 11:47:29 +0200580 'stdio_encoding': 'iso8859-1',
581 'stdio_errors': 'replace',
Victor Stinnerb2457ef2018-08-29 13:25:36 +0200582 'filesystem_encoding': 'utf-8',
583 'filesystem_errors': self.UTF8_MODE_ERRORS,
Victor Stinner56b29b62018-07-26 18:57:56 +0200584
585 'pycache_prefix': 'conf_pycache_prefix',
586 'program_name': './conf_program_name',
Victor Stinner2f549082019-03-29 15:13:46 +0100587 'argv': ['-c', 'arg2'],
Victor Stinner01de89c2018-11-14 17:39:45 +0100588 'xoptions': ['core_xoption1=3', 'core_xoption2=', 'core_xoption3'],
Victor Stinnerf8ba6f52019-03-26 16:58:50 +0100589 'warnoptions': ['error::ResourceWarning', 'default::BytesWarning'],
Victor Stinner2f549082019-03-29 15:13:46 +0100590 'run_command': 'pass\n',
Victor Stinner56b29b62018-07-26 18:57:56 +0200591
592 'site_import': 0,
593 'bytes_warning': 1,
594 'inspect': 1,
595 'interactive': 1,
596 'optimization_level': 2,
597 'write_bytecode': 0,
598 'verbose': 1,
599 'quiet': 1,
Victor Stinner54b43bb2019-05-16 18:30:15 +0200600 'configure_c_stdio': 0,
Victor Stinner98512272018-08-01 03:07:00 +0200601 'buffered_stdio': 0,
Victor Stinner56b29b62018-07-26 18:57:56 +0200602 'user_site_directory': 0,
603 'faulthandler': 1,
Victor Stinnerb75d7e22018-08-01 02:13:04 +0200604
Victor Stinnercb9fbd32019-05-01 23:51:56 -0400605 'check_hash_pycs_mode': 'always',
Victor Stinner54b43bb2019-05-16 18:30:15 +0200606 'pathconfig_warnings': 0,
Victor Stinner56b29b62018-07-26 18:57:56 +0200607 }
Victor Stinner1075d162019-03-25 23:19:57 +0100608 self.check_config("init_from_config", config, preconfig)
Victor Stinner56b29b62018-07-26 18:57:56 +0200609
Victor Stinner1075d162019-03-25 23:19:57 +0100610 INIT_ENV_PRECONFIG = {
611 'allocator': 'malloc',
Victor Stinner1075d162019-03-25 23:19:57 +0100612 }
Victor Stinnerb35be4b2019-03-05 17:37:44 +0100613 INIT_ENV_CONFIG = {
614 'use_hash_seed': 1,
615 'hash_seed': 42,
Victor Stinnerb35be4b2019-03-05 17:37:44 +0100616 'tracemalloc': 2,
617 'import_time': 1,
618 'malloc_stats': 1,
Victor Stinnerb35be4b2019-03-05 17:37:44 +0100619 'inspect': 1,
620 'optimization_level': 2,
621 'pycache_prefix': 'env_pycache_prefix',
622 'write_bytecode': 0,
623 'verbose': 1,
624 'buffered_stdio': 0,
625 'stdio_encoding': 'iso8859-1',
626 'stdio_errors': 'replace',
627 'user_site_directory': 0,
628 'faulthandler': 1,
629 }
630
Victor Stinner56b29b62018-07-26 18:57:56 +0200631 def test_init_env(self):
Victor Stinner1075d162019-03-25 23:19:57 +0100632 self.check_config("init_env", self.INIT_ENV_CONFIG, self.INIT_ENV_PRECONFIG)
Victor Stinnerb35be4b2019-03-05 17:37:44 +0100633
634 def test_init_env_dev_mode(self):
Victor Stinner1075d162019-03-25 23:19:57 +0100635 preconfig = dict(self.INIT_ENV_PRECONFIG,
Victor Stinner20004952019-03-26 02:31:11 +0100636 allocator='debug')
Victor Stinner1075d162019-03-25 23:19:57 +0100637 config = dict(self.INIT_ENV_CONFIG,
Victor Stinnerf8ba6f52019-03-26 16:58:50 +0100638 dev_mode=1,
639 warnoptions=['default'])
Victor Stinner1075d162019-03-25 23:19:57 +0100640 self.check_config("init_env_dev_mode", config, preconfig)
Victor Stinner56b29b62018-07-26 18:57:56 +0200641
Victor Stinner20004952019-03-26 02:31:11 +0100642 def test_init_env_dev_mode_alloc(self):
Victor Stinner1075d162019-03-25 23:19:57 +0100643 preconfig = dict(self.INIT_ENV_PRECONFIG,
Victor Stinner20004952019-03-26 02:31:11 +0100644 allocator='malloc')
645 config = dict(self.INIT_ENV_CONFIG,
Victor Stinnerf8ba6f52019-03-26 16:58:50 +0100646 dev_mode=1,
647 warnoptions=['default'])
Victor Stinner1075d162019-03-25 23:19:57 +0100648 self.check_config("init_env_dev_mode_alloc", config, preconfig)
Victor Stinner25d13f32019-03-06 12:51:53 +0100649
Victor Stinner56b29b62018-07-26 18:57:56 +0200650 def test_init_dev_mode(self):
Victor Stinner1075d162019-03-25 23:19:57 +0100651 preconfig = {
Victor Stinnerc656e252019-03-06 01:13:43 +0100652 'allocator': 'debug',
Victor Stinner56b29b62018-07-26 18:57:56 +0200653 }
Victor Stinner1075d162019-03-25 23:19:57 +0100654 config = {
655 'faulthandler': 1,
Victor Stinner20004952019-03-26 02:31:11 +0100656 'dev_mode': 1,
Victor Stinnerf8ba6f52019-03-26 16:58:50 +0100657 'warnoptions': ['default'],
Victor Stinner1075d162019-03-25 23:19:57 +0100658 }
659 self.check_config("init_dev_mode", config, preconfig)
Victor Stinner56b29b62018-07-26 18:57:56 +0200660
661 def test_init_isolated(self):
Victor Stinnerf8ba6f52019-03-26 16:58:50 +0100662 preconfig = {}
Victor Stinner1075d162019-03-25 23:19:57 +0100663 config = {
Victor Stinner20004952019-03-26 02:31:11 +0100664 'isolated': 1,
665 'use_environment': 0,
Victor Stinner56b29b62018-07-26 18:57:56 +0200666 'user_site_directory': 0,
667 }
Victor Stinner1075d162019-03-25 23:19:57 +0100668 self.check_config("init_isolated", config, preconfig)
Victor Stinner56b29b62018-07-26 18:57:56 +0200669
Victor Stinner6da20a42019-03-27 00:26:18 +0100670 def test_preinit_isolated1(self):
671 # _PyPreConfig.isolated=1, _PyCoreConfig.isolated not set
672 preconfig = {}
673 config = {
674 'isolated': 1,
675 'use_environment': 0,
676 'user_site_directory': 0,
677 }
678 self.check_config("preinit_isolated1", config, preconfig)
679
680 def test_preinit_isolated2(self):
681 # _PyPreConfig.isolated=0, _PyCoreConfig.isolated=1
682 preconfig = {}
683 config = {
684 'isolated': 1,
685 'use_environment': 0,
686 'user_site_directory': 0,
687 }
688 self.check_config("preinit_isolated2", config, preconfig)
689
Victor Stinner91c99872019-05-14 22:01:51 +0200690 def test_init_read_set(self):
691 preconfig = {}
692 core_config = {
693 'program_name': './init_read_set',
694 'executable': 'my_executable',
695 }
696 self.check_config("init_read_set", core_config, preconfig,
697 add_path="init_read_set_path")
698
Victor Stinner9ef5dca2019-05-16 17:38:16 +0200699 def test_init_run_main(self):
Victor Stinner5eb8b072019-05-15 02:12:48 +0200700 preconfig = {}
701 code = ('import _testinternalcapi, json; '
702 'print(json.dumps(_testinternalcapi.get_configs()))')
703 core_config = {
704 'argv': ['-c', 'arg2'],
Victor Stinner5eb8b072019-05-15 02:12:48 +0200705 'program_name': './python3',
706 'run_command': code + '\n',
707 }
Victor Stinner9ef5dca2019-05-16 17:38:16 +0200708 self.check_config("init_run_main", core_config, preconfig)
709
710 def test_init_main(self):
711 preconfig = {}
712 code = ('import _testinternalcapi, json; '
713 'print(json.dumps(_testinternalcapi.get_configs()))')
714 core_config = {
715 'argv': ['-c', 'arg2'],
Victor Stinner9ef5dca2019-05-16 17:38:16 +0200716 'program_name': './python3',
717 'run_command': code + '\n',
718 '_init_main': 0,
719 }
720 self.check_config("init_main", core_config, preconfig,
721 stderr="Run Python code before _Py_InitializeMain")
Victor Stinner5eb8b072019-05-15 02:12:48 +0200722
Victor Stinnerae239f62019-05-16 17:02:56 +0200723 def test_init_dont_parse_argv(self):
724 core_config = {
725 'argv': ['-v', '-c', 'arg1', '-W', 'arg2'],
726 'parse_argv': 0,
Victor Stinnerae239f62019-05-16 17:02:56 +0200727 }
728 self.check_config("init_dont_parse_argv", core_config, {})
729
Victor Stinner56b29b62018-07-26 18:57:56 +0200730
Nick Coghlan39f0bb52017-11-28 08:11:51 +1000731if __name__ == "__main__":
732 unittest.main()