Nick Coghlan | 9a76735 | 2013-12-17 22:17:26 +1000 | [diff] [blame] | 1 | # tests __main__ module handling in multiprocessing |
Terry Jan Reedy | bc7c96b | 2014-06-13 14:23:43 -0400 | [diff] [blame] | 2 | from test import support |
| 3 | # Skip tests if _thread or _multiprocessing wasn't built. |
| 4 | support.import_module('_thread') |
| 5 | support.import_module('_multiprocessing') |
Nick Coghlan | 9a76735 | 2013-12-17 22:17:26 +1000 | [diff] [blame] | 6 | |
| 7 | import importlib |
| 8 | import importlib.machinery |
| 9 | import zipimport |
| 10 | import unittest |
| 11 | import sys |
| 12 | import os |
| 13 | import os.path |
| 14 | import py_compile |
| 15 | |
Berker Peksag | ce64391 | 2015-05-06 06:33:17 +0300 | [diff] [blame] | 16 | from test.support.script_helper import ( |
Nick Coghlan | 9a76735 | 2013-12-17 22:17:26 +1000 | [diff] [blame] | 17 | make_pkg, make_script, make_zip_pkg, make_zip_script, |
Berker Peksag | ce64391 | 2015-05-06 06:33:17 +0300 | [diff] [blame] | 18 | assert_python_ok, assert_python_failure, spawn_python, kill_python) |
Nick Coghlan | 9a76735 | 2013-12-17 22:17:26 +1000 | [diff] [blame] | 19 | |
Nick Coghlan | 0538529 | 2013-12-20 22:14:03 +1000 | [diff] [blame] | 20 | # Look up which start methods are available to test |
| 21 | import multiprocessing |
| 22 | AVAILABLE_START_METHODS = set(multiprocessing.get_all_start_methods()) |
Nick Coghlan | 9a76735 | 2013-12-17 22:17:26 +1000 | [diff] [blame] | 23 | |
Victor Stinner | 2bb8a08 | 2014-09-03 23:48:08 +0200 | [diff] [blame] | 24 | # Issue #22332: Skip tests if sem_open implementation is broken. |
| 25 | support.import_module('multiprocessing.synchronize') |
| 26 | |
Nick Coghlan | 9a76735 | 2013-12-17 22:17:26 +1000 | [diff] [blame] | 27 | verbose = support.verbose |
| 28 | |
| 29 | test_source = """\ |
| 30 | # multiprocessing includes all sorts of shenanigans to make __main__ |
| 31 | # attributes accessible in the subprocess in a pickle compatible way. |
| 32 | |
| 33 | # We run the "doesn't work in the interactive interpreter" example from |
| 34 | # the docs to make sure it *does* work from an executed __main__, |
| 35 | # regardless of the invocation mechanism |
| 36 | |
| 37 | import sys |
| 38 | import time |
| 39 | from multiprocessing import Pool, set_start_method |
| 40 | |
| 41 | # We use this __main__ defined function in the map call below in order to |
| 42 | # check that multiprocessing in correctly running the unguarded |
| 43 | # code in child processes and then making it available as __main__ |
| 44 | def f(x): |
| 45 | return x*x |
| 46 | |
| 47 | # Check explicit relative imports |
| 48 | if "check_sibling" in __file__: |
| 49 | # We're inside a package and not in a __main__.py file |
| 50 | # so make sure explicit relative imports work correctly |
| 51 | from . import sibling |
| 52 | |
| 53 | if __name__ == '__main__': |
| 54 | start_method = sys.argv[1] |
| 55 | set_start_method(start_method) |
| 56 | p = Pool(5) |
| 57 | results = [] |
| 58 | p.map_async(f, [1, 2, 3], callback=results.extend) |
Nick Coghlan | 23f597e | 2013-12-23 18:17:20 +1000 | [diff] [blame] | 59 | deadline = time.time() + 10 # up to 10 s to report the results |
Nick Coghlan | 9a76735 | 2013-12-17 22:17:26 +1000 | [diff] [blame] | 60 | while not results: |
| 61 | time.sleep(0.05) |
| 62 | if time.time() > deadline: |
| 63 | raise RuntimeError("Timed out waiting for results") |
| 64 | results.sort() |
| 65 | print(start_method, "->", results) |
| 66 | """ |
| 67 | |
| 68 | test_source_main_skipped_in_children = """\ |
| 69 | # __main__.py files have an implied "if __name__ == '__main__'" so |
| 70 | # multiprocessing should always skip running them in child processes |
| 71 | |
| 72 | # This means we can't use __main__ defined functions in child processes, |
| 73 | # so we just use "int" as a passthrough operation below |
| 74 | |
| 75 | if __name__ != "__main__": |
| 76 | raise RuntimeError("Should only be called as __main__!") |
| 77 | |
| 78 | import sys |
| 79 | import time |
| 80 | from multiprocessing import Pool, set_start_method |
| 81 | |
| 82 | start_method = sys.argv[1] |
| 83 | set_start_method(start_method) |
| 84 | p = Pool(5) |
| 85 | results = [] |
| 86 | p.map_async(int, [1, 4, 9], callback=results.extend) |
Nick Coghlan | 23f597e | 2013-12-23 18:17:20 +1000 | [diff] [blame] | 87 | deadline = time.time() + 10 # up to 10 s to report the results |
Nick Coghlan | 9a76735 | 2013-12-17 22:17:26 +1000 | [diff] [blame] | 88 | while not results: |
| 89 | time.sleep(0.05) |
| 90 | if time.time() > deadline: |
| 91 | raise RuntimeError("Timed out waiting for results") |
| 92 | results.sort() |
| 93 | print(start_method, "->", results) |
| 94 | """ |
| 95 | |
| 96 | # These helpers were copied from test_cmd_line_script & tweaked a bit... |
| 97 | |
| 98 | def _make_test_script(script_dir, script_basename, |
| 99 | source=test_source, omit_suffix=False): |
| 100 | to_return = make_script(script_dir, script_basename, |
| 101 | source, omit_suffix) |
| 102 | # Hack to check explicit relative imports |
| 103 | if script_basename == "check_sibling": |
| 104 | make_script(script_dir, "sibling", "") |
| 105 | importlib.invalidate_caches() |
| 106 | return to_return |
| 107 | |
| 108 | def _make_test_zip_pkg(zip_dir, zip_basename, pkg_name, script_basename, |
| 109 | source=test_source, depth=1): |
| 110 | to_return = make_zip_pkg(zip_dir, zip_basename, pkg_name, script_basename, |
| 111 | source, depth) |
| 112 | importlib.invalidate_caches() |
| 113 | return to_return |
| 114 | |
| 115 | # There's no easy way to pass the script directory in to get |
| 116 | # -m to work (avoiding that is the whole point of making |
| 117 | # directories and zipfiles executable!) |
| 118 | # So we fake it for testing purposes with a custom launch script |
| 119 | launch_source = """\ |
| 120 | import sys, os.path, runpy |
| 121 | sys.path.insert(0, %s) |
| 122 | runpy._run_module_as_main(%r) |
| 123 | """ |
| 124 | |
| 125 | def _make_launch_script(script_dir, script_basename, module_name, path=None): |
| 126 | if path is None: |
| 127 | path = "os.path.dirname(__file__)" |
| 128 | else: |
| 129 | path = repr(path) |
| 130 | source = launch_source % (path, module_name) |
| 131 | to_return = make_script(script_dir, script_basename, source) |
| 132 | importlib.invalidate_caches() |
| 133 | return to_return |
| 134 | |
| 135 | class MultiProcessingCmdLineMixin(): |
| 136 | maxDiff = None # Show full tracebacks on subprocess failure |
| 137 | |
Nick Coghlan | feae73e | 2013-12-19 21:53:31 +1000 | [diff] [blame] | 138 | def setUp(self): |
Nick Coghlan | 0538529 | 2013-12-20 22:14:03 +1000 | [diff] [blame] | 139 | if self.start_method not in AVAILABLE_START_METHODS: |
Nick Coghlan | feae73e | 2013-12-19 21:53:31 +1000 | [diff] [blame] | 140 | self.skipTest("%r start method not available" % self.start_method) |
Nick Coghlan | 9a76735 | 2013-12-17 22:17:26 +1000 | [diff] [blame] | 141 | |
| 142 | def _check_output(self, script_name, exit_code, out, err): |
| 143 | if verbose > 1: |
| 144 | print("Output from test script %r:" % script_name) |
Serhiy Storchaka | b0749ca | 2015-03-25 01:33:19 +0200 | [diff] [blame] | 145 | print(repr(out)) |
Nick Coghlan | 9a76735 | 2013-12-17 22:17:26 +1000 | [diff] [blame] | 146 | self.assertEqual(exit_code, 0) |
| 147 | self.assertEqual(err.decode('utf-8'), '') |
| 148 | expected_results = "%s -> [1, 4, 9]" % self.start_method |
| 149 | self.assertEqual(out.decode('utf-8').strip(), expected_results) |
| 150 | |
| 151 | def _check_script(self, script_name, *cmd_line_switches): |
| 152 | if not __debug__: |
| 153 | cmd_line_switches += ('-' + 'O' * sys.flags.optimize,) |
| 154 | run_args = cmd_line_switches + (script_name, self.start_method) |
| 155 | rc, out, err = assert_python_ok(*run_args, __isolated=False) |
| 156 | self._check_output(script_name, rc, out, err) |
| 157 | |
| 158 | def test_basic_script(self): |
Berker Peksag | ce64391 | 2015-05-06 06:33:17 +0300 | [diff] [blame] | 159 | with support.temp_dir() as script_dir: |
Nick Coghlan | 9a76735 | 2013-12-17 22:17:26 +1000 | [diff] [blame] | 160 | script_name = _make_test_script(script_dir, 'script') |
| 161 | self._check_script(script_name) |
| 162 | |
| 163 | def test_basic_script_no_suffix(self): |
Berker Peksag | ce64391 | 2015-05-06 06:33:17 +0300 | [diff] [blame] | 164 | with support.temp_dir() as script_dir: |
Nick Coghlan | 9a76735 | 2013-12-17 22:17:26 +1000 | [diff] [blame] | 165 | script_name = _make_test_script(script_dir, 'script', |
| 166 | omit_suffix=True) |
| 167 | self._check_script(script_name) |
| 168 | |
| 169 | def test_ipython_workaround(self): |
| 170 | # Some versions of the IPython launch script are missing the |
| 171 | # __name__ = "__main__" guard, and multiprocessing has long had |
| 172 | # a workaround for that case |
| 173 | # See https://github.com/ipython/ipython/issues/4698 |
| 174 | source = test_source_main_skipped_in_children |
Berker Peksag | ce64391 | 2015-05-06 06:33:17 +0300 | [diff] [blame] | 175 | with support.temp_dir() as script_dir: |
Nick Coghlan | 9a76735 | 2013-12-17 22:17:26 +1000 | [diff] [blame] | 176 | script_name = _make_test_script(script_dir, 'ipython', |
| 177 | source=source) |
| 178 | self._check_script(script_name) |
| 179 | script_no_suffix = _make_test_script(script_dir, 'ipython', |
| 180 | source=source, |
| 181 | omit_suffix=True) |
| 182 | self._check_script(script_no_suffix) |
| 183 | |
| 184 | def test_script_compiled(self): |
Berker Peksag | ce64391 | 2015-05-06 06:33:17 +0300 | [diff] [blame] | 185 | with support.temp_dir() as script_dir: |
Nick Coghlan | 9a76735 | 2013-12-17 22:17:26 +1000 | [diff] [blame] | 186 | script_name = _make_test_script(script_dir, 'script') |
| 187 | py_compile.compile(script_name, doraise=True) |
| 188 | os.remove(script_name) |
| 189 | pyc_file = support.make_legacy_pyc(script_name) |
| 190 | self._check_script(pyc_file) |
| 191 | |
| 192 | def test_directory(self): |
| 193 | source = self.main_in_children_source |
Berker Peksag | ce64391 | 2015-05-06 06:33:17 +0300 | [diff] [blame] | 194 | with support.temp_dir() as script_dir: |
Nick Coghlan | 9a76735 | 2013-12-17 22:17:26 +1000 | [diff] [blame] | 195 | script_name = _make_test_script(script_dir, '__main__', |
| 196 | source=source) |
| 197 | self._check_script(script_dir) |
| 198 | |
| 199 | def test_directory_compiled(self): |
| 200 | source = self.main_in_children_source |
Berker Peksag | ce64391 | 2015-05-06 06:33:17 +0300 | [diff] [blame] | 201 | with support.temp_dir() as script_dir: |
Nick Coghlan | 9a76735 | 2013-12-17 22:17:26 +1000 | [diff] [blame] | 202 | script_name = _make_test_script(script_dir, '__main__', |
| 203 | source=source) |
| 204 | py_compile.compile(script_name, doraise=True) |
| 205 | os.remove(script_name) |
| 206 | pyc_file = support.make_legacy_pyc(script_name) |
| 207 | self._check_script(script_dir) |
| 208 | |
| 209 | def test_zipfile(self): |
| 210 | source = self.main_in_children_source |
Berker Peksag | ce64391 | 2015-05-06 06:33:17 +0300 | [diff] [blame] | 211 | with support.temp_dir() as script_dir: |
Nick Coghlan | 9a76735 | 2013-12-17 22:17:26 +1000 | [diff] [blame] | 212 | script_name = _make_test_script(script_dir, '__main__', |
| 213 | source=source) |
| 214 | zip_name, run_name = make_zip_script(script_dir, 'test_zip', script_name) |
| 215 | self._check_script(zip_name) |
| 216 | |
| 217 | def test_zipfile_compiled(self): |
| 218 | source = self.main_in_children_source |
Berker Peksag | ce64391 | 2015-05-06 06:33:17 +0300 | [diff] [blame] | 219 | with support.temp_dir() as script_dir: |
Nick Coghlan | 9a76735 | 2013-12-17 22:17:26 +1000 | [diff] [blame] | 220 | script_name = _make_test_script(script_dir, '__main__', |
| 221 | source=source) |
| 222 | compiled_name = py_compile.compile(script_name, doraise=True) |
| 223 | zip_name, run_name = make_zip_script(script_dir, 'test_zip', compiled_name) |
| 224 | self._check_script(zip_name) |
| 225 | |
| 226 | def test_module_in_package(self): |
Berker Peksag | ce64391 | 2015-05-06 06:33:17 +0300 | [diff] [blame] | 227 | with support.temp_dir() as script_dir: |
Nick Coghlan | 9a76735 | 2013-12-17 22:17:26 +1000 | [diff] [blame] | 228 | pkg_dir = os.path.join(script_dir, 'test_pkg') |
| 229 | make_pkg(pkg_dir) |
| 230 | script_name = _make_test_script(pkg_dir, 'check_sibling') |
| 231 | launch_name = _make_launch_script(script_dir, 'launch', |
| 232 | 'test_pkg.check_sibling') |
| 233 | self._check_script(launch_name) |
| 234 | |
| 235 | def test_module_in_package_in_zipfile(self): |
Berker Peksag | ce64391 | 2015-05-06 06:33:17 +0300 | [diff] [blame] | 236 | with support.temp_dir() as script_dir: |
Nick Coghlan | 9a76735 | 2013-12-17 22:17:26 +1000 | [diff] [blame] | 237 | zip_name, run_name = _make_test_zip_pkg(script_dir, 'test_zip', 'test_pkg', 'script') |
| 238 | launch_name = _make_launch_script(script_dir, 'launch', 'test_pkg.script', zip_name) |
| 239 | self._check_script(launch_name) |
| 240 | |
| 241 | def test_module_in_subpackage_in_zipfile(self): |
Berker Peksag | ce64391 | 2015-05-06 06:33:17 +0300 | [diff] [blame] | 242 | with support.temp_dir() as script_dir: |
Nick Coghlan | 9a76735 | 2013-12-17 22:17:26 +1000 | [diff] [blame] | 243 | zip_name, run_name = _make_test_zip_pkg(script_dir, 'test_zip', 'test_pkg', 'script', depth=2) |
| 244 | launch_name = _make_launch_script(script_dir, 'launch', 'test_pkg.test_pkg.script', zip_name) |
| 245 | self._check_script(launch_name) |
| 246 | |
| 247 | def test_package(self): |
| 248 | source = self.main_in_children_source |
Berker Peksag | ce64391 | 2015-05-06 06:33:17 +0300 | [diff] [blame] | 249 | with support.temp_dir() as script_dir: |
Nick Coghlan | 9a76735 | 2013-12-17 22:17:26 +1000 | [diff] [blame] | 250 | pkg_dir = os.path.join(script_dir, 'test_pkg') |
| 251 | make_pkg(pkg_dir) |
| 252 | script_name = _make_test_script(pkg_dir, '__main__', |
| 253 | source=source) |
| 254 | launch_name = _make_launch_script(script_dir, 'launch', 'test_pkg') |
| 255 | self._check_script(launch_name) |
| 256 | |
| 257 | def test_package_compiled(self): |
| 258 | source = self.main_in_children_source |
Berker Peksag | ce64391 | 2015-05-06 06:33:17 +0300 | [diff] [blame] | 259 | with support.temp_dir() as script_dir: |
Nick Coghlan | 9a76735 | 2013-12-17 22:17:26 +1000 | [diff] [blame] | 260 | pkg_dir = os.path.join(script_dir, 'test_pkg') |
| 261 | make_pkg(pkg_dir) |
| 262 | script_name = _make_test_script(pkg_dir, '__main__', |
| 263 | source=source) |
| 264 | compiled_name = py_compile.compile(script_name, doraise=True) |
| 265 | os.remove(script_name) |
| 266 | pyc_file = support.make_legacy_pyc(script_name) |
| 267 | launch_name = _make_launch_script(script_dir, 'launch', 'test_pkg') |
| 268 | self._check_script(launch_name) |
| 269 | |
| 270 | # Test all supported start methods (setupClass skips as appropriate) |
| 271 | |
| 272 | class SpawnCmdLineTest(MultiProcessingCmdLineMixin, unittest.TestCase): |
| 273 | start_method = 'spawn' |
| 274 | main_in_children_source = test_source_main_skipped_in_children |
| 275 | |
| 276 | class ForkCmdLineTest(MultiProcessingCmdLineMixin, unittest.TestCase): |
| 277 | start_method = 'fork' |
| 278 | main_in_children_source = test_source |
| 279 | |
| 280 | class ForkServerCmdLineTest(MultiProcessingCmdLineMixin, unittest.TestCase): |
| 281 | start_method = 'forkserver' |
| 282 | main_in_children_source = test_source_main_skipped_in_children |
| 283 | |
Nick Coghlan | feae73e | 2013-12-19 21:53:31 +1000 | [diff] [blame] | 284 | def tearDownModule(): |
Nick Coghlan | 9a76735 | 2013-12-17 22:17:26 +1000 | [diff] [blame] | 285 | support.reap_children() |
| 286 | |
| 287 | if __name__ == '__main__': |
Nick Coghlan | feae73e | 2013-12-19 21:53:31 +1000 | [diff] [blame] | 288 | unittest.main() |