Brett Cannon | 732ac65 | 2014-05-09 14:32:57 -0400 | [diff] [blame] | 1 | import builtins |
| 2 | import contextlib |
| 3 | import errno |
| 4 | import functools |
| 5 | import importlib |
| 6 | from importlib import machinery, util, invalidate_caches |
| 7 | import os |
Brett Cannon | 23cbd8a | 2009-01-18 00:24:28 +0000 | [diff] [blame] | 8 | import os.path |
Brett Cannon | a3d056e | 2009-04-02 05:17:54 +0000 | [diff] [blame] | 9 | from test import support |
Brett Cannon | 534b2cd | 2009-02-07 02:06:43 +0000 | [diff] [blame] | 10 | import unittest |
Brett Cannon | 23cbd8a | 2009-01-18 00:24:28 +0000 | [diff] [blame] | 11 | import sys |
Brett Cannon | 732ac65 | 2014-05-09 14:32:57 -0400 | [diff] [blame] | 12 | import tempfile |
Brett Cannon | ef88802 | 2013-06-15 18:39:21 -0400 | [diff] [blame] | 13 | import types |
Brett Cannon | 23cbd8a | 2009-01-18 00:24:28 +0000 | [diff] [blame] | 14 | |
| 15 | |
Brett Cannon | 732ac65 | 2014-05-09 14:32:57 -0400 | [diff] [blame] | 16 | BUILTINS = types.SimpleNamespace() |
| 17 | BUILTINS.good_name = None |
| 18 | BUILTINS.bad_name = None |
| 19 | if 'errno' in sys.builtin_module_names: |
| 20 | BUILTINS.good_name = 'errno' |
| 21 | if 'importlib' not in sys.builtin_module_names: |
| 22 | BUILTINS.bad_name = 'importlib' |
| 23 | |
| 24 | EXTENSIONS = types.SimpleNamespace() |
| 25 | EXTENSIONS.path = None |
| 26 | EXTENSIONS.ext = None |
| 27 | EXTENSIONS.filename = None |
| 28 | EXTENSIONS.file_path = None |
| 29 | EXTENSIONS.name = '_testcapi' |
| 30 | |
| 31 | def _extension_details(): |
| 32 | global EXTENSIONS |
| 33 | for path in sys.path: |
| 34 | for ext in machinery.EXTENSION_SUFFIXES: |
| 35 | filename = EXTENSIONS.name + ext |
| 36 | file_path = os.path.join(path, filename) |
| 37 | if os.path.exists(file_path): |
| 38 | EXTENSIONS.path = path |
| 39 | EXTENSIONS.ext = ext |
| 40 | EXTENSIONS.filename = filename |
| 41 | EXTENSIONS.file_path = file_path |
| 42 | return |
| 43 | |
| 44 | _extension_details() |
| 45 | |
| 46 | |
Brett Cannon | 3ad327e | 2013-10-04 14:47:14 -0400 | [diff] [blame] | 47 | def import_importlib(module_name): |
| 48 | """Import a module from importlib both w/ and w/o _frozen_importlib.""" |
| 49 | fresh = ('importlib',) if '.' in module_name else () |
| 50 | frozen = support.import_fresh_module(module_name) |
| 51 | source = support.import_fresh_module(module_name, fresh=fresh, |
Eric Snow | 183a941 | 2015-05-15 21:54:59 -0600 | [diff] [blame] | 52 | blocked=('_frozen_importlib', '_frozen_importlib_external')) |
Eric Snow | 3497c0b | 2014-05-16 11:40:40 -0600 | [diff] [blame] | 53 | return {'Frozen': frozen, 'Source': source} |
| 54 | |
| 55 | |
| 56 | def specialize_class(cls, kind, base=None, **kwargs): |
| 57 | # XXX Support passing in submodule names--load (and cache) them? |
| 58 | # That would clean up the test modules a bit more. |
| 59 | if base is None: |
| 60 | base = unittest.TestCase |
| 61 | elif not isinstance(base, type): |
| 62 | base = base[kind] |
| 63 | name = '{}_{}'.format(kind, cls.__name__) |
| 64 | bases = (cls, base) |
| 65 | specialized = types.new_class(name, bases) |
| 66 | specialized.__module__ = cls.__module__ |
| 67 | specialized._NAME = cls.__name__ |
| 68 | specialized._KIND = kind |
| 69 | for attr, values in kwargs.items(): |
| 70 | value = values[kind] |
| 71 | setattr(specialized, attr, value) |
| 72 | return specialized |
| 73 | |
| 74 | |
| 75 | def split_frozen(cls, base=None, **kwargs): |
| 76 | frozen = specialize_class(cls, 'Frozen', base, **kwargs) |
| 77 | source = specialize_class(cls, 'Source', base, **kwargs) |
Brett Cannon | 3ad327e | 2013-10-04 14:47:14 -0400 | [diff] [blame] | 78 | return frozen, source |
| 79 | |
| 80 | |
Eric Snow | 3497c0b | 2014-05-16 11:40:40 -0600 | [diff] [blame] | 81 | def test_both(test_class, base=None, **kwargs): |
| 82 | return split_frozen(test_class, base, **kwargs) |
Brett Cannon | 1340049 | 2013-10-18 15:40:11 -0400 | [diff] [blame] | 83 | |
| 84 | |
Brett Cannon | 4dc3193 | 2009-07-20 01:05:40 +0000 | [diff] [blame] | 85 | CASE_INSENSITIVE_FS = True |
| 86 | # Windows is the only OS that is *always* case-insensitive |
| 87 | # (OS X *can* be case-sensitive). |
| 88 | if sys.platform not in ('win32', 'cygwin'): |
| 89 | changed_name = __file__.upper() |
| 90 | if changed_name == __file__: |
| 91 | changed_name = __file__.lower() |
| 92 | if not os.path.exists(changed_name): |
| 93 | CASE_INSENSITIVE_FS = False |
| 94 | |
Eric Snow | 3497c0b | 2014-05-16 11:40:40 -0600 | [diff] [blame] | 95 | source_importlib = import_importlib('importlib')['Source'] |
| 96 | __import__ = {'Frozen': staticmethod(builtins.__import__), |
| 97 | 'Source': staticmethod(source_importlib.__import__)} |
Brett Cannon | 732ac65 | 2014-05-09 14:32:57 -0400 | [diff] [blame] | 98 | |
Brett Cannon | 4dc3193 | 2009-07-20 01:05:40 +0000 | [diff] [blame] | 99 | |
| 100 | def case_insensitive_tests(test): |
Brett Cannon | 1262e7c | 2009-05-11 01:47:11 +0000 | [diff] [blame] | 101 | """Class decorator that nullifies tests requiring a case-insensitive |
Brett Cannon | 2c5c79c | 2009-01-18 06:55:05 +0000 | [diff] [blame] | 102 | file system.""" |
Brett Cannon | 4dc3193 | 2009-07-20 01:05:40 +0000 | [diff] [blame] | 103 | return unittest.skipIf(not CASE_INSENSITIVE_FS, |
| 104 | "requires a case-insensitive filesystem")(test) |
Brett Cannon | 2c5c79c | 2009-01-18 06:55:05 +0000 | [diff] [blame] | 105 | |
| 106 | |
Eric Snow | 6029e08 | 2014-01-25 15:32:46 -0700 | [diff] [blame] | 107 | def submodule(parent, name, pkg_dir, content=''): |
| 108 | path = os.path.join(pkg_dir, name + '.py') |
| 109 | with open(path, 'w') as subfile: |
| 110 | subfile.write(content) |
| 111 | return '{}.{}'.format(parent, name), path |
| 112 | |
| 113 | |
Brett Cannon | 732ac65 | 2014-05-09 14:32:57 -0400 | [diff] [blame] | 114 | @contextlib.contextmanager |
Brett Cannon | 23cbd8a | 2009-01-18 00:24:28 +0000 | [diff] [blame] | 115 | def uncache(*names): |
| 116 | """Uncache a module from sys.modules. |
| 117 | |
| 118 | A basic sanity check is performed to prevent uncaching modules that either |
| 119 | cannot/shouldn't be uncached. |
| 120 | |
| 121 | """ |
| 122 | for name in names: |
| 123 | if name in ('sys', 'marshal', 'imp'): |
| 124 | raise ValueError( |
Brett Cannon | fd07415 | 2012-04-14 14:10:13 -0400 | [diff] [blame] | 125 | "cannot uncache {0}".format(name)) |
Brett Cannon | 23cbd8a | 2009-01-18 00:24:28 +0000 | [diff] [blame] | 126 | try: |
| 127 | del sys.modules[name] |
| 128 | except KeyError: |
| 129 | pass |
| 130 | try: |
| 131 | yield |
| 132 | finally: |
| 133 | for name in names: |
| 134 | try: |
| 135 | del sys.modules[name] |
| 136 | except KeyError: |
| 137 | pass |
| 138 | |
Eric Snow | 6029e08 | 2014-01-25 15:32:46 -0700 | [diff] [blame] | 139 | |
Brett Cannon | 732ac65 | 2014-05-09 14:32:57 -0400 | [diff] [blame] | 140 | @contextlib.contextmanager |
Eric Snow | 6029e08 | 2014-01-25 15:32:46 -0700 | [diff] [blame] | 141 | def temp_module(name, content='', *, pkg=False): |
| 142 | conflicts = [n for n in sys.modules if n.partition('.')[0] == name] |
| 143 | with support.temp_cwd(None) as cwd: |
| 144 | with uncache(name, *conflicts): |
| 145 | with support.DirsOnSysPath(cwd): |
| 146 | invalidate_caches() |
| 147 | |
| 148 | location = os.path.join(cwd, name) |
| 149 | if pkg: |
| 150 | modpath = os.path.join(location, '__init__.py') |
| 151 | os.mkdir(name) |
| 152 | else: |
| 153 | modpath = location + '.py' |
| 154 | if content is None: |
| 155 | # Make sure the module file gets created. |
| 156 | content = '' |
| 157 | if content is not None: |
| 158 | # not a namespace package |
| 159 | with open(modpath, 'w') as modfile: |
| 160 | modfile.write(content) |
| 161 | yield location |
| 162 | |
| 163 | |
Brett Cannon | 732ac65 | 2014-05-09 14:32:57 -0400 | [diff] [blame] | 164 | @contextlib.contextmanager |
Brett Cannon | 23cbd8a | 2009-01-18 00:24:28 +0000 | [diff] [blame] | 165 | def import_state(**kwargs): |
| 166 | """Context manager to manage the various importers and stored state in the |
| 167 | sys module. |
| 168 | |
| 169 | The 'modules' attribute is not supported as the interpreter state stores a |
| 170 | pointer to the dict that the interpreter uses internally; |
| 171 | reassigning to sys.modules does not have the desired effect. |
| 172 | |
| 173 | """ |
| 174 | originals = {} |
| 175 | try: |
| 176 | for attr, default in (('meta_path', []), ('path', []), |
| 177 | ('path_hooks', []), |
| 178 | ('path_importer_cache', {})): |
| 179 | originals[attr] = getattr(sys, attr) |
| 180 | if attr in kwargs: |
| 181 | new_value = kwargs[attr] |
| 182 | del kwargs[attr] |
| 183 | else: |
| 184 | new_value = default |
| 185 | setattr(sys, attr, new_value) |
| 186 | if len(kwargs): |
| 187 | raise ValueError( |
| 188 | 'unrecognized arguments: {0}'.format(kwargs.keys())) |
| 189 | yield |
| 190 | finally: |
| 191 | for attr, value in originals.items(): |
| 192 | setattr(sys, attr, value) |
| 193 | |
| 194 | |
Brett Cannon | 86aae6a | 2013-12-06 12:07:25 -0500 | [diff] [blame] | 195 | class _ImporterMock: |
Brett Cannon | 23cbd8a | 2009-01-18 00:24:28 +0000 | [diff] [blame] | 196 | |
Brett Cannon | 86aae6a | 2013-12-06 12:07:25 -0500 | [diff] [blame] | 197 | """Base class to help with creating importer mocks.""" |
Brett Cannon | 23cbd8a | 2009-01-18 00:24:28 +0000 | [diff] [blame] | 198 | |
Meador Inge | 416f12d | 2011-12-14 22:23:46 -0600 | [diff] [blame] | 199 | def __init__(self, *names, module_code={}): |
Brett Cannon | 23cbd8a | 2009-01-18 00:24:28 +0000 | [diff] [blame] | 200 | self.modules = {} |
Meador Inge | 416f12d | 2011-12-14 22:23:46 -0600 | [diff] [blame] | 201 | self.module_code = {} |
Brett Cannon | 23cbd8a | 2009-01-18 00:24:28 +0000 | [diff] [blame] | 202 | for name in names: |
| 203 | if not name.endswith('.__init__'): |
| 204 | import_name = name |
| 205 | else: |
| 206 | import_name = name[:-len('.__init__')] |
| 207 | if '.' not in name: |
| 208 | package = None |
| 209 | elif import_name == name: |
| 210 | package = name.rsplit('.', 1)[0] |
| 211 | else: |
| 212 | package = import_name |
Brett Cannon | ef88802 | 2013-06-15 18:39:21 -0400 | [diff] [blame] | 213 | module = types.ModuleType(import_name) |
Brett Cannon | 23cbd8a | 2009-01-18 00:24:28 +0000 | [diff] [blame] | 214 | module.__loader__ = self |
| 215 | module.__file__ = '<mock __file__>' |
| 216 | module.__package__ = package |
| 217 | module.attr = name |
| 218 | if import_name != name: |
| 219 | module.__path__ = ['<mock __path__>'] |
| 220 | self.modules[import_name] = module |
Meador Inge | 416f12d | 2011-12-14 22:23:46 -0600 | [diff] [blame] | 221 | if import_name in module_code: |
| 222 | self.module_code[import_name] = module_code[import_name] |
Brett Cannon | 23cbd8a | 2009-01-18 00:24:28 +0000 | [diff] [blame] | 223 | |
| 224 | def __getitem__(self, name): |
| 225 | return self.modules[name] |
| 226 | |
Brett Cannon | 86aae6a | 2013-12-06 12:07:25 -0500 | [diff] [blame] | 227 | def __enter__(self): |
| 228 | self._uncache = uncache(*self.modules.keys()) |
| 229 | self._uncache.__enter__() |
| 230 | return self |
| 231 | |
| 232 | def __exit__(self, *exc_info): |
| 233 | self._uncache.__exit__(None, None, None) |
| 234 | |
| 235 | |
| 236 | class mock_modules(_ImporterMock): |
| 237 | |
| 238 | """Importer mock using PEP 302 APIs.""" |
| 239 | |
Brett Cannon | 23cbd8a | 2009-01-18 00:24:28 +0000 | [diff] [blame] | 240 | def find_module(self, fullname, path=None): |
| 241 | if fullname not in self.modules: |
| 242 | return None |
| 243 | else: |
| 244 | return self |
| 245 | |
| 246 | def load_module(self, fullname): |
| 247 | if fullname not in self.modules: |
| 248 | raise ImportError |
| 249 | else: |
| 250 | sys.modules[fullname] = self.modules[fullname] |
Meador Inge | 416f12d | 2011-12-14 22:23:46 -0600 | [diff] [blame] | 251 | if fullname in self.module_code: |
Antoine Pitrou | 6efa50a | 2012-05-07 21:41:59 +0200 | [diff] [blame] | 252 | try: |
| 253 | self.module_code[fullname]() |
| 254 | except Exception: |
| 255 | del sys.modules[fullname] |
| 256 | raise |
Brett Cannon | 23cbd8a | 2009-01-18 00:24:28 +0000 | [diff] [blame] | 257 | return self.modules[fullname] |
| 258 | |
Brett Cannon | 732ac65 | 2014-05-09 14:32:57 -0400 | [diff] [blame] | 259 | |
Brett Cannon | 86aae6a | 2013-12-06 12:07:25 -0500 | [diff] [blame] | 260 | class mock_spec(_ImporterMock): |
Brett Cannon | 23cbd8a | 2009-01-18 00:24:28 +0000 | [diff] [blame] | 261 | |
Brett Cannon | 86aae6a | 2013-12-06 12:07:25 -0500 | [diff] [blame] | 262 | """Importer mock using PEP 451 APIs.""" |
| 263 | |
| 264 | def find_spec(self, fullname, path=None, parent=None): |
| 265 | try: |
| 266 | module = self.modules[fullname] |
| 267 | except KeyError: |
| 268 | return None |
| 269 | is_package = hasattr(module, '__path__') |
| 270 | spec = util.spec_from_file_location( |
| 271 | fullname, module.__file__, loader=self, |
| 272 | submodule_search_locations=getattr(module, '__path__', None)) |
| 273 | return spec |
| 274 | |
| 275 | def create_module(self, spec): |
| 276 | if spec.name not in self.modules: |
| 277 | raise ImportError |
| 278 | return self.modules[spec.name] |
| 279 | |
| 280 | def exec_module(self, module): |
| 281 | try: |
| 282 | self.module_code[module.__spec__.name]() |
| 283 | except KeyError: |
| 284 | pass |
Brett Cannon | 732ac65 | 2014-05-09 14:32:57 -0400 | [diff] [blame] | 285 | |
| 286 | |
| 287 | def writes_bytecode_files(fxn): |
| 288 | """Decorator to protect sys.dont_write_bytecode from mutation and to skip |
| 289 | tests that require it to be set to False.""" |
| 290 | if sys.dont_write_bytecode: |
| 291 | return lambda *args, **kwargs: None |
| 292 | @functools.wraps(fxn) |
| 293 | def wrapper(*args, **kwargs): |
| 294 | original = sys.dont_write_bytecode |
| 295 | sys.dont_write_bytecode = False |
| 296 | try: |
| 297 | to_return = fxn(*args, **kwargs) |
| 298 | finally: |
| 299 | sys.dont_write_bytecode = original |
| 300 | return to_return |
| 301 | return wrapper |
| 302 | |
| 303 | |
| 304 | def ensure_bytecode_path(bytecode_path): |
| 305 | """Ensure that the __pycache__ directory for PEP 3147 pyc file exists. |
| 306 | |
| 307 | :param bytecode_path: File system path to PEP 3147 pyc file. |
| 308 | """ |
| 309 | try: |
| 310 | os.mkdir(os.path.dirname(bytecode_path)) |
| 311 | except OSError as error: |
| 312 | if error.errno != errno.EEXIST: |
| 313 | raise |
| 314 | |
| 315 | |
| 316 | @contextlib.contextmanager |
| 317 | def create_modules(*names): |
| 318 | """Temporarily create each named module with an attribute (named 'attr') |
| 319 | that contains the name passed into the context manager that caused the |
| 320 | creation of the module. |
| 321 | |
| 322 | All files are created in a temporary directory returned by |
| 323 | tempfile.mkdtemp(). This directory is inserted at the beginning of |
| 324 | sys.path. When the context manager exits all created files (source and |
| 325 | bytecode) are explicitly deleted. |
| 326 | |
| 327 | No magic is performed when creating packages! This means that if you create |
| 328 | a module within a package you must also create the package's __init__ as |
| 329 | well. |
| 330 | |
| 331 | """ |
| 332 | source = 'attr = {0!r}' |
| 333 | created_paths = [] |
| 334 | mapping = {} |
| 335 | state_manager = None |
| 336 | uncache_manager = None |
| 337 | try: |
| 338 | temp_dir = tempfile.mkdtemp() |
| 339 | mapping['.root'] = temp_dir |
| 340 | import_names = set() |
| 341 | for name in names: |
| 342 | if not name.endswith('__init__'): |
| 343 | import_name = name |
| 344 | else: |
| 345 | import_name = name[:-len('.__init__')] |
| 346 | import_names.add(import_name) |
| 347 | if import_name in sys.modules: |
| 348 | del sys.modules[import_name] |
| 349 | name_parts = name.split('.') |
| 350 | file_path = temp_dir |
| 351 | for directory in name_parts[:-1]: |
| 352 | file_path = os.path.join(file_path, directory) |
| 353 | if not os.path.exists(file_path): |
| 354 | os.mkdir(file_path) |
| 355 | created_paths.append(file_path) |
| 356 | file_path = os.path.join(file_path, name_parts[-1] + '.py') |
| 357 | with open(file_path, 'w') as file: |
| 358 | file.write(source.format(name)) |
| 359 | created_paths.append(file_path) |
| 360 | mapping[name] = file_path |
| 361 | uncache_manager = uncache(*import_names) |
| 362 | uncache_manager.__enter__() |
| 363 | state_manager = import_state(path=[temp_dir]) |
| 364 | state_manager.__enter__() |
| 365 | yield mapping |
| 366 | finally: |
| 367 | if state_manager is not None: |
| 368 | state_manager.__exit__(None, None, None) |
| 369 | if uncache_manager is not None: |
| 370 | uncache_manager.__exit__(None, None, None) |
| 371 | support.rmtree(temp_dir) |
| 372 | |
| 373 | |
| 374 | def mock_path_hook(*entries, importer): |
| 375 | """A mock sys.path_hooks entry.""" |
| 376 | def hook(entry): |
| 377 | if entry not in entries: |
| 378 | raise ImportError |
| 379 | return importer |
| 380 | return hook |