blob: 420ecc83439aac24a2e85b8ae08315529a2e01ec [file] [log] [blame]
Eric Snow32439d62015-05-02 19:15:18 -06001"""Core implementation of path-based import.
2
3This module is NOT meant to be directly imported! It has been designed such
4that it can be bootstrapped into Python as the implementation of import. As
5such it requires the injection of specific modules and attributes in order to
6work. One should use importlib as the public-facing version of this module.
7
8"""
Barry Warsaw5ec0fee2018-01-15 15:07:11 -08009# IMPORTANT: Whenever making changes to this module, be sure to run a top-level
10# `make regen-importlib` followed by `make` in order to get the frozen version
11# of the module updated. Not doing so will result in the Makefile to fail for
12# all others who don't have a ./python around to freeze the module in the early
13# stages of compilation.
Eric Snow32439d62015-05-02 19:15:18 -060014#
15
16# See importlib._setup() for what is injected into the global namespace.
17
18# When editing this code be aware that code executed at import time CANNOT
19# reference any injected objects! This includes not only global code but also
20# anything specified at the class level.
21
22# Bootstrap-related code ######################################################
Brett Cannon7ca63cb2016-07-16 10:44:13 -070023_CASE_INSENSITIVE_PLATFORMS_STR_KEY = 'win',
24_CASE_INSENSITIVE_PLATFORMS_BYTES_KEY = 'cygwin', 'darwin'
25_CASE_INSENSITIVE_PLATFORMS = (_CASE_INSENSITIVE_PLATFORMS_BYTES_KEY
26 + _CASE_INSENSITIVE_PLATFORMS_STR_KEY)
Eric Snow32439d62015-05-02 19:15:18 -060027
28
29def _make_relax_case():
30 if sys.platform.startswith(_CASE_INSENSITIVE_PLATFORMS):
Brett Cannon7ca63cb2016-07-16 10:44:13 -070031 if sys.platform.startswith(_CASE_INSENSITIVE_PLATFORMS_STR_KEY):
32 key = 'PYTHONCASEOK'
33 else:
34 key = b'PYTHONCASEOK'
35
Eric Snow32439d62015-05-02 19:15:18 -060036 def _relax_case():
37 """True if filenames must be checked case-insensitively."""
Brett Cannon7ca63cb2016-07-16 10:44:13 -070038 return key in _os.environ
Eric Snow32439d62015-05-02 19:15:18 -060039 else:
40 def _relax_case():
41 """True if filenames must be checked case-insensitively."""
42 return False
43 return _relax_case
44
45
46def _w_long(x):
47 """Convert a 32-bit integer to little-endian."""
48 return (int(x) & 0xFFFFFFFF).to_bytes(4, 'little')
49
50
51def _r_long(int_bytes):
52 """Convert 4 bytes in little-endian to an integer."""
53 return int.from_bytes(int_bytes, 'little')
54
55
56def _path_join(*path_parts):
57 """Replacement for os.path.join()."""
58 return path_sep.join([part.rstrip(path_separators)
59 for part in path_parts if part])
60
61
62def _path_split(path):
63 """Replacement for os.path.split()."""
64 if len(path_separators) == 1:
65 front, _, tail = path.rpartition(path_sep)
66 return front, tail
67 for x in reversed(path):
68 if x in path_separators:
69 front, tail = path.rsplit(x, maxsplit=1)
70 return front, tail
71 return '', path
72
73
74def _path_stat(path):
75 """Stat the path.
76
77 Made a separate function to make it easier to override in experiments
78 (e.g. cache stat results).
79
80 """
81 return _os.stat(path)
82
83
84def _path_is_mode_type(path, mode):
85 """Test whether the path is the specified mode type."""
86 try:
87 stat_info = _path_stat(path)
88 except OSError:
89 return False
90 return (stat_info.st_mode & 0o170000) == mode
91
92
93def _path_isfile(path):
94 """Replacement for os.path.isfile."""
95 return _path_is_mode_type(path, 0o100000)
96
97
98def _path_isdir(path):
99 """Replacement for os.path.isdir."""
100 if not path:
101 path = _os.getcwd()
102 return _path_is_mode_type(path, 0o040000)
103
104
105def _write_atomic(path, data, mode=0o666):
106 """Best-effort function to write data to a path atomically.
107 Be prepared to handle a FileExistsError if concurrent writing of the
108 temporary file is attempted."""
109 # id() is used to generate a pseudo-random filename.
110 path_tmp = '{}.{}'.format(path, id(path))
111 fd = _os.open(path_tmp,
112 _os.O_EXCL | _os.O_CREAT | _os.O_WRONLY, mode & 0o666)
113 try:
114 # We first write data to a temporary file, and then use os.replace() to
115 # perform an atomic rename.
116 with _io.FileIO(fd, 'wb') as file:
117 file.write(data)
118 _os.replace(path_tmp, path)
119 except OSError:
120 try:
121 _os.unlink(path_tmp)
122 except OSError:
123 pass
124 raise
125
126
127_code_type = type(_write_atomic.__code__)
128
129
130# Finder/loader utility code ###############################################
131
132# Magic word to reject .pyc files generated by other Python versions.
133# It should change for each incompatible change to the bytecode.
134#
135# The value of CR and LF is incorporated so if you ever read or write
136# a .pyc file in text mode the magic number will be wrong; also, the
137# Apple MPW compiler swaps their values, botching string constants.
138#
Eric Snow32439d62015-05-02 19:15:18 -0600139# There were a variety of old schemes for setting the magic number.
140# The current working scheme is to increment the previous value by
141# 10.
142#
143# Starting with the adoption of PEP 3147 in Python 3.2, every bump in magic
144# number also includes a new "magic tag", i.e. a human readable string used
145# to represent the magic number in __pycache__ directories. When you change
146# the magic number, you must also set a new unique magic tag. Generally this
147# can be named after the Python major version of the magic number bump, but
148# it can really be anything, as long as it's different than anything else
149# that's come before. The tags are included in the following table, starting
150# with Python 3.2a0.
151#
152# Known values:
153# Python 1.5: 20121
154# Python 1.5.1: 20121
155# Python 1.5.2: 20121
156# Python 1.6: 50428
157# Python 2.0: 50823
158# Python 2.0.1: 50823
159# Python 2.1: 60202
160# Python 2.1.1: 60202
161# Python 2.1.2: 60202
162# Python 2.2: 60717
163# Python 2.3a0: 62011
164# Python 2.3a0: 62021
165# Python 2.3a0: 62011 (!)
166# Python 2.4a0: 62041
167# Python 2.4a3: 62051
168# Python 2.4b1: 62061
169# Python 2.5a0: 62071
170# Python 2.5a0: 62081 (ast-branch)
171# Python 2.5a0: 62091 (with)
172# Python 2.5a0: 62092 (changed WITH_CLEANUP opcode)
173# Python 2.5b3: 62101 (fix wrong code: for x, in ...)
174# Python 2.5b3: 62111 (fix wrong code: x += yield)
175# Python 2.5c1: 62121 (fix wrong lnotab with for loops and
176# storing constants that should have been removed)
177# Python 2.5c2: 62131 (fix wrong code: for x, in ... in listcomp/genexp)
178# Python 2.6a0: 62151 (peephole optimizations and STORE_MAP opcode)
179# Python 2.6a1: 62161 (WITH_CLEANUP optimization)
180# Python 2.7a0: 62171 (optimize list comprehensions/change LIST_APPEND)
181# Python 2.7a0: 62181 (optimize conditional branches:
182# introduce POP_JUMP_IF_FALSE and POP_JUMP_IF_TRUE)
183# Python 2.7a0 62191 (introduce SETUP_WITH)
184# Python 2.7a0 62201 (introduce BUILD_SET)
185# Python 2.7a0 62211 (introduce MAP_ADD and SET_ADD)
186# Python 3000: 3000
187# 3010 (removed UNARY_CONVERT)
188# 3020 (added BUILD_SET)
189# 3030 (added keyword-only parameters)
190# 3040 (added signature annotations)
191# 3050 (print becomes a function)
192# 3060 (PEP 3115 metaclass syntax)
193# 3061 (string literals become unicode)
194# 3071 (PEP 3109 raise changes)
195# 3081 (PEP 3137 make __file__ and __name__ unicode)
196# 3091 (kill str8 interning)
197# 3101 (merge from 2.6a0, see 62151)
198# 3103 (__file__ points to source file)
199# Python 3.0a4: 3111 (WITH_CLEANUP optimization).
Serhiy Storchaka4af8fd52018-02-22 22:26:23 +0200200# Python 3.0b1: 3131 (lexical exception stacking, including POP_EXCEPT
201 #3021)
202# Python 3.1a1: 3141 (optimize list, set and dict comprehensions:
203# change LIST_APPEND and SET_ADD, add MAP_ADD #2183)
204# Python 3.1a1: 3151 (optimize conditional branches:
205# introduce POP_JUMP_IF_FALSE and POP_JUMP_IF_TRUE
206 #4715)
207# Python 3.2a1: 3160 (add SETUP_WITH #6101)
Eric Snow32439d62015-05-02 19:15:18 -0600208# tag: cpython-32
Serhiy Storchaka4af8fd52018-02-22 22:26:23 +0200209# Python 3.2a2: 3170 (add DUP_TOP_TWO, remove DUP_TOPX and ROT_FOUR #9225)
Eric Snow32439d62015-05-02 19:15:18 -0600210# tag: cpython-32
Serhiy Storchaka4af8fd52018-02-22 22:26:23 +0200211# Python 3.2a3 3180 (add DELETE_DEREF #4617)
212# Python 3.3a1 3190 (__class__ super closure changed)
213# Python 3.3a1 3200 (PEP 3155 __qualname__ added #13448)
214# Python 3.3a1 3210 (added size modulo 2**32 to the pyc header #13645)
215# Python 3.3a2 3220 (changed PEP 380 implementation #14230)
216# Python 3.3a4 3230 (revert changes to implicit __class__ closure #14857)
Eric Snow32439d62015-05-02 19:15:18 -0600217# Python 3.4a1 3250 (evaluate positional default arguments before
Serhiy Storchaka4af8fd52018-02-22 22:26:23 +0200218# keyword-only defaults #16967)
Eric Snow32439d62015-05-02 19:15:18 -0600219# Python 3.4a1 3260 (add LOAD_CLASSDEREF; allow locals of class to override
Serhiy Storchaka4af8fd52018-02-22 22:26:23 +0200220# free vars #17853)
221# Python 3.4a1 3270 (various tweaks to the __class__ closure #12370)
Eric Snow32439d62015-05-02 19:15:18 -0600222# Python 3.4a1 3280 (remove implicit class argument)
Serhiy Storchaka4af8fd52018-02-22 22:26:23 +0200223# Python 3.4a4 3290 (changes to __qualname__ computation #19301)
224# Python 3.4a4 3300 (more changes to __qualname__ computation #19301)
225# Python 3.4rc2 3310 (alter __qualname__ computation #20625)
226# Python 3.5a1 3320 (PEP 465: Matrix multiplication operator #21176)
227# Python 3.5b1 3330 (PEP 448: Additional Unpacking Generalizations #2292)
Benjamin Petersonee853392015-05-28 14:30:26 -0500228# Python 3.5b2 3340 (fix dictionary display evaluation order #11205)
Serhiy Storchaka4af8fd52018-02-22 22:26:23 +0200229# Python 3.5b3 3350 (add GET_YIELD_FROM_ITER opcode #24400)
Serhiy Storchaka3c317e72016-06-12 09:22:01 +0300230# Python 3.5.2 3351 (fix BUILD_MAP_UNPACK_WITH_CALL opcode #27286)
Serhiy Storchaka4af8fd52018-02-22 22:26:23 +0200231# Python 3.6a0 3360 (add FORMAT_VALUE opcode #25483)
232# Python 3.6a1 3361 (lineno delta of code.co_lnotab becomes signed #26107)
233# Python 3.6a2 3370 (16 bit wordcode #26647)
234# Python 3.6a2 3371 (add BUILD_CONST_KEY_MAP opcode #27140)
235# Python 3.6a2 3372 (MAKE_FUNCTION simplification, remove MAKE_CLOSURE
Serhiy Storchakaea525a22016-09-06 22:07:53 +0300236# #27095)
237# Python 3.6b1 3373 (add BUILD_STRING opcode #27078)
Yury Selivanovf8cb8a12016-09-08 20:50:03 -0700238# Python 3.6b1 3375 (add SETUP_ANNOTATIONS and STORE_ANNOTATION opcodes
239# #27985)
Serhiy Storchaka4af8fd52018-02-22 22:26:23 +0200240# Python 3.6b1 3376 (simplify CALL_FUNCTIONs & BUILD_MAP_UNPACK_WITH_CALL
241 #27213)
Nick Coghlan944368e2016-09-11 14:45:49 +1000242# Python 3.6b1 3377 (set __class__ cell from type.__new__ #23722)
Serhiy Storchaka73442852016-10-02 10:33:46 +0300243# Python 3.6b2 3378 (add BUILD_TUPLE_UNPACK_WITH_CALL #28257)
Nick Coghlan19d24672016-12-05 16:47:55 +1000244# Python 3.6rc1 3379 (more thorough __class__ validation #23722)
Serhiy Storchaka4af8fd52018-02-22 22:26:23 +0200245# Python 3.7a1 3390 (add LOAD_METHOD and CALL_METHOD opcodes #26110)
246# Python 3.7a2 3391 (update GET_AITER #31709)
247# Python 3.7a4 3392 (PEP 552: Deterministic pycs #31650)
248# Python 3.7b1 3393 (remove STORE_ANNOTATION opcode #32550)
Serhiy Storchaka520b7ae2018-02-22 23:33:30 +0200249# Python 3.8a1 3400 (move frame block handling to compiler #17611)
Serhiy Storchaka702f8f32018-03-23 14:34:35 +0200250# Python 3.8a1 3401 (add END_ASYNC_FOR #33041)
Eric Snow32439d62015-05-02 19:15:18 -0600251#
252# MAGIC must change whenever the bytecode emitted by the compiler may no
253# longer be understood by older implementations of the eval loop (usually
254# due to the addition of new opcodes).
Steve Dower7ae61af2016-05-16 09:34:20 -0700255#
256# Whenever MAGIC_NUMBER is changed, the ranges in the magic_values array
257# in PC/launcher.c must also be updated.
Eric Snow32439d62015-05-02 19:15:18 -0600258
Serhiy Storchaka702f8f32018-03-23 14:34:35 +0200259MAGIC_NUMBER = (3401).to_bytes(2, 'little') + b'\r\n'
Eric Snow32439d62015-05-02 19:15:18 -0600260_RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little') # For import.c
261
262_PYCACHE = '__pycache__'
263_OPT = 'opt-'
264
265SOURCE_SUFFIXES = ['.py'] # _setup() adds .pyw as needed.
266
267BYTECODE_SUFFIXES = ['.pyc']
268# Deprecated.
269DEBUG_BYTECODE_SUFFIXES = OPTIMIZED_BYTECODE_SUFFIXES = BYTECODE_SUFFIXES
270
271def cache_from_source(path, debug_override=None, *, optimization=None):
272 """Given the path to a .py file, return the path to its .pyc file.
273
274 The .py file does not need to exist; this simply returns the path to the
275 .pyc file calculated as if the .py file were imported.
276
277 The 'optimization' parameter controls the presumed optimization level of
278 the bytecode file. If 'optimization' is not None, the string representation
279 of the argument is taken and verified to be alphanumeric (else ValueError
280 is raised).
281
282 The debug_override parameter is deprecated. If debug_override is not None,
283 a True value is the same as setting 'optimization' to the empty string
284 while a False value is equivalent to setting 'optimization' to '1'.
285
286 If sys.implementation.cache_tag is None then NotImplementedError is raised.
287
288 """
289 if debug_override is not None:
290 _warnings.warn('the debug_override parameter is deprecated; use '
291 "'optimization' instead", DeprecationWarning)
292 if optimization is not None:
293 message = 'debug_override or optimization must be set to None'
294 raise TypeError(message)
295 optimization = '' if debug_override else 1
Brett Cannon035a1002016-09-07 18:39:18 -0700296 path = _os.fspath(path)
Eric Snow32439d62015-05-02 19:15:18 -0600297 head, tail = _path_split(path)
298 base, sep, rest = tail.rpartition('.')
299 tag = sys.implementation.cache_tag
300 if tag is None:
301 raise NotImplementedError('sys.implementation.cache_tag is None')
302 almost_filename = ''.join([(base if base else rest), sep, tag])
303 if optimization is None:
304 if sys.flags.optimize == 0:
305 optimization = ''
306 else:
307 optimization = sys.flags.optimize
308 optimization = str(optimization)
309 if optimization != '':
310 if not optimization.isalnum():
311 raise ValueError('{!r} is not alphanumeric'.format(optimization))
312 almost_filename = '{}.{}{}'.format(almost_filename, _OPT, optimization)
313 return _path_join(head, _PYCACHE, almost_filename + BYTECODE_SUFFIXES[0])
314
315
316def source_from_cache(path):
317 """Given the path to a .pyc. file, return the path to its .py file.
318
319 The .pyc file does not need to exist; this simply returns the path to
320 the .py file calculated to correspond to the .pyc file. If path does
321 not conform to PEP 3147/488 format, ValueError will be raised. If
322 sys.implementation.cache_tag is None then NotImplementedError is raised.
323
324 """
325 if sys.implementation.cache_tag is None:
326 raise NotImplementedError('sys.implementation.cache_tag is None')
Brett Cannon035a1002016-09-07 18:39:18 -0700327 path = _os.fspath(path)
Eric Snow32439d62015-05-02 19:15:18 -0600328 head, pycache_filename = _path_split(path)
329 head, pycache = _path_split(head)
330 if pycache != _PYCACHE:
331 raise ValueError('{} not bottom-level directory in '
332 '{!r}'.format(_PYCACHE, path))
333 dot_count = pycache_filename.count('.')
334 if dot_count not in {2, 3}:
335 raise ValueError('expected only 2 or 3 dots in '
336 '{!r}'.format(pycache_filename))
337 elif dot_count == 3:
338 optimization = pycache_filename.rsplit('.', 2)[-2]
339 if not optimization.startswith(_OPT):
340 raise ValueError("optimization portion of filename does not start "
341 "with {!r}".format(_OPT))
342 opt_level = optimization[len(_OPT):]
343 if not opt_level.isalnum():
344 raise ValueError("optimization level {!r} is not an alphanumeric "
345 "value".format(optimization))
346 base_filename = pycache_filename.partition('.')[0]
347 return _path_join(head, base_filename + SOURCE_SUFFIXES[0])
348
349
350def _get_sourcefile(bytecode_path):
351 """Convert a bytecode file path to a source path (if possible).
352
353 This function exists purely for backwards-compatibility for
354 PyImport_ExecCodeModuleWithFilenames() in the C API.
355
356 """
357 if len(bytecode_path) == 0:
358 return None
359 rest, _, extension = bytecode_path.rpartition('.')
360 if not rest or extension.lower()[-3:-1] != 'py':
361 return bytecode_path
362 try:
363 source_path = source_from_cache(bytecode_path)
364 except (NotImplementedError, ValueError):
365 source_path = bytecode_path[:-1]
366 return source_path if _path_isfile(source_path) else bytecode_path
367
368
369def _get_cached(filename):
370 if filename.endswith(tuple(SOURCE_SUFFIXES)):
371 try:
372 return cache_from_source(filename)
373 except NotImplementedError:
374 pass
375 elif filename.endswith(tuple(BYTECODE_SUFFIXES)):
376 return filename
377 else:
378 return None
379
380
381def _calc_mode(path):
382 """Calculate the mode permissions for a bytecode file."""
383 try:
384 mode = _path_stat(path).st_mode
385 except OSError:
386 mode = 0o666
387 # We always ensure write access so we can update cached files
388 # later even when the source files are read-only on Windows (#6074)
389 mode |= 0o200
390 return mode
391
392
Eric Snow32439d62015-05-02 19:15:18 -0600393def _check_name(method):
394 """Decorator to verify that the module being requested matches the one the
395 loader can handle.
396
397 The first argument (self) must define _name which the second argument is
398 compared against. If the comparison fails then ImportError is raised.
399
400 """
401 def _check_name_wrapper(self, name=None, *args, **kwargs):
402 if name is None:
403 name = self.name
404 elif self.name != name:
Nick Coghland5cacbb2015-05-23 22:24:10 +1000405 raise ImportError('loader for %s cannot handle %s' %
406 (self.name, name), name=name)
Eric Snow32439d62015-05-02 19:15:18 -0600407 return method(self, name, *args, **kwargs)
408 try:
409 _wrap = _bootstrap._wrap
410 except NameError:
411 # XXX yuck
412 def _wrap(new, old):
413 for replace in ['__module__', '__name__', '__qualname__', '__doc__']:
414 if hasattr(old, replace):
415 setattr(new, replace, getattr(old, replace))
416 new.__dict__.update(old.__dict__)
417 _wrap(_check_name_wrapper, method)
418 return _check_name_wrapper
419
420
421def _find_module_shim(self, fullname):
422 """Try to find a loader for the specified module by delegating to
423 self.find_loader().
424
425 This method is deprecated in favor of finder.find_spec().
426
427 """
428 # Call find_loader(). If it returns a string (indicating this
429 # is a namespace package portion), generate a warning and
430 # return None.
431 loader, portions = self.find_loader(fullname)
432 if loader is None and len(portions):
433 msg = 'Not importing directory {}: missing __init__'
434 _warnings.warn(msg.format(portions[0]), ImportWarning)
435 return loader
436
437
Benjamin Peterson42aa93b2017-12-09 10:26:52 -0800438def _classify_pyc(data, name, exc_details):
439 """Perform basic validity checking of a pyc header and return the flags field,
440 which determines how the pyc should be further validated against the source.
Eric Snow32439d62015-05-02 19:15:18 -0600441
Benjamin Peterson42aa93b2017-12-09 10:26:52 -0800442 *data* is the contents of the pyc file. (Only the first 16 bytes are
443 required, though.)
Eric Snow32439d62015-05-02 19:15:18 -0600444
Benjamin Peterson42aa93b2017-12-09 10:26:52 -0800445 *name* is the name of the module being imported. It is used for logging.
446
447 *exc_details* is a dictionary passed to ImportError if it raised for
448 improved debugging.
449
450 ImportError is raised when the magic number is incorrect or when the flags
451 field is invalid. EOFError is raised when the data is found to be truncated.
Eric Snow32439d62015-05-02 19:15:18 -0600452
453 """
Eric Snow32439d62015-05-02 19:15:18 -0600454 magic = data[:4]
Eric Snow32439d62015-05-02 19:15:18 -0600455 if magic != MAGIC_NUMBER:
Benjamin Peterson42aa93b2017-12-09 10:26:52 -0800456 message = f'bad magic number in {name!r}: {magic!r}'
Serhiy Storchakaf22d8082015-10-01 11:55:52 +0300457 _bootstrap._verbose_message('{}', message)
Eric Snow32439d62015-05-02 19:15:18 -0600458 raise ImportError(message, **exc_details)
Benjamin Peterson42aa93b2017-12-09 10:26:52 -0800459 if len(data) < 16:
460 message = f'reached EOF while reading pyc header of {name!r}'
Serhiy Storchakaf22d8082015-10-01 11:55:52 +0300461 _bootstrap._verbose_message('{}', message)
Eric Snow32439d62015-05-02 19:15:18 -0600462 raise EOFError(message)
Benjamin Peterson42aa93b2017-12-09 10:26:52 -0800463 flags = _r_long(data[4:8])
464 # Only the first two flags are defined.
465 if flags & ~0b11:
466 message = f'invalid flags {flags!r} in {name!r}'
467 raise ImportError(message, **exc_details)
468 return flags
469
470
471def _validate_timestamp_pyc(data, source_mtime, source_size, name,
472 exc_details):
473 """Validate a pyc against the source last-modified time.
474
475 *data* is the contents of the pyc file. (Only the first 16 bytes are
476 required.)
477
478 *source_mtime* is the last modified timestamp of the source file.
479
480 *source_size* is None or the size of the source file in bytes.
481
482 *name* is the name of the module being imported. It is used for logging.
483
484 *exc_details* is a dictionary passed to ImportError if it raised for
485 improved debugging.
486
487 An ImportError is raised if the bytecode is stale.
488
489 """
490 if _r_long(data[8:12]) != (source_mtime & 0xFFFFFFFF):
491 message = f'bytecode is stale for {name!r}'
Serhiy Storchakaf22d8082015-10-01 11:55:52 +0300492 _bootstrap._verbose_message('{}', message)
Benjamin Peterson42aa93b2017-12-09 10:26:52 -0800493 raise ImportError(message, **exc_details)
494 if (source_size is not None and
495 _r_long(data[12:16]) != (source_size & 0xFFFFFFFF)):
496 raise ImportError(f'bytecode is stale for {name!r}', **exc_details)
497
498
499def _validate_hash_pyc(data, source_hash, name, exc_details):
500 """Validate a hash-based pyc by checking the real source hash against the one in
501 the pyc header.
502
503 *data* is the contents of the pyc file. (Only the first 16 bytes are
504 required.)
505
506 *source_hash* is the importlib.util.source_hash() of the source file.
507
508 *name* is the name of the module being imported. It is used for logging.
509
510 *exc_details* is a dictionary passed to ImportError if it raised for
511 improved debugging.
512
513 An ImportError is raised if the bytecode is stale.
514
515 """
516 if data[8:16] != source_hash:
517 raise ImportError(
518 f'hash in bytecode doesn\'t match hash of source {name!r}',
519 **exc_details,
520 )
Eric Snow32439d62015-05-02 19:15:18 -0600521
522
523def _compile_bytecode(data, name=None, bytecode_path=None, source_path=None):
Benjamin Peterson42aa93b2017-12-09 10:26:52 -0800524 """Compile bytecode as found in a pyc."""
Eric Snow32439d62015-05-02 19:15:18 -0600525 code = marshal.loads(data)
526 if isinstance(code, _code_type):
Brett Cannon1dae0c62015-09-25 13:05:13 -0700527 _bootstrap._verbose_message('code object from {!r}', bytecode_path)
Eric Snow32439d62015-05-02 19:15:18 -0600528 if source_path is not None:
529 _imp._fix_co_filename(code, source_path)
530 return code
531 else:
532 raise ImportError('Non-code object in {!r}'.format(bytecode_path),
533 name=name, path=bytecode_path)
534
Benjamin Peterson42aa93b2017-12-09 10:26:52 -0800535
536def _code_to_timestamp_pyc(code, mtime=0, source_size=0):
537 "Produce the data for a timestamp-based pyc."
Eric Snow32439d62015-05-02 19:15:18 -0600538 data = bytearray(MAGIC_NUMBER)
Benjamin Peterson42aa93b2017-12-09 10:26:52 -0800539 data.extend(_w_long(0))
Eric Snow32439d62015-05-02 19:15:18 -0600540 data.extend(_w_long(mtime))
541 data.extend(_w_long(source_size))
542 data.extend(marshal.dumps(code))
543 return data
544
545
Benjamin Peterson42aa93b2017-12-09 10:26:52 -0800546def _code_to_hash_pyc(code, source_hash, checked=True):
547 "Produce the data for a hash-based pyc."
548 data = bytearray(MAGIC_NUMBER)
549 flags = 0b1 | checked << 1
550 data.extend(_w_long(flags))
551 assert len(source_hash) == 8
552 data.extend(source_hash)
553 data.extend(marshal.dumps(code))
554 return data
555
556
Eric Snow32439d62015-05-02 19:15:18 -0600557def decode_source(source_bytes):
558 """Decode bytes representing source code and return the string.
559
560 Universal newline support is used in the decoding.
561 """
562 import tokenize # To avoid bootstrap issues.
563 source_bytes_readline = _io.BytesIO(source_bytes).readline
564 encoding = tokenize.detect_encoding(source_bytes_readline)
565 newline_decoder = _io.IncrementalNewlineDecoder(None, True)
566 return newline_decoder.decode(source_bytes.decode(encoding[0]))
567
568
569# Module specifications #######################################################
570
Eric Snow32439d62015-05-02 19:15:18 -0600571_POPULATE = object()
572
573
574def spec_from_file_location(name, location=None, *, loader=None,
575 submodule_search_locations=_POPULATE):
576 """Return a module spec based on a file location.
577
578 To indicate that the module is a package, set
579 submodule_search_locations to a list of directory paths. An
580 empty list is sufficient, though its not otherwise useful to the
581 import system.
582
583 The loader must take a spec as its only __init__() arg.
584
585 """
586 if location is None:
587 # The caller may simply want a partially populated location-
588 # oriented spec. So we set the location to a bogus value and
589 # fill in as much as we can.
590 location = '<unknown>'
591 if hasattr(loader, 'get_filename'):
592 # ExecutionLoader
593 try:
594 location = loader.get_filename(name)
595 except ImportError:
596 pass
Brett Cannon035a1002016-09-07 18:39:18 -0700597 else:
598 location = _os.fspath(location)
Eric Snow32439d62015-05-02 19:15:18 -0600599
600 # If the location is on the filesystem, but doesn't actually exist,
601 # we could return None here, indicating that the location is not
602 # valid. However, we don't have a good way of testing since an
603 # indirect location (e.g. a zip file or URL) will look like a
604 # non-existent file relative to the filesystem.
605
606 spec = _bootstrap.ModuleSpec(name, loader, origin=location)
607 spec._set_fileattr = True
608
609 # Pick a loader if one wasn't provided.
610 if loader is None:
611 for loader_class, suffixes in _get_supported_file_loaders():
612 if location.endswith(tuple(suffixes)):
613 loader = loader_class(name, location)
614 spec.loader = loader
615 break
616 else:
617 return None
618
619 # Set submodule_search_paths appropriately.
620 if submodule_search_locations is _POPULATE:
621 # Check the loader.
622 if hasattr(loader, 'is_package'):
623 try:
624 is_package = loader.is_package(name)
625 except ImportError:
626 pass
627 else:
628 if is_package:
629 spec.submodule_search_locations = []
630 else:
631 spec.submodule_search_locations = submodule_search_locations
632 if spec.submodule_search_locations == []:
633 if location:
634 dirname = _path_split(location)[0]
635 spec.submodule_search_locations.append(dirname)
636
637 return spec
638
639
640# Loaders #####################################################################
641
642class WindowsRegistryFinder:
643
644 """Meta path finder for modules declared in the Windows registry."""
645
646 REGISTRY_KEY = (
647 'Software\\Python\\PythonCore\\{sys_version}'
648 '\\Modules\\{fullname}')
649 REGISTRY_KEY_DEBUG = (
650 'Software\\Python\\PythonCore\\{sys_version}'
651 '\\Modules\\{fullname}\\Debug')
652 DEBUG_BUILD = False # Changed in _setup()
653
654 @classmethod
655 def _open_registry(cls, key):
656 try:
657 return _winreg.OpenKey(_winreg.HKEY_CURRENT_USER, key)
658 except OSError:
659 return _winreg.OpenKey(_winreg.HKEY_LOCAL_MACHINE, key)
660
661 @classmethod
662 def _search_registry(cls, fullname):
663 if cls.DEBUG_BUILD:
664 registry_key = cls.REGISTRY_KEY_DEBUG
665 else:
666 registry_key = cls.REGISTRY_KEY
667 key = registry_key.format(fullname=fullname,
Serhiy Storchaka885bdc42016-02-11 13:10:36 +0200668 sys_version='%d.%d' % sys.version_info[:2])
Eric Snow32439d62015-05-02 19:15:18 -0600669 try:
670 with cls._open_registry(key) as hkey:
671 filepath = _winreg.QueryValue(hkey, '')
672 except OSError:
673 return None
674 return filepath
675
676 @classmethod
677 def find_spec(cls, fullname, path=None, target=None):
678 filepath = cls._search_registry(fullname)
679 if filepath is None:
680 return None
681 try:
682 _path_stat(filepath)
683 except OSError:
684 return None
685 for loader, suffixes in _get_supported_file_loaders():
686 if filepath.endswith(tuple(suffixes)):
Eric Snow183a9412015-05-15 21:54:59 -0600687 spec = _bootstrap.spec_from_loader(fullname,
688 loader(fullname, filepath),
689 origin=filepath)
Eric Snow32439d62015-05-02 19:15:18 -0600690 return spec
691
692 @classmethod
693 def find_module(cls, fullname, path=None):
694 """Find module named in the registry.
695
696 This method is deprecated. Use exec_module() instead.
697
698 """
699 spec = cls.find_spec(fullname, path)
700 if spec is not None:
701 return spec.loader
702 else:
703 return None
704
705
706class _LoaderBasics:
707
708 """Base class of common code needed by both SourceLoader and
709 SourcelessFileLoader."""
710
711 def is_package(self, fullname):
712 """Concrete implementation of InspectLoader.is_package by checking if
713 the path returned by get_filename has a filename of '__init__.py'."""
714 filename = _path_split(self.get_filename(fullname))[1]
715 filename_base = filename.rsplit('.', 1)[0]
716 tail_name = fullname.rpartition('.')[2]
717 return filename_base == '__init__' and tail_name != '__init__'
718
719 def create_module(self, spec):
720 """Use default semantics for module creation."""
721
722 def exec_module(self, module):
723 """Execute the module."""
724 code = self.get_code(module.__name__)
725 if code is None:
726 raise ImportError('cannot load module {!r} when get_code() '
727 'returns None'.format(module.__name__))
728 _bootstrap._call_with_frames_removed(exec, code, module.__dict__)
729
Eric Snow183a9412015-05-15 21:54:59 -0600730 def load_module(self, fullname):
Brett Cannoneae30792015-12-28 17:55:27 -0800731 """This module is deprecated."""
Eric Snow183a9412015-05-15 21:54:59 -0600732 return _bootstrap._load_module_shim(self, fullname)
Eric Snow32439d62015-05-02 19:15:18 -0600733
734
735class SourceLoader(_LoaderBasics):
736
737 def path_mtime(self, path):
738 """Optional method that returns the modification time (an int) for the
739 specified path, where path is a str.
740
Serhiy Storchaka55fe1ae2017-04-16 10:46:38 +0300741 Raises OSError when the path cannot be handled.
Eric Snow32439d62015-05-02 19:15:18 -0600742 """
Serhiy Storchaka55fe1ae2017-04-16 10:46:38 +0300743 raise OSError
Eric Snow32439d62015-05-02 19:15:18 -0600744
745 def path_stats(self, path):
746 """Optional method returning a metadata dict for the specified path
747 to by the path (str).
748 Possible keys:
749 - 'mtime' (mandatory) is the numeric timestamp of last source
750 code modification;
751 - 'size' (optional) is the size in bytes of the source code.
752
753 Implementing this method allows the loader to read bytecode files.
Serhiy Storchaka55fe1ae2017-04-16 10:46:38 +0300754 Raises OSError when the path cannot be handled.
Eric Snow32439d62015-05-02 19:15:18 -0600755 """
756 return {'mtime': self.path_mtime(path)}
757
758 def _cache_bytecode(self, source_path, cache_path, data):
759 """Optional method which writes data (bytes) to a file path (a str).
760
761 Implementing this method allows for the writing of bytecode files.
762
763 The source path is needed in order to correctly transfer permissions
764 """
765 # For backwards compatibility, we delegate to set_data()
766 return self.set_data(cache_path, data)
767
768 def set_data(self, path, data):
769 """Optional method which writes data (bytes) to a file path (a str).
770
771 Implementing this method allows for the writing of bytecode files.
772 """
773
774
775 def get_source(self, fullname):
776 """Concrete implementation of InspectLoader.get_source."""
777 path = self.get_filename(fullname)
778 try:
779 source_bytes = self.get_data(path)
780 except OSError as exc:
781 raise ImportError('source not available through get_data()',
782 name=fullname) from exc
783 return decode_source(source_bytes)
784
785 def source_to_code(self, data, path, *, _optimize=-1):
786 """Return the code object compiled from source.
787
788 The 'data' argument can be any object type that compile() supports.
789 """
790 return _bootstrap._call_with_frames_removed(compile, data, path, 'exec',
791 dont_inherit=True, optimize=_optimize)
792
793 def get_code(self, fullname):
794 """Concrete implementation of InspectLoader.get_code.
795
796 Reading of bytecode requires path_stats to be implemented. To write
797 bytecode, set_data must also be implemented.
798
799 """
800 source_path = self.get_filename(fullname)
801 source_mtime = None
Benjamin Peterson42aa93b2017-12-09 10:26:52 -0800802 source_bytes = None
803 source_hash = None
804 hash_based = False
805 check_source = True
Eric Snow32439d62015-05-02 19:15:18 -0600806 try:
807 bytecode_path = cache_from_source(source_path)
808 except NotImplementedError:
809 bytecode_path = None
810 else:
811 try:
812 st = self.path_stats(source_path)
Serhiy Storchaka55fe1ae2017-04-16 10:46:38 +0300813 except OSError:
Eric Snow32439d62015-05-02 19:15:18 -0600814 pass
815 else:
816 source_mtime = int(st['mtime'])
817 try:
818 data = self.get_data(bytecode_path)
819 except OSError:
820 pass
821 else:
Benjamin Peterson42aa93b2017-12-09 10:26:52 -0800822 exc_details = {
823 'name': fullname,
824 'path': bytecode_path,
825 }
Eric Snow32439d62015-05-02 19:15:18 -0600826 try:
Benjamin Peterson42aa93b2017-12-09 10:26:52 -0800827 flags = _classify_pyc(data, fullname, exc_details)
828 bytes_data = memoryview(data)[16:]
829 hash_based = flags & 0b1 != 0
830 if hash_based:
831 check_source = flags & 0b10 != 0
832 if (_imp.check_hash_based_pycs != 'never' and
833 (check_source or
834 _imp.check_hash_based_pycs == 'always')):
835 source_bytes = self.get_data(source_path)
836 source_hash = _imp.source_hash(
837 _RAW_MAGIC_NUMBER,
838 source_bytes,
839 )
840 _validate_hash_pyc(data, source_hash, fullname,
841 exc_details)
842 else:
843 _validate_timestamp_pyc(
844 data,
845 source_mtime,
846 st['size'],
847 fullname,
848 exc_details,
849 )
Eric Snow32439d62015-05-02 19:15:18 -0600850 except (ImportError, EOFError):
851 pass
852 else:
Brett Cannon1dae0c62015-09-25 13:05:13 -0700853 _bootstrap._verbose_message('{} matches {}', bytecode_path,
854 source_path)
Eric Snow32439d62015-05-02 19:15:18 -0600855 return _compile_bytecode(bytes_data, name=fullname,
856 bytecode_path=bytecode_path,
857 source_path=source_path)
Benjamin Peterson42aa93b2017-12-09 10:26:52 -0800858 if source_bytes is None:
859 source_bytes = self.get_data(source_path)
Eric Snow32439d62015-05-02 19:15:18 -0600860 code_object = self.source_to_code(source_bytes, source_path)
Brett Cannon1dae0c62015-09-25 13:05:13 -0700861 _bootstrap._verbose_message('code object from {}', source_path)
Eric Snow32439d62015-05-02 19:15:18 -0600862 if (not sys.dont_write_bytecode and bytecode_path is not None and
863 source_mtime is not None):
Benjamin Peterson42aa93b2017-12-09 10:26:52 -0800864 if hash_based:
865 if source_hash is None:
866 source_hash = _imp.source_hash(source_bytes)
867 data = _code_to_hash_pyc(code_object, source_hash, check_source)
868 else:
869 data = _code_to_timestamp_pyc(code_object, source_mtime,
870 len(source_bytes))
Eric Snow32439d62015-05-02 19:15:18 -0600871 try:
872 self._cache_bytecode(source_path, bytecode_path, data)
Brett Cannon1dae0c62015-09-25 13:05:13 -0700873 _bootstrap._verbose_message('wrote {!r}', bytecode_path)
Eric Snow32439d62015-05-02 19:15:18 -0600874 except NotImplementedError:
875 pass
876 return code_object
877
878
879class FileLoader:
880
881 """Base file loader class which implements the loader protocol methods that
882 require file system usage."""
883
884 def __init__(self, fullname, path):
885 """Cache the module name and the path to the file found by the
886 finder."""
887 self.name = fullname
888 self.path = path
889
890 def __eq__(self, other):
891 return (self.__class__ == other.__class__ and
892 self.__dict__ == other.__dict__)
893
894 def __hash__(self):
895 return hash(self.name) ^ hash(self.path)
896
897 @_check_name
898 def load_module(self, fullname):
899 """Load a module from a file.
900
901 This method is deprecated. Use exec_module() instead.
902
903 """
904 # The only reason for this method is for the name check.
905 # Issue #14857: Avoid the zero-argument form of super so the implementation
906 # of that form can be updated without breaking the frozen module
907 return super(FileLoader, self).load_module(fullname)
908
909 @_check_name
910 def get_filename(self, fullname):
911 """Return the path to the source file as found by the finder."""
912 return self.path
913
914 def get_data(self, path):
915 """Return the data from path as raw bytes."""
916 with _io.FileIO(path, 'r') as file:
917 return file.read()
918
Barry Warsaw5ec0fee2018-01-15 15:07:11 -0800919 # ResourceReader ABC API.
920
921 @_check_name
922 def get_resource_reader(self, module):
923 if self.is_package(module):
924 return self
925 return None
926
927 def open_resource(self, resource):
928 path = _path_join(_path_split(self.path)[0], resource)
929 return _io.FileIO(path, 'r')
930
931 def resource_path(self, resource):
932 if not self.is_resource(resource):
933 raise FileNotFoundError
934 path = _path_join(_path_split(self.path)[0], resource)
935 return path
936
937 def is_resource(self, name):
938 if path_sep in name:
939 return False
940 path = _path_join(_path_split(self.path)[0], name)
941 return _path_isfile(path)
942
943 def contents(self):
944 return iter(_os.listdir(_path_split(self.path)[0]))
945
Eric Snow32439d62015-05-02 19:15:18 -0600946
947class SourceFileLoader(FileLoader, SourceLoader):
948
949 """Concrete implementation of SourceLoader using the file system."""
950
951 def path_stats(self, path):
952 """Return the metadata for the path."""
953 st = _path_stat(path)
954 return {'mtime': st.st_mtime, 'size': st.st_size}
955
956 def _cache_bytecode(self, source_path, bytecode_path, data):
957 # Adapt between the two APIs
958 mode = _calc_mode(source_path)
959 return self.set_data(bytecode_path, data, _mode=mode)
960
961 def set_data(self, path, data, *, _mode=0o666):
962 """Write bytes data to a file."""
963 parent, filename = _path_split(path)
964 path_parts = []
965 # Figure out what directories are missing.
966 while parent and not _path_isdir(parent):
967 parent, part = _path_split(parent)
968 path_parts.append(part)
969 # Create needed directories.
970 for part in reversed(path_parts):
971 parent = _path_join(parent, part)
972 try:
973 _os.mkdir(parent)
974 except FileExistsError:
975 # Probably another Python process already created the dir.
976 continue
977 except OSError as exc:
978 # Could be a permission error, read-only filesystem: just forget
979 # about writing the data.
Brett Cannon1dae0c62015-09-25 13:05:13 -0700980 _bootstrap._verbose_message('could not create {!r}: {!r}',
981 parent, exc)
Eric Snow32439d62015-05-02 19:15:18 -0600982 return
983 try:
984 _write_atomic(path, data, _mode)
Brett Cannon1dae0c62015-09-25 13:05:13 -0700985 _bootstrap._verbose_message('created {!r}', path)
Eric Snow32439d62015-05-02 19:15:18 -0600986 except OSError as exc:
987 # Same as above: just don't write the bytecode.
Brett Cannon1dae0c62015-09-25 13:05:13 -0700988 _bootstrap._verbose_message('could not create {!r}: {!r}', path,
989 exc)
Eric Snow32439d62015-05-02 19:15:18 -0600990
991
992class SourcelessFileLoader(FileLoader, _LoaderBasics):
993
994 """Loader which handles sourceless file imports."""
995
996 def get_code(self, fullname):
997 path = self.get_filename(fullname)
998 data = self.get_data(path)
Benjamin Peterson42aa93b2017-12-09 10:26:52 -0800999 # Call _classify_pyc to do basic validation of the pyc but ignore the
1000 # result. There's no source to check against.
1001 exc_details = {
1002 'name': fullname,
1003 'path': path,
1004 }
1005 _classify_pyc(data, fullname, exc_details)
1006 return _compile_bytecode(
1007 memoryview(data)[16:],
1008 name=fullname,
1009 bytecode_path=path,
1010 )
Eric Snow32439d62015-05-02 19:15:18 -06001011
1012 def get_source(self, fullname):
1013 """Return None as there is no source code."""
1014 return None
1015
1016
1017# Filled in by _setup().
1018EXTENSION_SUFFIXES = []
1019
1020
Nick Coghland5cacbb2015-05-23 22:24:10 +10001021class ExtensionFileLoader(FileLoader, _LoaderBasics):
Eric Snow32439d62015-05-02 19:15:18 -06001022
1023 """Loader for extension modules.
1024
1025 The constructor is designed to work with FileFinder.
1026
1027 """
1028
1029 def __init__(self, name, path):
1030 self.name = name
1031 self.path = path
1032
1033 def __eq__(self, other):
1034 return (self.__class__ == other.__class__ and
1035 self.__dict__ == other.__dict__)
1036
1037 def __hash__(self):
1038 return hash(self.name) ^ hash(self.path)
1039
Nick Coghland5cacbb2015-05-23 22:24:10 +10001040 def create_module(self, spec):
1041 """Create an unitialized extension module"""
1042 module = _bootstrap._call_with_frames_removed(
1043 _imp.create_dynamic, spec)
Brett Cannon1dae0c62015-09-25 13:05:13 -07001044 _bootstrap._verbose_message('extension module {!r} loaded from {!r}',
Nick Coghland5cacbb2015-05-23 22:24:10 +10001045 spec.name, self.path)
Eric Snow32439d62015-05-02 19:15:18 -06001046 return module
1047
Nick Coghland5cacbb2015-05-23 22:24:10 +10001048 def exec_module(self, module):
1049 """Initialize an extension module"""
1050 _bootstrap._call_with_frames_removed(_imp.exec_dynamic, module)
Brett Cannon1dae0c62015-09-25 13:05:13 -07001051 _bootstrap._verbose_message('extension module {!r} executed from {!r}',
Nick Coghland5cacbb2015-05-23 22:24:10 +10001052 self.name, self.path)
1053
Eric Snow32439d62015-05-02 19:15:18 -06001054 def is_package(self, fullname):
1055 """Return True if the extension module is a package."""
1056 file_name = _path_split(self.path)[1]
1057 return any(file_name == '__init__' + suffix
1058 for suffix in EXTENSION_SUFFIXES)
1059
1060 def get_code(self, fullname):
1061 """Return None as an extension module cannot create a code object."""
1062 return None
1063
1064 def get_source(self, fullname):
1065 """Return None as extension modules have no source code."""
1066 return None
1067
1068 @_check_name
1069 def get_filename(self, fullname):
1070 """Return the path to the source file as found by the finder."""
1071 return self.path
1072
1073
1074class _NamespacePath:
1075 """Represents a namespace package's path. It uses the module name
1076 to find its parent module, and from there it looks up the parent's
1077 __path__. When this changes, the module's own path is recomputed,
1078 using path_finder. For top-level modules, the parent module's path
1079 is sys.path."""
1080
1081 def __init__(self, name, path, path_finder):
1082 self._name = name
1083 self._path = path
1084 self._last_parent_path = tuple(self._get_parent_path())
1085 self._path_finder = path_finder
1086
1087 def _find_parent_path_names(self):
1088 """Returns a tuple of (parent-module-name, parent-path-attr-name)"""
1089 parent, dot, me = self._name.rpartition('.')
1090 if dot == '':
1091 # This is a top-level module. sys.path contains the parent path.
1092 return 'sys', 'path'
1093 # Not a top-level module. parent-module.__path__ contains the
1094 # parent path.
1095 return parent, '__path__'
1096
1097 def _get_parent_path(self):
1098 parent_module_name, path_attr_name = self._find_parent_path_names()
1099 return getattr(sys.modules[parent_module_name], path_attr_name)
1100
1101 def _recalculate(self):
1102 # If the parent's path has changed, recalculate _path
1103 parent_path = tuple(self._get_parent_path()) # Make a copy
1104 if parent_path != self._last_parent_path:
1105 spec = self._path_finder(self._name, parent_path)
1106 # Note that no changes are made if a loader is returned, but we
1107 # do remember the new parent path
1108 if spec is not None and spec.loader is None:
1109 if spec.submodule_search_locations:
1110 self._path = spec.submodule_search_locations
1111 self._last_parent_path = parent_path # Save the copy
1112 return self._path
1113
1114 def __iter__(self):
1115 return iter(self._recalculate())
1116
Victor Stinner82f04e22016-03-15 23:08:44 +01001117 def __setitem__(self, index, path):
1118 self._path[index] = path
1119
Eric Snow32439d62015-05-02 19:15:18 -06001120 def __len__(self):
1121 return len(self._recalculate())
1122
1123 def __repr__(self):
1124 return '_NamespacePath({!r})'.format(self._path)
1125
1126 def __contains__(self, item):
1127 return item in self._recalculate()
1128
1129 def append(self, item):
1130 self._path.append(item)
1131
1132
1133# We use this exclusively in module_from_spec() for backward-compatibility.
1134class _NamespaceLoader:
1135 def __init__(self, name, path, path_finder):
1136 self._path = _NamespacePath(name, path, path_finder)
1137
1138 @classmethod
1139 def module_repr(cls, module):
1140 """Return repr for the module.
1141
1142 The method is deprecated. The import machinery does the job itself.
1143
1144 """
1145 return '<module {!r} (namespace)>'.format(module.__name__)
1146
1147 def is_package(self, fullname):
1148 return True
1149
1150 def get_source(self, fullname):
1151 return ''
1152
1153 def get_code(self, fullname):
1154 return compile('', '<string>', 'exec', dont_inherit=True)
1155
1156 def create_module(self, spec):
1157 """Use default semantics for module creation."""
1158
1159 def exec_module(self, module):
1160 pass
1161
1162 def load_module(self, fullname):
1163 """Load a namespace module.
1164
1165 This method is deprecated. Use exec_module() instead.
1166
1167 """
1168 # The import system never calls this method.
Brett Cannon1dae0c62015-09-25 13:05:13 -07001169 _bootstrap._verbose_message('namespace module loaded with path {!r}',
1170 self._path)
Eric Snow183a9412015-05-15 21:54:59 -06001171 return _bootstrap._load_module_shim(self, fullname)
Eric Snow32439d62015-05-02 19:15:18 -06001172
1173
1174# Finders #####################################################################
1175
1176class PathFinder:
1177
1178 """Meta path finder for sys.path and package __path__ attributes."""
1179
1180 @classmethod
1181 def invalidate_caches(cls):
1182 """Call the invalidate_caches() method on all path entry finders
1183 stored in sys.path_importer_caches (where implemented)."""
1184 for finder in sys.path_importer_cache.values():
1185 if hasattr(finder, 'invalidate_caches'):
1186 finder.invalidate_caches()
1187
1188 @classmethod
1189 def _path_hooks(cls, path):
Brett Cannon3fef2ee2016-07-08 11:09:35 -07001190 """Search sys.path_hooks for a finder for 'path'."""
Eric Snow32439d62015-05-02 19:15:18 -06001191 if sys.path_hooks is not None and not sys.path_hooks:
1192 _warnings.warn('sys.path_hooks is empty', ImportWarning)
1193 for hook in sys.path_hooks:
1194 try:
1195 return hook(path)
1196 except ImportError:
1197 continue
1198 else:
1199 return None
1200
1201 @classmethod
1202 def _path_importer_cache(cls, path):
1203 """Get the finder for the path entry from sys.path_importer_cache.
1204
1205 If the path entry is not in the cache, find the appropriate finder
1206 and cache it. If no finder is available, store None.
1207
1208 """
1209 if path == '':
1210 try:
1211 path = _os.getcwd()
1212 except FileNotFoundError:
1213 # Don't cache the failure as the cwd can easily change to
1214 # a valid directory later on.
1215 return None
1216 try:
1217 finder = sys.path_importer_cache[path]
1218 except KeyError:
1219 finder = cls._path_hooks(path)
1220 sys.path_importer_cache[path] = finder
1221 return finder
1222
1223 @classmethod
1224 def _legacy_get_spec(cls, fullname, finder):
1225 # This would be a good place for a DeprecationWarning if
1226 # we ended up going that route.
1227 if hasattr(finder, 'find_loader'):
1228 loader, portions = finder.find_loader(fullname)
1229 else:
1230 loader = finder.find_module(fullname)
1231 portions = []
1232 if loader is not None:
Eric Snow183a9412015-05-15 21:54:59 -06001233 return _bootstrap.spec_from_loader(fullname, loader)
Eric Snow32439d62015-05-02 19:15:18 -06001234 spec = _bootstrap.ModuleSpec(fullname, None)
1235 spec.submodule_search_locations = portions
1236 return spec
1237
1238 @classmethod
1239 def _get_spec(cls, fullname, path, target=None):
1240 """Find the loader or namespace_path for this module/package name."""
1241 # If this ends up being a namespace package, namespace_path is
1242 # the list of paths that will become its __path__
1243 namespace_path = []
1244 for entry in path:
1245 if not isinstance(entry, (str, bytes)):
1246 continue
1247 finder = cls._path_importer_cache(entry)
1248 if finder is not None:
1249 if hasattr(finder, 'find_spec'):
1250 spec = finder.find_spec(fullname, target)
1251 else:
1252 spec = cls._legacy_get_spec(fullname, finder)
1253 if spec is None:
1254 continue
1255 if spec.loader is not None:
1256 return spec
1257 portions = spec.submodule_search_locations
1258 if portions is None:
1259 raise ImportError('spec missing loader')
1260 # This is possibly part of a namespace package.
1261 # Remember these path entries (if any) for when we
1262 # create a namespace package, and continue iterating
1263 # on path.
1264 namespace_path.extend(portions)
1265 else:
1266 spec = _bootstrap.ModuleSpec(fullname, None)
1267 spec.submodule_search_locations = namespace_path
1268 return spec
1269
1270 @classmethod
1271 def find_spec(cls, fullname, path=None, target=None):
Brett Cannon3fef2ee2016-07-08 11:09:35 -07001272 """Try to find a spec for 'fullname' on sys.path or 'path'.
1273
1274 The search is based on sys.path_hooks and sys.path_importer_cache.
1275 """
Eric Snow32439d62015-05-02 19:15:18 -06001276 if path is None:
1277 path = sys.path
1278 spec = cls._get_spec(fullname, path, target)
1279 if spec is None:
1280 return None
1281 elif spec.loader is None:
1282 namespace_path = spec.submodule_search_locations
1283 if namespace_path:
Barry Warsawbbbcf862018-02-02 15:15:58 -05001284 # We found at least one namespace path. Return a spec which
1285 # can create the namespace package.
1286 spec.origin = None
Eric Snow32439d62015-05-02 19:15:18 -06001287 spec.submodule_search_locations = _NamespacePath(fullname, namespace_path, cls._get_spec)
1288 return spec
1289 else:
1290 return None
1291 else:
1292 return spec
1293
1294 @classmethod
1295 def find_module(cls, fullname, path=None):
1296 """find the module on sys.path or 'path' based on sys.path_hooks and
1297 sys.path_importer_cache.
1298
1299 This method is deprecated. Use find_spec() instead.
1300
1301 """
1302 spec = cls.find_spec(fullname, path)
1303 if spec is None:
1304 return None
1305 return spec.loader
1306
1307
1308class FileFinder:
1309
1310 """File-based finder.
1311
1312 Interactions with the file system are cached for performance, being
1313 refreshed when the directory the finder is handling has been modified.
1314
1315 """
1316
1317 def __init__(self, path, *loader_details):
1318 """Initialize with the path to search on and a variable number of
1319 2-tuples containing the loader and the file suffixes the loader
1320 recognizes."""
1321 loaders = []
1322 for loader, suffixes in loader_details:
1323 loaders.extend((suffix, loader) for suffix in suffixes)
1324 self._loaders = loaders
1325 # Base (directory) path
1326 self.path = path or '.'
1327 self._path_mtime = -1
1328 self._path_cache = set()
1329 self._relaxed_path_cache = set()
1330
1331 def invalidate_caches(self):
1332 """Invalidate the directory mtime."""
1333 self._path_mtime = -1
1334
1335 find_module = _find_module_shim
1336
1337 def find_loader(self, fullname):
1338 """Try to find a loader for the specified module, or the namespace
1339 package portions. Returns (loader, list-of-portions).
1340
1341 This method is deprecated. Use find_spec() instead.
1342
1343 """
1344 spec = self.find_spec(fullname)
1345 if spec is None:
1346 return None, []
1347 return spec.loader, spec.submodule_search_locations or []
1348
1349 def _get_spec(self, loader_class, fullname, path, smsl, target):
1350 loader = loader_class(fullname, path)
1351 return spec_from_file_location(fullname, path, loader=loader,
1352 submodule_search_locations=smsl)
1353
1354 def find_spec(self, fullname, target=None):
Brett Cannon3fef2ee2016-07-08 11:09:35 -07001355 """Try to find a spec for the specified module.
1356
1357 Returns the matching spec, or None if not found.
1358 """
Eric Snow32439d62015-05-02 19:15:18 -06001359 is_namespace = False
1360 tail_module = fullname.rpartition('.')[2]
1361 try:
1362 mtime = _path_stat(self.path or _os.getcwd()).st_mtime
1363 except OSError:
1364 mtime = -1
1365 if mtime != self._path_mtime:
1366 self._fill_cache()
1367 self._path_mtime = mtime
1368 # tail_module keeps the original casing, for __file__ and friends
1369 if _relax_case():
1370 cache = self._relaxed_path_cache
1371 cache_module = tail_module.lower()
1372 else:
1373 cache = self._path_cache
1374 cache_module = tail_module
1375 # Check if the module is the name of a directory (and thus a package).
1376 if cache_module in cache:
1377 base_path = _path_join(self.path, tail_module)
1378 for suffix, loader_class in self._loaders:
1379 init_filename = '__init__' + suffix
1380 full_path = _path_join(base_path, init_filename)
1381 if _path_isfile(full_path):
1382 return self._get_spec(loader_class, fullname, full_path, [base_path], target)
1383 else:
1384 # If a namespace package, return the path if we don't
1385 # find a module in the next section.
1386 is_namespace = _path_isdir(base_path)
1387 # Check for a file w/ a proper suffix exists.
1388 for suffix, loader_class in self._loaders:
1389 full_path = _path_join(self.path, tail_module + suffix)
Brett Cannon1dae0c62015-09-25 13:05:13 -07001390 _bootstrap._verbose_message('trying {}', full_path, verbosity=2)
Eric Snow32439d62015-05-02 19:15:18 -06001391 if cache_module + suffix in cache:
1392 if _path_isfile(full_path):
Brett Cannon1dae0c62015-09-25 13:05:13 -07001393 return self._get_spec(loader_class, fullname, full_path,
1394 None, target)
Eric Snow32439d62015-05-02 19:15:18 -06001395 if is_namespace:
Brett Cannon1dae0c62015-09-25 13:05:13 -07001396 _bootstrap._verbose_message('possible namespace for {}', base_path)
Eric Snow32439d62015-05-02 19:15:18 -06001397 spec = _bootstrap.ModuleSpec(fullname, None)
1398 spec.submodule_search_locations = [base_path]
1399 return spec
1400 return None
1401
1402 def _fill_cache(self):
1403 """Fill the cache of potential modules and packages for this directory."""
1404 path = self.path
1405 try:
1406 contents = _os.listdir(path or _os.getcwd())
1407 except (FileNotFoundError, PermissionError, NotADirectoryError):
1408 # Directory has either been removed, turned into a file, or made
1409 # unreadable.
1410 contents = []
1411 # We store two cached versions, to handle runtime changes of the
1412 # PYTHONCASEOK environment variable.
1413 if not sys.platform.startswith('win'):
1414 self._path_cache = set(contents)
1415 else:
1416 # Windows users can import modules with case-insensitive file
1417 # suffixes (for legacy reasons). Make the suffix lowercase here
1418 # so it's done once instead of for every import. This is safe as
1419 # the specified suffixes to check against are always specified in a
1420 # case-sensitive manner.
1421 lower_suffix_contents = set()
1422 for item in contents:
1423 name, dot, suffix = item.partition('.')
1424 if dot:
1425 new_name = '{}.{}'.format(name, suffix.lower())
1426 else:
1427 new_name = name
1428 lower_suffix_contents.add(new_name)
1429 self._path_cache = lower_suffix_contents
1430 if sys.platform.startswith(_CASE_INSENSITIVE_PLATFORMS):
1431 self._relaxed_path_cache = {fn.lower() for fn in contents}
1432
1433 @classmethod
1434 def path_hook(cls, *loader_details):
1435 """A class method which returns a closure to use on sys.path_hook
1436 which will return an instance using the specified loaders and the path
1437 called on the closure.
1438
1439 If the path called on the closure is not a directory, ImportError is
1440 raised.
1441
1442 """
1443 def path_hook_for_FileFinder(path):
1444 """Path hook for importlib.machinery.FileFinder."""
1445 if not _path_isdir(path):
1446 raise ImportError('only directories are supported', path=path)
1447 return cls(path, *loader_details)
1448
1449 return path_hook_for_FileFinder
1450
1451 def __repr__(self):
1452 return 'FileFinder({!r})'.format(self.path)
1453
1454
1455# Import setup ###############################################################
1456
1457def _fix_up_module(ns, name, pathname, cpathname=None):
1458 # This function is used by PyImport_ExecCodeModuleObject().
1459 loader = ns.get('__loader__')
1460 spec = ns.get('__spec__')
1461 if not loader:
1462 if spec:
1463 loader = spec.loader
1464 elif pathname == cpathname:
1465 loader = SourcelessFileLoader(name, pathname)
1466 else:
1467 loader = SourceFileLoader(name, pathname)
1468 if not spec:
1469 spec = spec_from_file_location(name, pathname, loader=loader)
1470 try:
1471 ns['__spec__'] = spec
1472 ns['__loader__'] = loader
1473 ns['__file__'] = pathname
1474 ns['__cached__'] = cpathname
1475 except Exception:
1476 # Not important enough to report.
1477 pass
1478
1479
1480def _get_supported_file_loaders():
1481 """Returns a list of file-based module loaders.
1482
1483 Each item is a tuple (loader, suffixes).
1484 """
1485 extensions = ExtensionFileLoader, _imp.extension_suffixes()
1486 source = SourceFileLoader, SOURCE_SUFFIXES
1487 bytecode = SourcelessFileLoader, BYTECODE_SUFFIXES
1488 return [extensions, source, bytecode]
1489
1490
1491def _setup(_bootstrap_module):
1492 """Setup the path-based importers for importlib by importing needed
1493 built-in modules and injecting them into the global namespace.
1494
1495 Other components are extracted from the core bootstrap module.
1496
1497 """
1498 global sys, _imp, _bootstrap
1499 _bootstrap = _bootstrap_module
1500 sys = _bootstrap.sys
1501 _imp = _bootstrap._imp
1502
1503 # Directly load built-in modules needed during bootstrap.
1504 self_module = sys.modules[__name__]
1505 for builtin_name in ('_io', '_warnings', 'builtins', 'marshal'):
1506 if builtin_name not in sys.modules:
1507 builtin_module = _bootstrap._builtin_from_name(builtin_name)
1508 else:
1509 builtin_module = sys.modules[builtin_name]
1510 setattr(self_module, builtin_name, builtin_module)
1511
1512 # Directly load the os module (needed during bootstrap).
1513 os_details = ('posix', ['/']), ('nt', ['\\', '/'])
1514 for builtin_os, path_separators in os_details:
1515 # Assumption made in _path_join()
1516 assert all(len(sep) == 1 for sep in path_separators)
1517 path_sep = path_separators[0]
1518 if builtin_os in sys.modules:
1519 os_module = sys.modules[builtin_os]
1520 break
1521 else:
1522 try:
1523 os_module = _bootstrap._builtin_from_name(builtin_os)
1524 break
1525 except ImportError:
1526 continue
1527 else:
1528 raise ImportError('importlib requires posix or nt')
1529 setattr(self_module, '_os', os_module)
1530 setattr(self_module, 'path_sep', path_sep)
1531 setattr(self_module, 'path_separators', ''.join(path_separators))
1532
1533 # Directly load the _thread module (needed during bootstrap).
Antoine Pitrou88c60c92017-09-18 23:50:44 +02001534 thread_module = _bootstrap._builtin_from_name('_thread')
Eric Snow32439d62015-05-02 19:15:18 -06001535 setattr(self_module, '_thread', thread_module)
1536
1537 # Directly load the _weakref module (needed during bootstrap).
1538 weakref_module = _bootstrap._builtin_from_name('_weakref')
1539 setattr(self_module, '_weakref', weakref_module)
1540
1541 # Directly load the winreg module (needed during bootstrap).
1542 if builtin_os == 'nt':
1543 winreg_module = _bootstrap._builtin_from_name('winreg')
1544 setattr(self_module, '_winreg', winreg_module)
1545
1546 # Constants
1547 setattr(self_module, '_relax_case', _make_relax_case())
1548 EXTENSION_SUFFIXES.extend(_imp.extension_suffixes())
1549 if builtin_os == 'nt':
1550 SOURCE_SUFFIXES.append('.pyw')
1551 if '_d.pyd' in EXTENSION_SUFFIXES:
1552 WindowsRegistryFinder.DEBUG_BUILD = True
1553
1554
1555def _install(_bootstrap_module):
1556 """Install the path-based import components."""
1557 _setup(_bootstrap_module)
1558 supported_loaders = _get_supported_file_loaders()
1559 sys.path_hooks.extend([FileFinder.path_hook(*supported_loaders)])
Eric Snow32439d62015-05-02 19:15:18 -06001560 sys.meta_path.append(PathFinder)