Jingwen Chen | 475b3cc | 2021-01-05 21:45:16 -0500 | [diff] [blame] | 1 | """Utility code for constructing importers, etc.""" |
| 2 | from . import abc |
| 3 | from ._bootstrap import module_from_spec |
| 4 | from ._bootstrap import _resolve_name |
| 5 | from ._bootstrap import spec_from_loader |
| 6 | from ._bootstrap import _find_spec |
| 7 | from ._bootstrap_external import MAGIC_NUMBER |
| 8 | from ._bootstrap_external import _RAW_MAGIC_NUMBER |
| 9 | from ._bootstrap_external import cache_from_source |
| 10 | from ._bootstrap_external import decode_source |
| 11 | from ._bootstrap_external import source_from_cache |
| 12 | from ._bootstrap_external import spec_from_file_location |
| 13 | |
| 14 | from contextlib import contextmanager |
| 15 | import _imp |
| 16 | import functools |
| 17 | import sys |
| 18 | import types |
| 19 | import warnings |
| 20 | |
| 21 | |
| 22 | def source_hash(source_bytes): |
| 23 | "Return the hash of *source_bytes* as used in hash-based pyc files." |
| 24 | return _imp.source_hash(_RAW_MAGIC_NUMBER, source_bytes) |
| 25 | |
| 26 | |
| 27 | def resolve_name(name, package): |
| 28 | """Resolve a relative module name to an absolute one.""" |
| 29 | if not name.startswith('.'): |
| 30 | return name |
| 31 | elif not package: |
| 32 | raise ValueError(f'no package specified for {repr(name)} ' |
| 33 | '(required for relative module names)') |
| 34 | level = 0 |
| 35 | for character in name: |
| 36 | if character != '.': |
| 37 | break |
| 38 | level += 1 |
| 39 | return _resolve_name(name[level:], package, level) |
| 40 | |
| 41 | |
| 42 | def _find_spec_from_path(name, path=None): |
| 43 | """Return the spec for the specified module. |
| 44 | |
| 45 | First, sys.modules is checked to see if the module was already imported. If |
| 46 | so, then sys.modules[name].__spec__ is returned. If that happens to be |
| 47 | set to None, then ValueError is raised. If the module is not in |
| 48 | sys.modules, then sys.meta_path is searched for a suitable spec with the |
| 49 | value of 'path' given to the finders. None is returned if no spec could |
| 50 | be found. |
| 51 | |
| 52 | Dotted names do not have their parent packages implicitly imported. You will |
| 53 | most likely need to explicitly import all parent packages in the proper |
| 54 | order for a submodule to get the correct spec. |
| 55 | |
| 56 | """ |
| 57 | if name not in sys.modules: |
| 58 | return _find_spec(name, path) |
| 59 | else: |
| 60 | module = sys.modules[name] |
| 61 | if module is None: |
| 62 | return None |
| 63 | try: |
| 64 | spec = module.__spec__ |
| 65 | except AttributeError: |
| 66 | raise ValueError('{}.__spec__ is not set'.format(name)) from None |
| 67 | else: |
| 68 | if spec is None: |
| 69 | raise ValueError('{}.__spec__ is None'.format(name)) |
| 70 | return spec |
| 71 | |
| 72 | |
| 73 | def find_spec(name, package=None): |
| 74 | """Return the spec for the specified module. |
| 75 | |
| 76 | First, sys.modules is checked to see if the module was already imported. If |
| 77 | so, then sys.modules[name].__spec__ is returned. If that happens to be |
| 78 | set to None, then ValueError is raised. If the module is not in |
| 79 | sys.modules, then sys.meta_path is searched for a suitable spec with the |
| 80 | value of 'path' given to the finders. None is returned if no spec could |
| 81 | be found. |
| 82 | |
| 83 | If the name is for submodule (contains a dot), the parent module is |
| 84 | automatically imported. |
| 85 | |
| 86 | The name and package arguments work the same as importlib.import_module(). |
| 87 | In other words, relative module names (with leading dots) work. |
| 88 | |
| 89 | """ |
| 90 | fullname = resolve_name(name, package) if name.startswith('.') else name |
| 91 | if fullname not in sys.modules: |
| 92 | parent_name = fullname.rpartition('.')[0] |
| 93 | if parent_name: |
| 94 | parent = __import__(parent_name, fromlist=['__path__']) |
| 95 | try: |
| 96 | parent_path = parent.__path__ |
| 97 | except AttributeError as e: |
| 98 | raise ModuleNotFoundError( |
| 99 | f"__path__ attribute not found on {parent_name!r} " |
| 100 | f"while trying to find {fullname!r}", name=fullname) from e |
| 101 | else: |
| 102 | parent_path = None |
| 103 | return _find_spec(fullname, parent_path) |
| 104 | else: |
| 105 | module = sys.modules[fullname] |
| 106 | if module is None: |
| 107 | return None |
| 108 | try: |
| 109 | spec = module.__spec__ |
| 110 | except AttributeError: |
| 111 | raise ValueError('{}.__spec__ is not set'.format(name)) from None |
| 112 | else: |
| 113 | if spec is None: |
| 114 | raise ValueError('{}.__spec__ is None'.format(name)) |
| 115 | return spec |
| 116 | |
| 117 | |
| 118 | @contextmanager |
| 119 | def _module_to_load(name): |
| 120 | is_reload = name in sys.modules |
| 121 | |
| 122 | module = sys.modules.get(name) |
| 123 | if not is_reload: |
| 124 | # This must be done before open() is called as the 'io' module |
| 125 | # implicitly imports 'locale' and would otherwise trigger an |
| 126 | # infinite loop. |
| 127 | module = type(sys)(name) |
| 128 | # This must be done before putting the module in sys.modules |
| 129 | # (otherwise an optimization shortcut in import.c becomes wrong) |
| 130 | module.__initializing__ = True |
| 131 | sys.modules[name] = module |
| 132 | try: |
| 133 | yield module |
| 134 | except Exception: |
| 135 | if not is_reload: |
| 136 | try: |
| 137 | del sys.modules[name] |
| 138 | except KeyError: |
| 139 | pass |
| 140 | finally: |
| 141 | module.__initializing__ = False |
| 142 | |
| 143 | |
| 144 | def set_package(fxn): |
| 145 | """Set __package__ on the returned module. |
| 146 | |
| 147 | This function is deprecated. |
| 148 | |
| 149 | """ |
| 150 | @functools.wraps(fxn) |
| 151 | def set_package_wrapper(*args, **kwargs): |
| 152 | warnings.warn('The import system now takes care of this automatically.', |
| 153 | DeprecationWarning, stacklevel=2) |
| 154 | module = fxn(*args, **kwargs) |
| 155 | if getattr(module, '__package__', None) is None: |
| 156 | module.__package__ = module.__name__ |
| 157 | if not hasattr(module, '__path__'): |
| 158 | module.__package__ = module.__package__.rpartition('.')[0] |
| 159 | return module |
| 160 | return set_package_wrapper |
| 161 | |
| 162 | |
| 163 | def set_loader(fxn): |
| 164 | """Set __loader__ on the returned module. |
| 165 | |
| 166 | This function is deprecated. |
| 167 | |
| 168 | """ |
| 169 | @functools.wraps(fxn) |
| 170 | def set_loader_wrapper(self, *args, **kwargs): |
| 171 | warnings.warn('The import system now takes care of this automatically.', |
| 172 | DeprecationWarning, stacklevel=2) |
| 173 | module = fxn(self, *args, **kwargs) |
| 174 | if getattr(module, '__loader__', None) is None: |
| 175 | module.__loader__ = self |
| 176 | return module |
| 177 | return set_loader_wrapper |
| 178 | |
| 179 | |
| 180 | def module_for_loader(fxn): |
| 181 | """Decorator to handle selecting the proper module for loaders. |
| 182 | |
| 183 | The decorated function is passed the module to use instead of the module |
| 184 | name. The module passed in to the function is either from sys.modules if |
| 185 | it already exists or is a new module. If the module is new, then __name__ |
| 186 | is set the first argument to the method, __loader__ is set to self, and |
| 187 | __package__ is set accordingly (if self.is_package() is defined) will be set |
| 188 | before it is passed to the decorated function (if self.is_package() does |
| 189 | not work for the module it will be set post-load). |
| 190 | |
| 191 | If an exception is raised and the decorator created the module it is |
| 192 | subsequently removed from sys.modules. |
| 193 | |
| 194 | The decorator assumes that the decorated function takes the module name as |
| 195 | the second argument. |
| 196 | |
| 197 | """ |
| 198 | warnings.warn('The import system now takes care of this automatically.', |
| 199 | DeprecationWarning, stacklevel=2) |
| 200 | @functools.wraps(fxn) |
| 201 | def module_for_loader_wrapper(self, fullname, *args, **kwargs): |
| 202 | with _module_to_load(fullname) as module: |
| 203 | module.__loader__ = self |
| 204 | try: |
| 205 | is_package = self.is_package(fullname) |
| 206 | except (ImportError, AttributeError): |
| 207 | pass |
| 208 | else: |
| 209 | if is_package: |
| 210 | module.__package__ = fullname |
| 211 | else: |
| 212 | module.__package__ = fullname.rpartition('.')[0] |
| 213 | # If __package__ was not set above, __import__() will do it later. |
| 214 | return fxn(self, module, *args, **kwargs) |
| 215 | |
| 216 | return module_for_loader_wrapper |
| 217 | |
| 218 | |
| 219 | class _LazyModule(types.ModuleType): |
| 220 | |
| 221 | """A subclass of the module type which triggers loading upon attribute access.""" |
| 222 | |
| 223 | def __getattribute__(self, attr): |
| 224 | """Trigger the load of the module and return the attribute.""" |
| 225 | # All module metadata must be garnered from __spec__ in order to avoid |
| 226 | # using mutated values. |
| 227 | # Stop triggering this method. |
| 228 | self.__class__ = types.ModuleType |
| 229 | # Get the original name to make sure no object substitution occurred |
| 230 | # in sys.modules. |
| 231 | original_name = self.__spec__.name |
| 232 | # Figure out exactly what attributes were mutated between the creation |
| 233 | # of the module and now. |
| 234 | attrs_then = self.__spec__.loader_state['__dict__'] |
| 235 | original_type = self.__spec__.loader_state['__class__'] |
| 236 | attrs_now = self.__dict__ |
| 237 | attrs_updated = {} |
| 238 | for key, value in attrs_now.items(): |
| 239 | # Code that set the attribute may have kept a reference to the |
| 240 | # assigned object, making identity more important than equality. |
| 241 | if key not in attrs_then: |
| 242 | attrs_updated[key] = value |
| 243 | elif id(attrs_now[key]) != id(attrs_then[key]): |
| 244 | attrs_updated[key] = value |
| 245 | self.__spec__.loader.exec_module(self) |
| 246 | # If exec_module() was used directly there is no guarantee the module |
| 247 | # object was put into sys.modules. |
| 248 | if original_name in sys.modules: |
| 249 | if id(self) != id(sys.modules[original_name]): |
| 250 | raise ValueError(f"module object for {original_name!r} " |
| 251 | "substituted in sys.modules during a lazy " |
| 252 | "load") |
| 253 | # Update after loading since that's what would happen in an eager |
| 254 | # loading situation. |
| 255 | self.__dict__.update(attrs_updated) |
| 256 | return getattr(self, attr) |
| 257 | |
| 258 | def __delattr__(self, attr): |
| 259 | """Trigger the load and then perform the deletion.""" |
| 260 | # To trigger the load and raise an exception if the attribute |
| 261 | # doesn't exist. |
| 262 | self.__getattribute__(attr) |
| 263 | delattr(self, attr) |
| 264 | |
| 265 | |
| 266 | class LazyLoader(abc.Loader): |
| 267 | |
| 268 | """A loader that creates a module which defers loading until attribute access.""" |
| 269 | |
| 270 | @staticmethod |
| 271 | def __check_eager_loader(loader): |
| 272 | if not hasattr(loader, 'exec_module'): |
| 273 | raise TypeError('loader must define exec_module()') |
| 274 | |
| 275 | @classmethod |
| 276 | def factory(cls, loader): |
| 277 | """Construct a callable which returns the eager loader made lazy.""" |
| 278 | cls.__check_eager_loader(loader) |
| 279 | return lambda *args, **kwargs: cls(loader(*args, **kwargs)) |
| 280 | |
| 281 | def __init__(self, loader): |
| 282 | self.__check_eager_loader(loader) |
| 283 | self.loader = loader |
| 284 | |
| 285 | def create_module(self, spec): |
| 286 | return self.loader.create_module(spec) |
| 287 | |
| 288 | def exec_module(self, module): |
| 289 | """Make the module load lazily.""" |
| 290 | module.__spec__.loader = self.loader |
| 291 | module.__loader__ = self.loader |
| 292 | # Don't need to worry about deep-copying as trying to set an attribute |
| 293 | # on an object would have triggered the load, |
| 294 | # e.g. ``module.__spec__.loader = None`` would trigger a load from |
| 295 | # trying to access module.__spec__. |
| 296 | loader_state = {} |
| 297 | loader_state['__dict__'] = module.__dict__.copy() |
| 298 | loader_state['__class__'] = module.__class__ |
| 299 | module.__spec__.loader_state = loader_state |
| 300 | module.__class__ = _LazyModule |