Brett Cannon | d2e7b33 | 2009-02-17 02:45:03 +0000 | [diff] [blame] | 1 | """Utility code for constructing importers, etc.""" |
Brett Cannon | a04dbe4 | 2014-04-04 13:53:38 -0400 | [diff] [blame] | 2 | from . import abc |
Brett Cannon | 2a17bde | 2014-05-30 14:55:29 -0400 | [diff] [blame] | 3 | from ._bootstrap import module_from_spec |
Brett Cannon | d200bf5 | 2012-05-13 13:45:09 -0400 | [diff] [blame] | 4 | from ._bootstrap import _resolve_name |
Eric Snow | 32439d6 | 2015-05-02 19:15:18 -0600 | [diff] [blame] | 5 | from ._bootstrap import spec_from_loader |
Eric Snow | 6029e08 | 2014-01-25 15:32:46 -0700 | [diff] [blame] | 6 | from ._bootstrap import _find_spec |
Eric Snow | 32439d6 | 2015-05-02 19:15:18 -0600 | [diff] [blame] | 7 | from ._bootstrap_external import MAGIC_NUMBER |
| 8 | from ._bootstrap_external import cache_from_source |
| 9 | from ._bootstrap_external import decode_source |
| 10 | from ._bootstrap_external import source_from_cache |
| 11 | from ._bootstrap_external import spec_from_file_location |
Brett Cannon | d200bf5 | 2012-05-13 13:45:09 -0400 | [diff] [blame] | 12 | |
Eric Snow | b523f84 | 2013-11-22 09:05:39 -0700 | [diff] [blame] | 13 | from contextlib import contextmanager |
Brett Cannon | 0dbb4c7 | 2013-05-31 18:56:47 -0400 | [diff] [blame] | 14 | import functools |
Eric Snow | b523f84 | 2013-11-22 09:05:39 -0700 | [diff] [blame] | 15 | import sys |
Brett Cannon | a04dbe4 | 2014-04-04 13:53:38 -0400 | [diff] [blame] | 16 | import types |
Brett Cannon | 0dbb4c7 | 2013-05-31 18:56:47 -0400 | [diff] [blame] | 17 | import warnings |
| 18 | |
Brett Cannon | d200bf5 | 2012-05-13 13:45:09 -0400 | [diff] [blame] | 19 | |
| 20 | def resolve_name(name, package): |
| 21 | """Resolve a relative module name to an absolute one.""" |
| 22 | if not name.startswith('.'): |
| 23 | return name |
| 24 | elif not package: |
| 25 | raise ValueError('{!r} is not a relative name ' |
| 26 | '(no leading dot)'.format(name)) |
| 27 | level = 0 |
| 28 | for character in name: |
| 29 | if character != '.': |
| 30 | break |
| 31 | level += 1 |
| 32 | return _resolve_name(name[level:], package, level) |
Brett Cannon | 0dbb4c7 | 2013-05-31 18:56:47 -0400 | [diff] [blame] | 33 | |
| 34 | |
Eric Snow | 6029e08 | 2014-01-25 15:32:46 -0700 | [diff] [blame] | 35 | def _find_spec_from_path(name, path=None): |
| 36 | """Return the spec for the specified module. |
| 37 | |
| 38 | First, sys.modules is checked to see if the module was already imported. If |
| 39 | so, then sys.modules[name].__spec__ is returned. If that happens to be |
| 40 | set to None, then ValueError is raised. If the module is not in |
| 41 | sys.modules, then sys.meta_path is searched for a suitable spec with the |
| 42 | value of 'path' given to the finders. None is returned if no spec could |
| 43 | be found. |
| 44 | |
| 45 | Dotted names do not have their parent packages implicitly imported. You will |
| 46 | most likely need to explicitly import all parent packages in the proper |
| 47 | order for a submodule to get the correct spec. |
| 48 | |
| 49 | """ |
| 50 | if name not in sys.modules: |
| 51 | return _find_spec(name, path) |
| 52 | else: |
| 53 | module = sys.modules[name] |
| 54 | if module is None: |
| 55 | return None |
| 56 | try: |
| 57 | spec = module.__spec__ |
| 58 | except AttributeError: |
Serhiy Storchaka | c446405 | 2014-11-21 20:33:57 +0200 | [diff] [blame] | 59 | raise ValueError('{}.__spec__ is not set'.format(name)) from None |
Eric Snow | 6029e08 | 2014-01-25 15:32:46 -0700 | [diff] [blame] | 60 | else: |
| 61 | if spec is None: |
| 62 | raise ValueError('{}.__spec__ is None'.format(name)) |
| 63 | return spec |
| 64 | |
| 65 | |
| 66 | def find_spec(name, package=None): |
| 67 | """Return the spec for the specified module. |
| 68 | |
| 69 | First, sys.modules is checked to see if the module was already imported. If |
| 70 | so, then sys.modules[name].__spec__ is returned. If that happens to be |
| 71 | set to None, then ValueError is raised. If the module is not in |
| 72 | sys.modules, then sys.meta_path is searched for a suitable spec with the |
| 73 | value of 'path' given to the finders. None is returned if no spec could |
| 74 | be found. |
| 75 | |
| 76 | If the name is for submodule (contains a dot), the parent module is |
| 77 | automatically imported. |
| 78 | |
| 79 | The name and package arguments work the same as importlib.import_module(). |
| 80 | In other words, relative module names (with leading dots) work. |
| 81 | |
| 82 | """ |
| 83 | fullname = resolve_name(name, package) if name.startswith('.') else name |
| 84 | if fullname not in sys.modules: |
| 85 | parent_name = fullname.rpartition('.')[0] |
| 86 | if parent_name: |
| 87 | # Use builtins.__import__() in case someone replaced it. |
| 88 | parent = __import__(parent_name, fromlist=['__path__']) |
| 89 | return _find_spec(fullname, parent.__path__) |
| 90 | else: |
| 91 | return _find_spec(fullname, None) |
| 92 | else: |
| 93 | module = sys.modules[fullname] |
| 94 | if module is None: |
| 95 | return None |
| 96 | try: |
| 97 | spec = module.__spec__ |
| 98 | except AttributeError: |
Serhiy Storchaka | c446405 | 2014-11-21 20:33:57 +0200 | [diff] [blame] | 99 | raise ValueError('{}.__spec__ is not set'.format(name)) from None |
Eric Snow | 6029e08 | 2014-01-25 15:32:46 -0700 | [diff] [blame] | 100 | else: |
| 101 | if spec is None: |
| 102 | raise ValueError('{}.__spec__ is None'.format(name)) |
| 103 | return spec |
| 104 | |
| 105 | |
Eric Snow | b523f84 | 2013-11-22 09:05:39 -0700 | [diff] [blame] | 106 | @contextmanager |
| 107 | def _module_to_load(name): |
| 108 | is_reload = name in sys.modules |
| 109 | |
| 110 | module = sys.modules.get(name) |
| 111 | if not is_reload: |
| 112 | # This must be done before open() is called as the 'io' module |
| 113 | # implicitly imports 'locale' and would otherwise trigger an |
| 114 | # infinite loop. |
| 115 | module = type(sys)(name) |
| 116 | # This must be done before putting the module in sys.modules |
| 117 | # (otherwise an optimization shortcut in import.c becomes wrong) |
| 118 | module.__initializing__ = True |
| 119 | sys.modules[name] = module |
| 120 | try: |
| 121 | yield module |
| 122 | except Exception: |
| 123 | if not is_reload: |
| 124 | try: |
| 125 | del sys.modules[name] |
| 126 | except KeyError: |
| 127 | pass |
| 128 | finally: |
| 129 | module.__initializing__ = False |
| 130 | |
| 131 | |
Eric Snow | b523f84 | 2013-11-22 09:05:39 -0700 | [diff] [blame] | 132 | def set_package(fxn): |
Eric Snow | 1500d49 | 2014-01-06 20:49:04 -0700 | [diff] [blame] | 133 | """Set __package__ on the returned module. |
| 134 | |
| 135 | This function is deprecated. |
| 136 | |
| 137 | """ |
Eric Snow | b523f84 | 2013-11-22 09:05:39 -0700 | [diff] [blame] | 138 | @functools.wraps(fxn) |
| 139 | def set_package_wrapper(*args, **kwargs): |
Eric Snow | 1500d49 | 2014-01-06 20:49:04 -0700 | [diff] [blame] | 140 | warnings.warn('The import system now takes care of this automatically.', |
| 141 | DeprecationWarning, stacklevel=2) |
Eric Snow | b523f84 | 2013-11-22 09:05:39 -0700 | [diff] [blame] | 142 | module = fxn(*args, **kwargs) |
| 143 | if getattr(module, '__package__', None) is None: |
| 144 | module.__package__ = module.__name__ |
| 145 | if not hasattr(module, '__path__'): |
| 146 | module.__package__ = module.__package__.rpartition('.')[0] |
| 147 | return module |
| 148 | return set_package_wrapper |
| 149 | |
| 150 | |
Eric Snow | b523f84 | 2013-11-22 09:05:39 -0700 | [diff] [blame] | 151 | def set_loader(fxn): |
Eric Snow | 1500d49 | 2014-01-06 20:49:04 -0700 | [diff] [blame] | 152 | """Set __loader__ on the returned module. |
| 153 | |
| 154 | This function is deprecated. |
| 155 | |
| 156 | """ |
Eric Snow | b523f84 | 2013-11-22 09:05:39 -0700 | [diff] [blame] | 157 | @functools.wraps(fxn) |
| 158 | def set_loader_wrapper(self, *args, **kwargs): |
Eric Snow | 1500d49 | 2014-01-06 20:49:04 -0700 | [diff] [blame] | 159 | warnings.warn('The import system now takes care of this automatically.', |
| 160 | DeprecationWarning, stacklevel=2) |
Eric Snow | b523f84 | 2013-11-22 09:05:39 -0700 | [diff] [blame] | 161 | module = fxn(self, *args, **kwargs) |
| 162 | if getattr(module, '__loader__', None) is None: |
| 163 | module.__loader__ = self |
| 164 | return module |
| 165 | return set_loader_wrapper |
| 166 | |
| 167 | |
Brett Cannon | 0dbb4c7 | 2013-05-31 18:56:47 -0400 | [diff] [blame] | 168 | def module_for_loader(fxn): |
| 169 | """Decorator to handle selecting the proper module for loaders. |
| 170 | |
| 171 | The decorated function is passed the module to use instead of the module |
| 172 | name. The module passed in to the function is either from sys.modules if |
| 173 | it already exists or is a new module. If the module is new, then __name__ |
| 174 | is set the first argument to the method, __loader__ is set to self, and |
| 175 | __package__ is set accordingly (if self.is_package() is defined) will be set |
| 176 | before it is passed to the decorated function (if self.is_package() does |
| 177 | not work for the module it will be set post-load). |
| 178 | |
| 179 | If an exception is raised and the decorator created the module it is |
| 180 | subsequently removed from sys.modules. |
| 181 | |
| 182 | The decorator assumes that the decorated function takes the module name as |
| 183 | the second argument. |
| 184 | |
| 185 | """ |
Eric Snow | b523f84 | 2013-11-22 09:05:39 -0700 | [diff] [blame] | 186 | warnings.warn('The import system now takes care of this automatically.', |
Eric Snow | 1500d49 | 2014-01-06 20:49:04 -0700 | [diff] [blame] | 187 | DeprecationWarning, stacklevel=2) |
Brett Cannon | 0dbb4c7 | 2013-05-31 18:56:47 -0400 | [diff] [blame] | 188 | @functools.wraps(fxn) |
| 189 | def module_for_loader_wrapper(self, fullname, *args, **kwargs): |
Eric Snow | b523f84 | 2013-11-22 09:05:39 -0700 | [diff] [blame] | 190 | with _module_to_load(fullname) as module: |
Brett Cannon | 0dbb4c7 | 2013-05-31 18:56:47 -0400 | [diff] [blame] | 191 | module.__loader__ = self |
| 192 | try: |
| 193 | is_package = self.is_package(fullname) |
| 194 | except (ImportError, AttributeError): |
| 195 | pass |
| 196 | else: |
| 197 | if is_package: |
| 198 | module.__package__ = fullname |
| 199 | else: |
| 200 | module.__package__ = fullname.rpartition('.')[0] |
| 201 | # If __package__ was not set above, __import__() will do it later. |
| 202 | return fxn(self, module, *args, **kwargs) |
| 203 | |
Brett Cannon | 0e75c06 | 2013-05-31 18:57:45 -0400 | [diff] [blame] | 204 | return module_for_loader_wrapper |
Brett Cannon | a04dbe4 | 2014-04-04 13:53:38 -0400 | [diff] [blame] | 205 | |
| 206 | |
| 207 | class _Module(types.ModuleType): |
| 208 | |
| 209 | """A subclass of the module type to allow __class__ manipulation.""" |
| 210 | |
| 211 | |
| 212 | class _LazyModule(types.ModuleType): |
| 213 | |
| 214 | """A subclass of the module type which triggers loading upon attribute access.""" |
| 215 | |
| 216 | def __getattribute__(self, attr): |
| 217 | """Trigger the load of the module and return the attribute.""" |
| 218 | # All module metadata must be garnered from __spec__ in order to avoid |
| 219 | # using mutated values. |
| 220 | # Stop triggering this method. |
| 221 | self.__class__ = _Module |
| 222 | # Get the original name to make sure no object substitution occurred |
| 223 | # in sys.modules. |
| 224 | original_name = self.__spec__.name |
| 225 | # Figure out exactly what attributes were mutated between the creation |
| 226 | # of the module and now. |
| 227 | attrs_then = self.__spec__.loader_state |
| 228 | attrs_now = self.__dict__ |
| 229 | attrs_updated = {} |
| 230 | for key, value in attrs_now.items(): |
| 231 | # Code that set the attribute may have kept a reference to the |
| 232 | # assigned object, making identity more important than equality. |
| 233 | if key not in attrs_then: |
| 234 | attrs_updated[key] = value |
| 235 | elif id(attrs_now[key]) != id(attrs_then[key]): |
| 236 | attrs_updated[key] = value |
| 237 | self.__spec__.loader.exec_module(self) |
| 238 | # If exec_module() was used directly there is no guarantee the module |
| 239 | # object was put into sys.modules. |
| 240 | if original_name in sys.modules: |
| 241 | if id(self) != id(sys.modules[original_name]): |
| 242 | msg = ('module object for {!r} substituted in sys.modules ' |
| 243 | 'during a lazy load') |
| 244 | raise ValueError(msg.format(original_name)) |
| 245 | # Update after loading since that's what would happen in an eager |
| 246 | # loading situation. |
| 247 | self.__dict__.update(attrs_updated) |
| 248 | return getattr(self, attr) |
| 249 | |
| 250 | def __delattr__(self, attr): |
| 251 | """Trigger the load and then perform the deletion.""" |
| 252 | # To trigger the load and raise an exception if the attribute |
| 253 | # doesn't exist. |
| 254 | self.__getattribute__(attr) |
| 255 | delattr(self, attr) |
| 256 | |
| 257 | |
| 258 | class LazyLoader(abc.Loader): |
| 259 | |
| 260 | """A loader that creates a module which defers loading until attribute access.""" |
| 261 | |
| 262 | @staticmethod |
| 263 | def __check_eager_loader(loader): |
| 264 | if not hasattr(loader, 'exec_module'): |
| 265 | raise TypeError('loader must define exec_module()') |
| 266 | elif hasattr(loader.__class__, 'create_module'): |
| 267 | if abc.Loader.create_module != loader.__class__.create_module: |
| 268 | # Only care if create_module() is overridden in a subclass of |
| 269 | # importlib.abc.Loader. |
| 270 | raise TypeError('loader cannot define create_module()') |
| 271 | |
| 272 | @classmethod |
| 273 | def factory(cls, loader): |
| 274 | """Construct a callable which returns the eager loader made lazy.""" |
| 275 | cls.__check_eager_loader(loader) |
| 276 | return lambda *args, **kwargs: cls(loader(*args, **kwargs)) |
| 277 | |
| 278 | def __init__(self, loader): |
| 279 | self.__check_eager_loader(loader) |
| 280 | self.loader = loader |
| 281 | |
| 282 | def create_module(self, spec): |
| 283 | """Create a module which can have its __class__ manipulated.""" |
| 284 | return _Module(spec.name) |
| 285 | |
| 286 | def exec_module(self, module): |
| 287 | """Make the module load lazily.""" |
| 288 | module.__spec__.loader = self.loader |
| 289 | module.__loader__ = self.loader |
| 290 | # Don't need to worry about deep-copying as trying to set an attribute |
| 291 | # on an object would have triggered the load, |
| 292 | # e.g. ``module.__spec__.loader = None`` would trigger a load from |
| 293 | # trying to access module.__spec__. |
| 294 | module.__spec__.loader_state = module.__dict__.copy() |
| 295 | module.__class__ = _LazyModule |