blob: 3d7f19f39981de689f861418859e364f5f374211 [file] [log] [blame]
Guido van Rossuma4deda02002-12-23 16:30:00 +00001"""Utilities to support packages."""
2
Eric Snowd5f92232016-09-07 18:37:17 -07003from collections import namedtuple
Łukasz Langa6f692512013-06-05 12:20:24 +02004from functools import singledispatch as simplegeneric
Łukasz Langa6f692512013-06-05 12:20:24 +02005import importlib
Brett Cannone4f41de2013-06-16 13:13:40 -04006import importlib.util
Nick Coghlan862542e2013-10-27 00:27:39 +10007import importlib.machinery
Łukasz Langa6f692512013-06-05 12:20:24 +02008import os
Thomas Wouters49fd7fa2006-04-21 10:40:58 +00009import os.path
Łukasz Langa6f692512013-06-05 12:20:24 +020010import sys
Thomas Wouters49fd7fa2006-04-21 10:40:58 +000011from types import ModuleType
Brett Cannone4f41de2013-06-16 13:13:40 -040012import warnings
Thomas Wouters49fd7fa2006-04-21 10:40:58 +000013
14__all__ = [
15 'get_importer', 'iter_importers', 'get_loader', 'find_loader',
Éric Araujoa4e2d4f2011-05-02 22:59:15 +020016 'walk_packages', 'iter_modules', 'get_data',
Thomas Wouters49fd7fa2006-04-21 10:40:58 +000017 'ImpImporter', 'ImpLoader', 'read_code', 'extend_path',
Eric Snowd5f92232016-09-07 18:37:17 -070018 'ModuleInfo',
Thomas Wouters49fd7fa2006-04-21 10:40:58 +000019]
20
Eric Snow37148b22014-01-04 15:09:53 -070021
Eric Snowd5f92232016-09-07 18:37:17 -070022ModuleInfo = namedtuple('ModuleInfo', 'module_finder name ispkg')
23ModuleInfo.__doc__ = 'A namedtuple with minimal info about a module.'
24
25
Eric Snow37148b22014-01-04 15:09:53 -070026def _get_spec(finder, name):
27 """Return the finder-specific module spec."""
28 # Works with legacy finders.
29 try:
30 find_spec = finder.find_spec
31 except AttributeError:
32 loader = finder.find_module(name)
33 if loader is None:
34 return None
35 return importlib.util.spec_from_loader(name, loader)
36 else:
37 return find_spec(name)
38
39
Thomas Wouters49fd7fa2006-04-21 10:40:58 +000040def read_code(stream):
41 # This helper is needed in order for the PEP 302 emulation to
42 # correctly handle compiled files
43 import marshal
44
45 magic = stream.read(4)
Brett Cannone4f41de2013-06-16 13:13:40 -040046 if magic != importlib.util.MAGIC_NUMBER:
Thomas Wouters49fd7fa2006-04-21 10:40:58 +000047 return None
48
Benjamin Peterson42aa93b2017-12-09 10:26:52 -080049 stream.read(12) # Skip rest of the header
Thomas Wouters49fd7fa2006-04-21 10:40:58 +000050 return marshal.load(stream)
51
52
Thomas Wouters49fd7fa2006-04-21 10:40:58 +000053def walk_packages(path=None, prefix='', onerror=None):
Eric Snowd5f92232016-09-07 18:37:17 -070054 """Yields ModuleInfo for all modules recursively
Thomas Wouters0e3f5912006-08-11 14:57:12 +000055 on path, or, if path is None, all accessible modules.
56
57 'path' should be either None or a list of paths to look for
58 modules in.
59
60 'prefix' is a string to output on the front of every module name
61 on output.
62
63 Note that this function must import all *packages* (NOT all
64 modules!) on the given path, in order to access the __path__
65 attribute to find submodules.
66
67 'onerror' is a function which gets called with one argument (the
68 name of the package which was being imported) if any exception
69 occurs while trying to import a package. If no onerror function is
70 supplied, ImportErrors are caught and ignored, while all other
71 exceptions are propagated, terminating the search.
72
73 Examples:
74
75 # list all modules python can access
76 walk_packages()
77
78 # list all submodules of ctypes
79 walk_packages(ctypes.__path__, ctypes.__name__+'.')
80 """
Thomas Wouters49fd7fa2006-04-21 10:40:58 +000081
Thomas Wouters477c8d52006-05-27 19:21:47 +000082 def seen(p, m={}):
83 if p in m:
84 return True
Thomas Wouters49fd7fa2006-04-21 10:40:58 +000085 m[p] = True
86
Eric Snowd5f92232016-09-07 18:37:17 -070087 for info in iter_modules(path, prefix):
88 yield info
Thomas Wouters49fd7fa2006-04-21 10:40:58 +000089
Eric Snowd5f92232016-09-07 18:37:17 -070090 if info.ispkg:
Thomas Wouters49fd7fa2006-04-21 10:40:58 +000091 try:
Eric Snowd5f92232016-09-07 18:37:17 -070092 __import__(info.name)
Thomas Wouters49fd7fa2006-04-21 10:40:58 +000093 except ImportError:
94 if onerror is not None:
Eric Snowd5f92232016-09-07 18:37:17 -070095 onerror(info.name)
Thomas Wouters0e3f5912006-08-11 14:57:12 +000096 except Exception:
97 if onerror is not None:
Eric Snowd5f92232016-09-07 18:37:17 -070098 onerror(info.name)
Thomas Wouters0e3f5912006-08-11 14:57:12 +000099 else:
100 raise
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000101 else:
Eric Snowd5f92232016-09-07 18:37:17 -0700102 path = getattr(sys.modules[info.name], '__path__', None) or []
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000103
104 # don't traverse path items we've seen before
105 path = [p for p in path if not seen(p)]
106
Eric Snowd5f92232016-09-07 18:37:17 -0700107 yield from walk_packages(path, info.name+'.', onerror)
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000108
109
110def iter_modules(path=None, prefix=''):
Eric Snowd5f92232016-09-07 18:37:17 -0700111 """Yields ModuleInfo for all submodules on path,
Thomas Wouters0e3f5912006-08-11 14:57:12 +0000112 or, if path is None, all top-level modules on sys.path.
113
114 'path' should be either None or a list of paths to look for
115 modules in.
116
117 'prefix' is a string to output on the front of every module name
118 on output.
119 """
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000120 if path is None:
121 importers = iter_importers()
Sanyam Khuranab9c3da52017-06-13 22:41:14 +0530122 elif isinstance(path, str):
123 raise ValueError("path must be None or list of paths to look for "
124 "modules in")
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000125 else:
126 importers = map(get_importer, path)
127
128 yielded = {}
129 for i in importers:
130 for name, ispkg in iter_importer_modules(i, prefix):
131 if name not in yielded:
132 yielded[name] = 1
Eric Snowd5f92232016-09-07 18:37:17 -0700133 yield ModuleInfo(i, name, ispkg)
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000134
135
Łukasz Langa6f692512013-06-05 12:20:24 +0200136@simplegeneric
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000137def iter_importer_modules(importer, prefix=''):
Thomas Wouters477c8d52006-05-27 19:21:47 +0000138 if not hasattr(importer, 'iter_modules'):
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000139 return []
140 return importer.iter_modules(prefix)
141
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000142
Nick Coghlan8ecf5042012-07-15 21:19:18 +1000143# Implement a file walker for the normal importlib path hook
144def _iter_file_finder_modules(importer, prefix=''):
145 if importer.path is None or not os.path.isdir(importer.path):
146 return
147
148 yielded = {}
149 import inspect
150 try:
151 filenames = os.listdir(importer.path)
152 except OSError:
153 # ignore unreadable directories like import does
154 filenames = []
155 filenames.sort() # handle packages before same-named modules
156
157 for fn in filenames:
158 modname = inspect.getmodulename(fn)
159 if modname=='__init__' or modname in yielded:
160 continue
161
162 path = os.path.join(importer.path, fn)
163 ispkg = False
164
165 if not modname and os.path.isdir(path) and '.' not in fn:
166 modname = fn
167 try:
168 dircontents = os.listdir(path)
169 except OSError:
170 # ignore unreadable directories like import does
171 dircontents = []
172 for fn in dircontents:
173 subname = inspect.getmodulename(fn)
174 if subname=='__init__':
175 ispkg = True
176 break
177 else:
178 continue # not a package
179
180 if modname and '.' not in modname:
181 yielded[modname] = 1
182 yield prefix + modname, ispkg
183
184iter_importer_modules.register(
185 importlib.machinery.FileFinder, _iter_file_finder_modules)
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000186
Brett Cannone4f41de2013-06-16 13:13:40 -0400187
188def _import_imp():
189 global imp
190 with warnings.catch_warnings():
Brett Cannonc0d91af2015-10-16 12:21:37 -0700191 warnings.simplefilter('ignore', DeprecationWarning)
Brett Cannone4f41de2013-06-16 13:13:40 -0400192 imp = importlib.import_module('imp')
193
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000194class ImpImporter:
Brett Cannonfdcdd9e2016-07-08 11:00:00 -0700195 """PEP 302 Finder that wraps Python's "classic" import algorithm
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000196
Brett Cannonfdcdd9e2016-07-08 11:00:00 -0700197 ImpImporter(dirname) produces a PEP 302 finder that searches that
198 directory. ImpImporter(None) produces a PEP 302 finder that searches
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000199 the current sys.path, plus any modules that are frozen or built-in.
200
201 Note that ImpImporter does not currently support being used by placement
202 on sys.meta_path.
203 """
204
205 def __init__(self, path=None):
Brett Cannone4f41de2013-06-16 13:13:40 -0400206 global imp
207 warnings.warn("This emulation is deprecated, use 'importlib' instead",
Nick Coghlan85e729e2012-07-15 18:09:52 +1000208 DeprecationWarning)
Brett Cannone4f41de2013-06-16 13:13:40 -0400209 _import_imp()
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000210 self.path = path
211
212 def find_module(self, fullname, path=None):
213 # Note: we ignore 'path' argument since it is only used via meta_path
214 subname = fullname.split(".")[-1]
215 if subname != fullname and self.path is None:
216 return None
217 if self.path is None:
218 path = None
219 else:
220 path = [os.path.realpath(self.path)]
221 try:
222 file, filename, etc = imp.find_module(subname, path)
223 except ImportError:
224 return None
225 return ImpLoader(fullname, file, filename, etc)
226
227 def iter_modules(self, prefix=''):
228 if self.path is None or not os.path.isdir(self.path):
229 return
230
231 yielded = {}
232 import inspect
Ned Deilyed27df72011-10-06 14:19:08 -0700233 try:
234 filenames = os.listdir(self.path)
235 except OSError:
236 # ignore unreadable directories like import does
237 filenames = []
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000238 filenames.sort() # handle packages before same-named modules
239
240 for fn in filenames:
241 modname = inspect.getmodulename(fn)
242 if modname=='__init__' or modname in yielded:
243 continue
244
245 path = os.path.join(self.path, fn)
246 ispkg = False
247
248 if not modname and os.path.isdir(path) and '.' not in fn:
249 modname = fn
Ned Deilyed27df72011-10-06 14:19:08 -0700250 try:
251 dircontents = os.listdir(path)
252 except OSError:
253 # ignore unreadable directories like import does
254 dircontents = []
255 for fn in dircontents:
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000256 subname = inspect.getmodulename(fn)
257 if subname=='__init__':
258 ispkg = True
259 break
260 else:
261 continue # not a package
262
263 if modname and '.' not in modname:
264 yielded[modname] = 1
265 yield prefix + modname, ispkg
266
267
268class ImpLoader:
269 """PEP 302 Loader that wraps Python's "classic" import algorithm
270 """
271 code = source = None
272
273 def __init__(self, fullname, file, filename, etc):
Brett Cannone4f41de2013-06-16 13:13:40 -0400274 warnings.warn("This emulation is deprecated, use 'importlib' instead",
275 DeprecationWarning)
276 _import_imp()
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000277 self.file = file
278 self.filename = filename
279 self.fullname = fullname
280 self.etc = etc
281
282 def load_module(self, fullname):
283 self._reopen()
284 try:
285 mod = imp.load_module(fullname, self.file, self.filename, self.etc)
286 finally:
287 if self.file:
288 self.file.close()
289 # Note: we don't set __loader__ because we want the module to look
290 # normal; i.e. this is just a wrapper for standard import machinery
291 return mod
292
293 def get_data(self, pathname):
Brett Cannon1ab58df2010-10-29 22:36:53 +0000294 with open(pathname, "rb") as file:
295 return file.read()
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000296
297 def _reopen(self):
298 if self.file and self.file.closed:
Thomas Wouters0e3f5912006-08-11 14:57:12 +0000299 mod_type = self.etc[2]
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000300 if mod_type==imp.PY_SOURCE:
Victor Stinner4e86d5b2011-05-04 13:55:36 +0200301 self.file = open(self.filename, 'r')
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000302 elif mod_type in (imp.PY_COMPILED, imp.C_EXTENSION):
303 self.file = open(self.filename, 'rb')
304
305 def _fix_name(self, fullname):
306 if fullname is None:
307 fullname = self.fullname
308 elif fullname != self.fullname:
309 raise ImportError("Loader for module %s cannot handle "
310 "module %s" % (self.fullname, fullname))
311 return fullname
312
313 def is_package(self, fullname):
314 fullname = self._fix_name(fullname)
315 return self.etc[2]==imp.PKG_DIRECTORY
316
317 def get_code(self, fullname=None):
318 fullname = self._fix_name(fullname)
319 if self.code is None:
320 mod_type = self.etc[2]
321 if mod_type==imp.PY_SOURCE:
322 source = self.get_source(fullname)
323 self.code = compile(source, self.filename, 'exec')
324 elif mod_type==imp.PY_COMPILED:
325 self._reopen()
326 try:
327 self.code = read_code(self.file)
328 finally:
329 self.file.close()
330 elif mod_type==imp.PKG_DIRECTORY:
331 self.code = self._get_delegate().get_code()
332 return self.code
333
334 def get_source(self, fullname=None):
335 fullname = self._fix_name(fullname)
336 if self.source is None:
337 mod_type = self.etc[2]
338 if mod_type==imp.PY_SOURCE:
339 self._reopen()
340 try:
341 self.source = self.file.read()
342 finally:
343 self.file.close()
344 elif mod_type==imp.PY_COMPILED:
345 if os.path.exists(self.filename[:-1]):
Giampaolo Rodola'2f50aaf2013-02-12 02:04:27 +0100346 with open(self.filename[:-1], 'r') as f:
347 self.source = f.read()
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000348 elif mod_type==imp.PKG_DIRECTORY:
349 self.source = self._get_delegate().get_source()
350 return self.source
351
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000352 def _get_delegate(self):
Eric Snow37148b22014-01-04 15:09:53 -0700353 finder = ImpImporter(self.filename)
354 spec = _get_spec(finder, '__init__')
355 return spec.loader
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000356
357 def get_filename(self, fullname=None):
358 fullname = self._fix_name(fullname)
359 mod_type = self.etc[2]
Éric Araujo0cfb81d2011-09-17 03:35:57 +0200360 if mod_type==imp.PKG_DIRECTORY:
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000361 return self._get_delegate().get_filename()
Éric Araujo0cfb81d2011-09-17 03:35:57 +0200362 elif mod_type in (imp.PY_SOURCE, imp.PY_COMPILED, imp.C_EXTENSION):
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000363 return self.filename
364 return None
365
366
367try:
368 import zipimport
369 from zipimport import zipimporter
370
371 def iter_zipimport_modules(importer, prefix=''):
Alexandre Vassalotti515a74f2009-07-05 06:42:44 +0000372 dirlist = sorted(zipimport._zip_directory_cache[importer.archive])
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000373 _prefix = importer.prefix
374 plen = len(_prefix)
375 yielded = {}
376 import inspect
377 for fn in dirlist:
378 if not fn.startswith(_prefix):
379 continue
380
381 fn = fn[plen:].split(os.sep)
382
383 if len(fn)==2 and fn[1].startswith('__init__.py'):
384 if fn[0] not in yielded:
385 yielded[fn[0]] = 1
Łukasz Langa0d18c152016-06-11 18:02:46 -0700386 yield prefix + fn[0], True
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000387
388 if len(fn)!=1:
389 continue
390
391 modname = inspect.getmodulename(fn[0])
392 if modname=='__init__':
393 continue
394
395 if modname and '.' not in modname and modname not in yielded:
396 yielded[modname] = 1
397 yield prefix + modname, False
398
399 iter_importer_modules.register(zipimporter, iter_zipimport_modules)
400
401except ImportError:
402 pass
403
404
405def get_importer(path_item):
Senthil Kumaran46720602016-09-05 17:11:51 -0700406 """Retrieve a finder for the given path item
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000407
Brett Cannonfdcdd9e2016-07-08 11:00:00 -0700408 The returned finder is cached in sys.path_importer_cache
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000409 if it was newly created by a path hook.
410
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000411 The cache (or part of it) can be cleared manually if a
412 rescan of sys.path_hooks is necessary.
413 """
414 try:
415 importer = sys.path_importer_cache[path_item]
416 except KeyError:
417 for path_hook in sys.path_hooks:
418 try:
419 importer = path_hook(path_item)
Brett Cannone0d88a12012-04-25 20:54:04 -0400420 sys.path_importer_cache.setdefault(path_item, importer)
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000421 break
422 except ImportError:
423 pass
424 else:
Nick Coghlan85e729e2012-07-15 18:09:52 +1000425 importer = None
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000426 return importer
427
428
429def iter_importers(fullname=""):
Senthil Kumaran46720602016-09-05 17:11:51 -0700430 """Yield finders for the given module name
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000431
Brett Cannonfdcdd9e2016-07-08 11:00:00 -0700432 If fullname contains a '.', the finders will be for the package
Nick Coghlan85e729e2012-07-15 18:09:52 +1000433 containing fullname, otherwise they will be all registered top level
Brett Cannonfdcdd9e2016-07-08 11:00:00 -0700434 finders (i.e. those on both sys.meta_path and sys.path_hooks).
Nick Coghlan85e729e2012-07-15 18:09:52 +1000435
436 If the named module is in a package, that package is imported as a side
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000437 effect of invoking this function.
438
Brett Cannonfdcdd9e2016-07-08 11:00:00 -0700439 If no module name is specified, all top level finders are produced.
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000440 """
441 if fullname.startswith('.'):
Nick Coghlan85e729e2012-07-15 18:09:52 +1000442 msg = "Relative module name {!r} not supported".format(fullname)
443 raise ImportError(msg)
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000444 if '.' in fullname:
445 # Get the containing package's __path__
Nick Coghlan85e729e2012-07-15 18:09:52 +1000446 pkg_name = fullname.rpartition(".")[0]
Nick Coghlanc4e0d982013-04-14 22:30:42 +1000447 pkg = importlib.import_module(pkg_name)
448 path = getattr(pkg, '__path__', None)
Nick Coghlan85e729e2012-07-15 18:09:52 +1000449 if path is None:
450 return
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000451 else:
Andrew Svetlov2aa5f3c2012-10-07 23:21:15 +0300452 yield from sys.meta_path
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000453 path = sys.path
454 for item in path:
455 yield get_importer(item)
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000456
Eric Snowb523f842013-11-22 09:05:39 -0700457
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000458def get_loader(module_or_name):
Senthil Kumaran46720602016-09-05 17:11:51 -0700459 """Get a "loader" object for module_or_name
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000460
Nick Coghlan85e729e2012-07-15 18:09:52 +1000461 Returns None if the module cannot be found or imported.
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000462 If the named module is not already imported, its containing package
463 (if any) is imported, in order to establish the package __path__.
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000464 """
465 if module_or_name in sys.modules:
466 module_or_name = sys.modules[module_or_name]
Brett Cannon8447c702014-05-23 12:30:37 -0400467 if module_or_name is None:
468 return None
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000469 if isinstance(module_or_name, ModuleType):
470 module = module_or_name
Thomas Wouters477c8d52006-05-27 19:21:47 +0000471 loader = getattr(module, '__loader__', None)
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000472 if loader is not None:
473 return loader
Eric Snow658af312014-04-19 00:13:23 -0600474 if getattr(module, '__spec__', None) is None:
475 return None
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000476 fullname = module.__name__
477 else:
478 fullname = module_or_name
479 return find_loader(fullname)
480
Nick Coghlan85e729e2012-07-15 18:09:52 +1000481
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000482def find_loader(fullname):
Senthil Kumaran46720602016-09-05 17:11:51 -0700483 """Find a "loader" object for fullname
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000484
Nick Coghlan62b4b9e2014-03-04 20:39:42 +1000485 This is a backwards compatibility wrapper around
486 importlib.util.find_spec that converts most failures to ImportError
487 and only returns the loader rather than the full spec
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000488 """
Nick Coghlan85e729e2012-07-15 18:09:52 +1000489 if fullname.startswith('.'):
490 msg = "Relative module name {!r} not supported".format(fullname)
491 raise ImportError(msg)
Nick Coghlan85e729e2012-07-15 18:09:52 +1000492 try:
Nick Coghlan62b4b9e2014-03-04 20:39:42 +1000493 spec = importlib.util.find_spec(fullname)
Nick Coghlan85e729e2012-07-15 18:09:52 +1000494 except (ImportError, AttributeError, TypeError, ValueError) as ex:
495 # This hack fixes an impedance mismatch between pkgutil and
Andrew Svetlov5b898402012-12-18 21:26:36 +0200496 # importlib, where the latter raises other errors for cases where
Andrew Svetlov1f415cf2012-12-19 22:54:47 +0200497 # pkgutil previously raised ImportError
Nick Coghlan85e729e2012-07-15 18:09:52 +1000498 msg = "Error while finding loader for {!r} ({}: {})"
499 raise ImportError(msg.format(fullname, type(ex), ex)) from ex
Brett Cannon8447c702014-05-23 12:30:37 -0400500 return spec.loader if spec is not None else None
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000501
Guido van Rossuma4deda02002-12-23 16:30:00 +0000502
503def extend_path(path, name):
504 """Extend a package's path.
505
506 Intended use is to place the following code in a package's __init__.py:
507
508 from pkgutil import extend_path
509 __path__ = extend_path(__path__, __name__)
510
511 This will add to the package's __path__ all subdirectories of
512 directories on sys.path named after the package. This is useful
513 if one wants to distribute different parts of a single logical
514 package as multiple directories.
515
516 It also looks for *.pkg files beginning where * matches the name
517 argument. This feature is similar to *.pth files (see site.py),
518 except that it doesn't special-case lines starting with 'import'.
519 A *.pkg file is trusted at face value: apart from checking for
520 duplicates, all entries found in a *.pkg file are added to the
521 path, regardless of whether they are exist the filesystem. (This
522 is a feature.)
523
524 If the input path is not a list (as is the case for frozen
525 packages) it is returned unchanged. The input path is not
526 modified; an extended copy is returned. Items are only appended
527 to the copy at the end.
528
529 It is assumed that sys.path is a sequence. Items of sys.path that
530 are not (unicode or 8-bit) strings referring to existing
531 directories are ignored. Unicode items of sys.path that cause
532 errors when used as filenames may cause this function to raise an
533 exception (in line with os.path.isdir() behavior).
534 """
535
536 if not isinstance(path, list):
537 # This could happen e.g. when this is called from inside a
538 # frozen package. Return the path unchanged in that case.
539 return path
540
Skip Montanaro7a98be22007-08-16 14:35:24 +0000541 sname_pkg = name + ".pkg"
Guido van Rossuma4deda02002-12-23 16:30:00 +0000542
543 path = path[:] # Start with a copy of the existing path
544
Antoine Pitroub2dd8802012-07-09 21:23:58 +0200545 parent_package, _, final_name = name.rpartition('.')
546 if parent_package:
547 try:
548 search_path = sys.modules[parent_package].__path__
549 except (KeyError, AttributeError):
550 # We can't do anything: find_loader() returns None when
551 # passed a dotted name.
552 return path
553 else:
554 search_path = sys.path
555
556 for dir in search_path:
Eric V. Smith984b11f2012-05-24 20:21:04 -0400557 if not isinstance(dir, str):
Guido van Rossuma4deda02002-12-23 16:30:00 +0000558 continue
Eric V. Smith984b11f2012-05-24 20:21:04 -0400559
560 finder = get_importer(dir)
561 if finder is not None:
Eric Snow02b9f9d2014-01-06 20:42:59 -0700562 portions = []
563 if hasattr(finder, 'find_spec'):
564 spec = finder.find_spec(final_name)
565 if spec is not None:
566 portions = spec.submodule_search_locations or []
Eric V. Smith984b11f2012-05-24 20:21:04 -0400567 # Is this finder PEP 420 compliant?
Eric Snow02b9f9d2014-01-06 20:42:59 -0700568 elif hasattr(finder, 'find_loader'):
569 _, portions = finder.find_loader(final_name)
Eric V. Smith984b11f2012-05-24 20:21:04 -0400570
571 for portion in portions:
572 # XXX This may still add duplicate entries to path on
573 # case-insensitive filesystems
574 if portion not in path:
575 path.append(portion)
576
Guido van Rossuma4deda02002-12-23 16:30:00 +0000577 # XXX Is this the right thing for subpackages like zope.app?
578 # It looks for a file named "zope.app.pkg"
579 pkgfile = os.path.join(dir, sname_pkg)
580 if os.path.isfile(pkgfile):
581 try:
582 f = open(pkgfile)
Andrew Svetlovf7a17b42012-12-25 16:47:37 +0200583 except OSError as msg:
Guido van Rossuma4deda02002-12-23 16:30:00 +0000584 sys.stderr.write("Can't open %s: %s\n" %
585 (pkgfile, msg))
586 else:
Giampaolo Rodola'2f50aaf2013-02-12 02:04:27 +0100587 with f:
588 for line in f:
589 line = line.rstrip('\n')
590 if not line or line.startswith('#'):
591 continue
592 path.append(line) # Don't check for existence!
Guido van Rossuma4deda02002-12-23 16:30:00 +0000593
594 return path
Christian Heimesdae2a892008-04-19 00:55:37 +0000595
Eric Snowb523f842013-11-22 09:05:39 -0700596
Christian Heimesdae2a892008-04-19 00:55:37 +0000597def get_data(package, resource):
598 """Get a resource from a package.
599
600 This is a wrapper round the PEP 302 loader get_data API. The package
601 argument should be the name of a package, in standard module format
602 (foo.bar). The resource argument should be in the form of a relative
603 filename, using '/' as the path separator. The parent directory name '..'
604 is not allowed, and nor is a rooted name (starting with a '/').
605
606 The function returns a binary string, which is the contents of the
607 specified resource.
608
609 For packages located in the filesystem, which have already been imported,
610 this is the rough equivalent of
611
612 d = os.path.dirname(sys.modules[package].__file__)
613 data = open(os.path.join(d, resource), 'rb').read()
614
615 If the package cannot be located or loaded, or it uses a PEP 302 loader
616 which does not support get_data(), then None is returned.
617 """
618
Eric Snow6029e082014-01-25 15:32:46 -0700619 spec = importlib.util.find_spec(package)
Eric Snowb523f842013-11-22 09:05:39 -0700620 if spec is None:
621 return None
622 loader = spec.loader
Christian Heimesdae2a892008-04-19 00:55:37 +0000623 if loader is None or not hasattr(loader, 'get_data'):
624 return None
Eric Snowb523f842013-11-22 09:05:39 -0700625 # XXX needs test
626 mod = (sys.modules.get(package) or
Brett Cannon2a17bde2014-05-30 14:55:29 -0400627 importlib._bootstrap._load(spec))
Christian Heimesdae2a892008-04-19 00:55:37 +0000628 if mod is None or not hasattr(mod, '__file__'):
629 return None
630
631 # Modify the resource name to be compatible with the loader.get_data
632 # signature - an os.path format "filename" starting with the dirname of
633 # the package's __file__
634 parts = resource.split('/')
635 parts.insert(0, os.path.dirname(mod.__file__))
636 resource_name = os.path.join(*parts)
637 return loader.get_data(resource_name)
Vinay Sajip1ed61612020-02-14 22:02:13 +0000638
639
Victor Stinner98ce7b12020-06-17 19:11:50 +0200640_NAME_PATTERN = None
Vinay Sajip1ed61612020-02-14 22:02:13 +0000641
642def resolve_name(name):
643 """
644 Resolve a name to an object.
645
646 It is expected that `name` will be a string in one of the following
647 formats, where W is shorthand for a valid Python identifier and dot stands
648 for a literal period in these pseudo-regexes:
649
650 W(.W)*
651 W(.W)*:(W(.W)*)?
652
653 The first form is intended for backward compatibility only. It assumes that
654 some part of the dotted name is a package, and the rest is an object
655 somewhere within that package, possibly nested inside other objects.
656 Because the place where the package stops and the object hierarchy starts
657 can't be inferred by inspection, repeated attempts to import must be done
658 with this form.
659
660 In the second form, the caller makes the division point clear through the
661 provision of a single colon: the dotted name to the left of the colon is a
662 package to be imported, and the dotted name to the right is the object
663 hierarchy within that package. Only one import is needed in this form. If
664 it ends with the colon, then a module object is returned.
665
666 The function will return an object (which might be a module), or raise one
667 of the following exceptions:
668
669 ValueError - if `name` isn't in a recognised format
670 ImportError - if an import failed when it shouldn't have
671 AttributeError - if a failure occurred when traversing the object hierarchy
672 within the imported package to get to the desired object)
673 """
Victor Stinner98ce7b12020-06-17 19:11:50 +0200674 global _NAME_PATTERN
675 if _NAME_PATTERN is None:
676 # Lazy import to speedup Python startup time
677 import re
678 dotted_words = r'(?!\d)(\w+)(\.(?!\d)(\w+))*'
679 _NAME_PATTERN = re.compile(f'^(?P<pkg>{dotted_words})'
680 f'(?P<cln>:(?P<obj>{dotted_words})?)?$',
681 re.UNICODE)
682
Vinay Sajip1ed61612020-02-14 22:02:13 +0000683 m = _NAME_PATTERN.match(name)
684 if not m:
685 raise ValueError(f'invalid format: {name!r}')
Vinay Sajip4f17c5c2020-02-28 14:26:27 +0000686 gd = m.groupdict()
687 if gd.get('cln'):
Vinay Sajip1ed61612020-02-14 22:02:13 +0000688 # there is a colon - a one-step import is all that's needed
Vinay Sajip4f17c5c2020-02-28 14:26:27 +0000689 mod = importlib.import_module(gd['pkg'])
690 parts = gd.get('obj')
691 parts = parts.split('.') if parts else []
Vinay Sajip1ed61612020-02-14 22:02:13 +0000692 else:
693 # no colon - have to iterate to find the package boundary
694 parts = name.split('.')
695 modname = parts.pop(0)
696 # first part *must* be a module/package.
697 mod = importlib.import_module(modname)
698 while parts:
699 p = parts[0]
700 s = f'{modname}.{p}'
701 try:
702 mod = importlib.import_module(s)
703 parts.pop(0)
704 modname = s
705 except ImportError:
706 break
707 # if we reach this point, mod is the module, already imported, and
708 # parts is the list of parts in the object hierarchy to be traversed, or
709 # an empty list if just the module is wanted.
710 result = mod
711 for p in parts:
712 result = getattr(result, p)
713 return result