blob: 51da0b1bb52641d818852f4e3d6c45c822ed08b5 [file] [log] [blame]
Guido van Rossuma4deda02002-12-23 16:30:00 +00001"""Utilities to support packages."""
2
3import os
4import sys
Thomas Wouters49fd7fa2006-04-21 10:40:58 +00005import imp
6import os.path
7from types import ModuleType
8
9__all__ = [
10 'get_importer', 'iter_importers', 'get_loader', 'find_loader',
Éric Araujoa4e2d4f2011-05-02 22:59:15 +020011 'walk_packages', 'iter_modules', 'get_data',
Thomas Wouters49fd7fa2006-04-21 10:40:58 +000012 'ImpImporter', 'ImpLoader', 'read_code', 'extend_path',
13]
14
15def read_code(stream):
16 # This helper is needed in order for the PEP 302 emulation to
17 # correctly handle compiled files
18 import marshal
19
20 magic = stream.read(4)
21 if magic != imp.get_magic():
22 return None
23
24 stream.read(4) # Skip timestamp
25 return marshal.load(stream)
26
27
28def simplegeneric(func):
29 """Make a trivial single-dispatch generic function"""
30 registry = {}
Thomas Wouters477c8d52006-05-27 19:21:47 +000031 def wrapper(*args, **kw):
Thomas Wouters49fd7fa2006-04-21 10:40:58 +000032 ob = args[0]
33 try:
34 cls = ob.__class__
35 except AttributeError:
36 cls = type(ob)
37 try:
38 mro = cls.__mro__
39 except AttributeError:
40 try:
Thomas Wouters477c8d52006-05-27 19:21:47 +000041 class cls(cls, object):
42 pass
Thomas Wouters49fd7fa2006-04-21 10:40:58 +000043 mro = cls.__mro__[1:]
44 except TypeError:
45 mro = object, # must be an ExtensionClass or some such :(
46 for t in mro:
47 if t in registry:
Thomas Wouters477c8d52006-05-27 19:21:47 +000048 return registry[t](*args, **kw)
Thomas Wouters49fd7fa2006-04-21 10:40:58 +000049 else:
Thomas Wouters477c8d52006-05-27 19:21:47 +000050 return func(*args, **kw)
Thomas Wouters49fd7fa2006-04-21 10:40:58 +000051 try:
52 wrapper.__name__ = func.__name__
Thomas Wouters477c8d52006-05-27 19:21:47 +000053 except (TypeError, AttributeError):
Thomas Wouters49fd7fa2006-04-21 10:40:58 +000054 pass # Python 2.3 doesn't allow functions to be renamed
55
56 def register(typ, func=None):
57 if func is None:
58 return lambda f: register(typ, f)
59 registry[typ] = func
60 return func
61
62 wrapper.__dict__ = func.__dict__
63 wrapper.__doc__ = func.__doc__
64 wrapper.register = register
65 return wrapper
66
67
68def walk_packages(path=None, prefix='', onerror=None):
Thomas Wouters0e3f5912006-08-11 14:57:12 +000069 """Yields (module_loader, name, ispkg) for all modules recursively
70 on path, or, if path is None, all accessible modules.
71
72 'path' should be either None or a list of paths to look for
73 modules in.
74
75 'prefix' is a string to output on the front of every module name
76 on output.
77
78 Note that this function must import all *packages* (NOT all
79 modules!) on the given path, in order to access the __path__
80 attribute to find submodules.
81
82 'onerror' is a function which gets called with one argument (the
83 name of the package which was being imported) if any exception
84 occurs while trying to import a package. If no onerror function is
85 supplied, ImportErrors are caught and ignored, while all other
86 exceptions are propagated, terminating the search.
87
88 Examples:
89
90 # list all modules python can access
91 walk_packages()
92
93 # list all submodules of ctypes
94 walk_packages(ctypes.__path__, ctypes.__name__+'.')
95 """
Thomas Wouters49fd7fa2006-04-21 10:40:58 +000096
Thomas Wouters477c8d52006-05-27 19:21:47 +000097 def seen(p, m={}):
98 if p in m:
99 return True
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000100 m[p] = True
101
102 for importer, name, ispkg in iter_modules(path, prefix):
103 yield importer, name, ispkg
104
105 if ispkg:
106 try:
107 __import__(name)
108 except ImportError:
109 if onerror is not None:
Thomas Wouters0e3f5912006-08-11 14:57:12 +0000110 onerror(name)
111 except Exception:
112 if onerror is not None:
113 onerror(name)
114 else:
115 raise
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000116 else:
117 path = getattr(sys.modules[name], '__path__', None) or []
118
119 # don't traverse path items we've seen before
120 path = [p for p in path if not seen(p)]
121
Thomas Wouters0e3f5912006-08-11 14:57:12 +0000122 for item in walk_packages(path, name+'.', onerror):
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000123 yield item
124
125
126def iter_modules(path=None, prefix=''):
Thomas Wouters0e3f5912006-08-11 14:57:12 +0000127 """Yields (module_loader, name, ispkg) for all submodules on path,
128 or, if path is None, all top-level modules on sys.path.
129
130 'path' should be either None or a list of paths to look for
131 modules in.
132
133 'prefix' is a string to output on the front of every module name
134 on output.
135 """
136
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000137 if path is None:
138 importers = iter_importers()
139 else:
140 importers = map(get_importer, path)
141
142 yielded = {}
143 for i in importers:
144 for name, ispkg in iter_importer_modules(i, prefix):
145 if name not in yielded:
146 yielded[name] = 1
147 yield i, name, ispkg
148
149
150#@simplegeneric
151def iter_importer_modules(importer, prefix=''):
Thomas Wouters477c8d52006-05-27 19:21:47 +0000152 if not hasattr(importer, 'iter_modules'):
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000153 return []
154 return importer.iter_modules(prefix)
155
156iter_importer_modules = simplegeneric(iter_importer_modules)
157
158
159class ImpImporter:
160 """PEP 302 Importer that wraps Python's "classic" import algorithm
161
162 ImpImporter(dirname) produces a PEP 302 importer that searches that
163 directory. ImpImporter(None) produces a PEP 302 importer that searches
164 the current sys.path, plus any modules that are frozen or built-in.
165
166 Note that ImpImporter does not currently support being used by placement
167 on sys.meta_path.
168 """
169
170 def __init__(self, path=None):
171 self.path = path
172
173 def find_module(self, fullname, path=None):
174 # Note: we ignore 'path' argument since it is only used via meta_path
175 subname = fullname.split(".")[-1]
176 if subname != fullname and self.path is None:
177 return None
178 if self.path is None:
179 path = None
180 else:
181 path = [os.path.realpath(self.path)]
182 try:
183 file, filename, etc = imp.find_module(subname, path)
184 except ImportError:
185 return None
186 return ImpLoader(fullname, file, filename, etc)
187
188 def iter_modules(self, prefix=''):
189 if self.path is None or not os.path.isdir(self.path):
190 return
191
192 yielded = {}
193 import inspect
Ned Deilyed27df72011-10-06 14:19:08 -0700194 try:
195 filenames = os.listdir(self.path)
196 except OSError:
197 # ignore unreadable directories like import does
198 filenames = []
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000199 filenames.sort() # handle packages before same-named modules
200
201 for fn in filenames:
202 modname = inspect.getmodulename(fn)
203 if modname=='__init__' or modname in yielded:
204 continue
205
206 path = os.path.join(self.path, fn)
207 ispkg = False
208
209 if not modname and os.path.isdir(path) and '.' not in fn:
210 modname = fn
Ned Deilyed27df72011-10-06 14:19:08 -0700211 try:
212 dircontents = os.listdir(path)
213 except OSError:
214 # ignore unreadable directories like import does
215 dircontents = []
216 for fn in dircontents:
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000217 subname = inspect.getmodulename(fn)
218 if subname=='__init__':
219 ispkg = True
220 break
221 else:
222 continue # not a package
223
224 if modname and '.' not in modname:
225 yielded[modname] = 1
226 yield prefix + modname, ispkg
227
228
229class ImpLoader:
230 """PEP 302 Loader that wraps Python's "classic" import algorithm
231 """
232 code = source = None
233
234 def __init__(self, fullname, file, filename, etc):
235 self.file = file
236 self.filename = filename
237 self.fullname = fullname
238 self.etc = etc
239
240 def load_module(self, fullname):
241 self._reopen()
242 try:
243 mod = imp.load_module(fullname, self.file, self.filename, self.etc)
244 finally:
245 if self.file:
246 self.file.close()
247 # Note: we don't set __loader__ because we want the module to look
248 # normal; i.e. this is just a wrapper for standard import machinery
249 return mod
250
251 def get_data(self, pathname):
Brett Cannon1ab58df2010-10-29 22:36:53 +0000252 with open(pathname, "rb") as file:
253 return file.read()
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000254
255 def _reopen(self):
256 if self.file and self.file.closed:
Thomas Wouters0e3f5912006-08-11 14:57:12 +0000257 mod_type = self.etc[2]
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000258 if mod_type==imp.PY_SOURCE:
259 self.file = open(self.filename, 'rU')
260 elif mod_type in (imp.PY_COMPILED, imp.C_EXTENSION):
261 self.file = open(self.filename, 'rb')
262
263 def _fix_name(self, fullname):
264 if fullname is None:
265 fullname = self.fullname
266 elif fullname != self.fullname:
267 raise ImportError("Loader for module %s cannot handle "
268 "module %s" % (self.fullname, fullname))
269 return fullname
270
271 def is_package(self, fullname):
272 fullname = self._fix_name(fullname)
273 return self.etc[2]==imp.PKG_DIRECTORY
274
275 def get_code(self, fullname=None):
276 fullname = self._fix_name(fullname)
277 if self.code is None:
278 mod_type = self.etc[2]
279 if mod_type==imp.PY_SOURCE:
280 source = self.get_source(fullname)
281 self.code = compile(source, self.filename, 'exec')
282 elif mod_type==imp.PY_COMPILED:
283 self._reopen()
284 try:
285 self.code = read_code(self.file)
286 finally:
287 self.file.close()
288 elif mod_type==imp.PKG_DIRECTORY:
289 self.code = self._get_delegate().get_code()
290 return self.code
291
292 def get_source(self, fullname=None):
293 fullname = self._fix_name(fullname)
294 if self.source is None:
295 mod_type = self.etc[2]
296 if mod_type==imp.PY_SOURCE:
297 self._reopen()
298 try:
299 self.source = self.file.read()
300 finally:
301 self.file.close()
302 elif mod_type==imp.PY_COMPILED:
303 if os.path.exists(self.filename[:-1]):
304 f = open(self.filename[:-1], 'rU')
305 self.source = f.read()
306 f.close()
307 elif mod_type==imp.PKG_DIRECTORY:
308 self.source = self._get_delegate().get_source()
309 return self.source
310
311
312 def _get_delegate(self):
313 return ImpImporter(self.filename).find_module('__init__')
314
315 def get_filename(self, fullname=None):
316 fullname = self._fix_name(fullname)
317 mod_type = self.etc[2]
318 if self.etc[2]==imp.PKG_DIRECTORY:
319 return self._get_delegate().get_filename()
320 elif self.etc[2] in (imp.PY_SOURCE, imp.PY_COMPILED, imp.C_EXTENSION):
321 return self.filename
322 return None
323
324
325try:
326 import zipimport
327 from zipimport import zipimporter
328
329 def iter_zipimport_modules(importer, prefix=''):
Alexandre Vassalotti515a74f2009-07-05 06:42:44 +0000330 dirlist = sorted(zipimport._zip_directory_cache[importer.archive])
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000331 _prefix = importer.prefix
332 plen = len(_prefix)
333 yielded = {}
334 import inspect
335 for fn in dirlist:
336 if not fn.startswith(_prefix):
337 continue
338
339 fn = fn[plen:].split(os.sep)
340
341 if len(fn)==2 and fn[1].startswith('__init__.py'):
342 if fn[0] not in yielded:
343 yielded[fn[0]] = 1
344 yield fn[0], True
345
346 if len(fn)!=1:
347 continue
348
349 modname = inspect.getmodulename(fn[0])
350 if modname=='__init__':
351 continue
352
353 if modname and '.' not in modname and modname not in yielded:
354 yielded[modname] = 1
355 yield prefix + modname, False
356
357 iter_importer_modules.register(zipimporter, iter_zipimport_modules)
358
359except ImportError:
360 pass
361
362
363def get_importer(path_item):
364 """Retrieve a PEP 302 importer for the given path item
365
366 The returned importer is cached in sys.path_importer_cache
367 if it was newly created by a path hook.
368
369 If there is no importer, a wrapper around the basic import
370 machinery is returned. This wrapper is never inserted into
371 the importer cache (None is inserted instead).
372
373 The cache (or part of it) can be cleared manually if a
374 rescan of sys.path_hooks is necessary.
375 """
376 try:
377 importer = sys.path_importer_cache[path_item]
378 except KeyError:
379 for path_hook in sys.path_hooks:
380 try:
381 importer = path_hook(path_item)
382 break
383 except ImportError:
384 pass
385 else:
386 importer = None
Thomas Wouters477c8d52006-05-27 19:21:47 +0000387 sys.path_importer_cache.setdefault(path_item, importer)
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000388
Thomas Wouters0e3f5912006-08-11 14:57:12 +0000389 if importer is None:
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000390 try:
391 importer = ImpImporter(path_item)
392 except ImportError:
Thomas Wouters477c8d52006-05-27 19:21:47 +0000393 importer = None
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000394 return importer
395
396
397def iter_importers(fullname=""):
398 """Yield PEP 302 importers for the given module name
399
400 If fullname contains a '.', the importers will be for the package
401 containing fullname, otherwise they will be importers for sys.meta_path,
402 sys.path, and Python's "classic" import machinery, in that order. If
403 the named module is in a package, that package is imported as a side
404 effect of invoking this function.
405
406 Non PEP 302 mechanisms (e.g. the Windows registry) used by the
407 standard import machinery to find files in alternative locations
408 are partially supported, but are searched AFTER sys.path. Normally,
409 these locations are searched BEFORE sys.path, preventing sys.path
410 entries from shadowing them.
411
412 For this to cause a visible difference in behaviour, there must
413 be a module or package name that is accessible via both sys.path
414 and one of the non PEP 302 file system mechanisms. In this case,
415 the emulation will find the former version, while the builtin
416 import mechanism will find the latter.
417
418 Items of the following types can be affected by this discrepancy:
419 imp.C_EXTENSION, imp.PY_SOURCE, imp.PY_COMPILED, imp.PKG_DIRECTORY
420 """
421 if fullname.startswith('.'):
422 raise ImportError("Relative module names not supported")
423 if '.' in fullname:
424 # Get the containing package's __path__
425 pkg = '.'.join(fullname.split('.')[:-1])
426 if pkg not in sys.modules:
427 __import__(pkg)
Thomas Wouters477c8d52006-05-27 19:21:47 +0000428 path = getattr(sys.modules[pkg], '__path__', None) or []
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000429 else:
430 for importer in sys.meta_path:
431 yield importer
432 path = sys.path
433 for item in path:
434 yield get_importer(item)
435 if '.' not in fullname:
436 yield ImpImporter()
437
438def get_loader(module_or_name):
439 """Get a PEP 302 "loader" object for module_or_name
440
441 If the module or package is accessible via the normal import
442 mechanism, a wrapper around the relevant part of that machinery
443 is returned. Returns None if the module cannot be found or imported.
444 If the named module is not already imported, its containing package
445 (if any) is imported, in order to establish the package __path__.
446
447 This function uses iter_importers(), and is thus subject to the same
448 limitations regarding platform-specific special import locations such
449 as the Windows registry.
450 """
451 if module_or_name in sys.modules:
452 module_or_name = sys.modules[module_or_name]
453 if isinstance(module_or_name, ModuleType):
454 module = module_or_name
Thomas Wouters477c8d52006-05-27 19:21:47 +0000455 loader = getattr(module, '__loader__', None)
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000456 if loader is not None:
457 return loader
458 fullname = module.__name__
459 else:
460 fullname = module_or_name
461 return find_loader(fullname)
462
463def find_loader(fullname):
464 """Find a PEP 302 "loader" object for fullname
465
466 If fullname contains dots, path must be the containing package's __path__.
467 Returns None if the module cannot be found or imported. This function uses
468 iter_importers(), and is thus subject to the same limitations regarding
469 platform-specific special import locations such as the Windows registry.
470 """
471 for importer in iter_importers(fullname):
472 loader = importer.find_module(fullname)
473 if loader is not None:
474 return loader
475
476 return None
477
Guido van Rossuma4deda02002-12-23 16:30:00 +0000478
479def extend_path(path, name):
480 """Extend a package's path.
481
482 Intended use is to place the following code in a package's __init__.py:
483
484 from pkgutil import extend_path
485 __path__ = extend_path(__path__, __name__)
486
487 This will add to the package's __path__ all subdirectories of
488 directories on sys.path named after the package. This is useful
489 if one wants to distribute different parts of a single logical
490 package as multiple directories.
491
492 It also looks for *.pkg files beginning where * matches the name
493 argument. This feature is similar to *.pth files (see site.py),
494 except that it doesn't special-case lines starting with 'import'.
495 A *.pkg file is trusted at face value: apart from checking for
496 duplicates, all entries found in a *.pkg file are added to the
497 path, regardless of whether they are exist the filesystem. (This
498 is a feature.)
499
500 If the input path is not a list (as is the case for frozen
501 packages) it is returned unchanged. The input path is not
502 modified; an extended copy is returned. Items are only appended
503 to the copy at the end.
504
505 It is assumed that sys.path is a sequence. Items of sys.path that
506 are not (unicode or 8-bit) strings referring to existing
507 directories are ignored. Unicode items of sys.path that cause
508 errors when used as filenames may cause this function to raise an
509 exception (in line with os.path.isdir() behavior).
510 """
511
512 if not isinstance(path, list):
513 # This could happen e.g. when this is called from inside a
514 # frozen package. Return the path unchanged in that case.
515 return path
516
517 pname = os.path.join(*name.split('.')) # Reconstitute as relative path
Skip Montanaro7a98be22007-08-16 14:35:24 +0000518 sname_pkg = name + ".pkg"
519 init_py = "__init__.py"
Guido van Rossuma4deda02002-12-23 16:30:00 +0000520
521 path = path[:] # Start with a copy of the existing path
522
523 for dir in sys.path:
Guido van Rossum3172c5d2007-10-16 18:12:55 +0000524 if not isinstance(dir, str) or not os.path.isdir(dir):
Guido van Rossuma4deda02002-12-23 16:30:00 +0000525 continue
526 subdir = os.path.join(dir, pname)
527 # XXX This may still add duplicate entries to path on
528 # case-insensitive filesystems
529 initfile = os.path.join(subdir, init_py)
530 if subdir not in path and os.path.isfile(initfile):
531 path.append(subdir)
532 # XXX Is this the right thing for subpackages like zope.app?
533 # It looks for a file named "zope.app.pkg"
534 pkgfile = os.path.join(dir, sname_pkg)
535 if os.path.isfile(pkgfile):
536 try:
537 f = open(pkgfile)
Guido van Rossumb940e112007-01-10 16:19:56 +0000538 except IOError as msg:
Guido van Rossuma4deda02002-12-23 16:30:00 +0000539 sys.stderr.write("Can't open %s: %s\n" %
540 (pkgfile, msg))
541 else:
542 for line in f:
543 line = line.rstrip('\n')
544 if not line or line.startswith('#'):
545 continue
546 path.append(line) # Don't check for existence!
547 f.close()
548
549 return path
Christian Heimesdae2a892008-04-19 00:55:37 +0000550
551def get_data(package, resource):
552 """Get a resource from a package.
553
554 This is a wrapper round the PEP 302 loader get_data API. The package
555 argument should be the name of a package, in standard module format
556 (foo.bar). The resource argument should be in the form of a relative
557 filename, using '/' as the path separator. The parent directory name '..'
558 is not allowed, and nor is a rooted name (starting with a '/').
559
560 The function returns a binary string, which is the contents of the
561 specified resource.
562
563 For packages located in the filesystem, which have already been imported,
564 this is the rough equivalent of
565
566 d = os.path.dirname(sys.modules[package].__file__)
567 data = open(os.path.join(d, resource), 'rb').read()
568
569 If the package cannot be located or loaded, or it uses a PEP 302 loader
570 which does not support get_data(), then None is returned.
571 """
572
573 loader = get_loader(package)
574 if loader is None or not hasattr(loader, 'get_data'):
575 return None
576 mod = sys.modules.get(package) or loader.load_module(package)
577 if mod is None or not hasattr(mod, '__file__'):
578 return None
579
580 # Modify the resource name to be compatible with the loader.get_data
581 # signature - an os.path format "filename" starting with the dirname of
582 # the package's __file__
583 parts = resource.split('/')
584 parts.insert(0, os.path.dirname(mod.__file__))
585 resource_name = os.path.join(*parts)
586 return loader.get_data(resource_name)