blob: 27dd887d869b06fa2076c3e027c7e0ad317f90b1 [file] [log] [blame]
Guido van Rossuma4deda02002-12-23 16:30:00 +00001"""Utilities to support packages."""
2
3import os
4import sys
Thomas Wouters49fd7fa2006-04-21 10:40:58 +00005import imp
6import os.path
7from types import ModuleType
8
9__all__ = [
10 'get_importer', 'iter_importers', 'get_loader', 'find_loader',
11 'walk_packages', 'iter_modules',
12 'ImpImporter', 'ImpLoader', 'read_code', 'extend_path',
13]
14
15def read_code(stream):
16 # This helper is needed in order for the PEP 302 emulation to
17 # correctly handle compiled files
18 import marshal
19
20 magic = stream.read(4)
21 if magic != imp.get_magic():
22 return None
23
24 stream.read(4) # Skip timestamp
25 return marshal.load(stream)
26
27
28def simplegeneric(func):
29 """Make a trivial single-dispatch generic function"""
30 registry = {}
Thomas Wouters477c8d52006-05-27 19:21:47 +000031 def wrapper(*args, **kw):
Thomas Wouters49fd7fa2006-04-21 10:40:58 +000032 ob = args[0]
33 try:
34 cls = ob.__class__
35 except AttributeError:
36 cls = type(ob)
37 try:
38 mro = cls.__mro__
39 except AttributeError:
40 try:
Thomas Wouters477c8d52006-05-27 19:21:47 +000041 class cls(cls, object):
42 pass
Thomas Wouters49fd7fa2006-04-21 10:40:58 +000043 mro = cls.__mro__[1:]
44 except TypeError:
45 mro = object, # must be an ExtensionClass or some such :(
46 for t in mro:
47 if t in registry:
Thomas Wouters477c8d52006-05-27 19:21:47 +000048 return registry[t](*args, **kw)
Thomas Wouters49fd7fa2006-04-21 10:40:58 +000049 else:
Thomas Wouters477c8d52006-05-27 19:21:47 +000050 return func(*args, **kw)
Thomas Wouters49fd7fa2006-04-21 10:40:58 +000051 try:
52 wrapper.__name__ = func.__name__
Thomas Wouters477c8d52006-05-27 19:21:47 +000053 except (TypeError, AttributeError):
Thomas Wouters49fd7fa2006-04-21 10:40:58 +000054 pass # Python 2.3 doesn't allow functions to be renamed
55
56 def register(typ, func=None):
57 if func is None:
58 return lambda f: register(typ, f)
59 registry[typ] = func
60 return func
61
62 wrapper.__dict__ = func.__dict__
63 wrapper.__doc__ = func.__doc__
64 wrapper.register = register
65 return wrapper
66
67
68def walk_packages(path=None, prefix='', onerror=None):
Thomas Wouters0e3f5912006-08-11 14:57:12 +000069 """Yields (module_loader, name, ispkg) for all modules recursively
70 on path, or, if path is None, all accessible modules.
71
72 'path' should be either None or a list of paths to look for
73 modules in.
74
75 'prefix' is a string to output on the front of every module name
76 on output.
77
78 Note that this function must import all *packages* (NOT all
79 modules!) on the given path, in order to access the __path__
80 attribute to find submodules.
81
82 'onerror' is a function which gets called with one argument (the
83 name of the package which was being imported) if any exception
84 occurs while trying to import a package. If no onerror function is
85 supplied, ImportErrors are caught and ignored, while all other
86 exceptions are propagated, terminating the search.
87
88 Examples:
89
90 # list all modules python can access
91 walk_packages()
92
93 # list all submodules of ctypes
94 walk_packages(ctypes.__path__, ctypes.__name__+'.')
95 """
Thomas Wouters49fd7fa2006-04-21 10:40:58 +000096
Thomas Wouters477c8d52006-05-27 19:21:47 +000097 def seen(p, m={}):
98 if p in m:
99 return True
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000100 m[p] = True
101
102 for importer, name, ispkg in iter_modules(path, prefix):
103 yield importer, name, ispkg
104
105 if ispkg:
106 try:
107 __import__(name)
108 except ImportError:
109 if onerror is not None:
Thomas Wouters0e3f5912006-08-11 14:57:12 +0000110 onerror(name)
111 except Exception:
112 if onerror is not None:
113 onerror(name)
114 else:
115 raise
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000116 else:
117 path = getattr(sys.modules[name], '__path__', None) or []
118
119 # don't traverse path items we've seen before
120 path = [p for p in path if not seen(p)]
121
Thomas Wouters0e3f5912006-08-11 14:57:12 +0000122 for item in walk_packages(path, name+'.', onerror):
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000123 yield item
124
125
126def iter_modules(path=None, prefix=''):
Thomas Wouters0e3f5912006-08-11 14:57:12 +0000127 """Yields (module_loader, name, ispkg) for all submodules on path,
128 or, if path is None, all top-level modules on sys.path.
129
130 'path' should be either None or a list of paths to look for
131 modules in.
132
133 'prefix' is a string to output on the front of every module name
134 on output.
135 """
136
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000137 if path is None:
138 importers = iter_importers()
139 else:
140 importers = map(get_importer, path)
141
142 yielded = {}
143 for i in importers:
144 for name, ispkg in iter_importer_modules(i, prefix):
145 if name not in yielded:
146 yielded[name] = 1
147 yield i, name, ispkg
148
149
150#@simplegeneric
151def iter_importer_modules(importer, prefix=''):
Thomas Wouters477c8d52006-05-27 19:21:47 +0000152 if not hasattr(importer, 'iter_modules'):
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000153 return []
154 return importer.iter_modules(prefix)
155
156iter_importer_modules = simplegeneric(iter_importer_modules)
157
158
159class ImpImporter:
160 """PEP 302 Importer that wraps Python's "classic" import algorithm
161
162 ImpImporter(dirname) produces a PEP 302 importer that searches that
163 directory. ImpImporter(None) produces a PEP 302 importer that searches
164 the current sys.path, plus any modules that are frozen or built-in.
165
166 Note that ImpImporter does not currently support being used by placement
167 on sys.meta_path.
168 """
169
170 def __init__(self, path=None):
171 self.path = path
172
173 def find_module(self, fullname, path=None):
174 # Note: we ignore 'path' argument since it is only used via meta_path
175 subname = fullname.split(".")[-1]
176 if subname != fullname and self.path is None:
177 return None
178 if self.path is None:
179 path = None
180 else:
181 path = [os.path.realpath(self.path)]
182 try:
183 file, filename, etc = imp.find_module(subname, path)
184 except ImportError:
185 return None
186 return ImpLoader(fullname, file, filename, etc)
187
188 def iter_modules(self, prefix=''):
189 if self.path is None or not os.path.isdir(self.path):
190 return
191
192 yielded = {}
193 import inspect
194
195 filenames = os.listdir(self.path)
196 filenames.sort() # handle packages before same-named modules
197
198 for fn in filenames:
199 modname = inspect.getmodulename(fn)
200 if modname=='__init__' or modname in yielded:
201 continue
202
203 path = os.path.join(self.path, fn)
204 ispkg = False
205
206 if not modname and os.path.isdir(path) and '.' not in fn:
207 modname = fn
208 for fn in os.listdir(path):
209 subname = inspect.getmodulename(fn)
210 if subname=='__init__':
211 ispkg = True
212 break
213 else:
214 continue # not a package
215
216 if modname and '.' not in modname:
217 yielded[modname] = 1
218 yield prefix + modname, ispkg
219
220
221class ImpLoader:
222 """PEP 302 Loader that wraps Python's "classic" import algorithm
223 """
224 code = source = None
225
226 def __init__(self, fullname, file, filename, etc):
227 self.file = file
228 self.filename = filename
229 self.fullname = fullname
230 self.etc = etc
231
232 def load_module(self, fullname):
233 self._reopen()
234 try:
235 mod = imp.load_module(fullname, self.file, self.filename, self.etc)
236 finally:
237 if self.file:
238 self.file.close()
239 # Note: we don't set __loader__ because we want the module to look
240 # normal; i.e. this is just a wrapper for standard import machinery
241 return mod
242
243 def get_data(self, pathname):
244 return open(pathname, "rb").read()
245
246 def _reopen(self):
247 if self.file and self.file.closed:
Thomas Wouters0e3f5912006-08-11 14:57:12 +0000248 mod_type = self.etc[2]
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000249 if mod_type==imp.PY_SOURCE:
250 self.file = open(self.filename, 'rU')
251 elif mod_type in (imp.PY_COMPILED, imp.C_EXTENSION):
252 self.file = open(self.filename, 'rb')
253
254 def _fix_name(self, fullname):
255 if fullname is None:
256 fullname = self.fullname
257 elif fullname != self.fullname:
258 raise ImportError("Loader for module %s cannot handle "
259 "module %s" % (self.fullname, fullname))
260 return fullname
261
262 def is_package(self, fullname):
263 fullname = self._fix_name(fullname)
264 return self.etc[2]==imp.PKG_DIRECTORY
265
266 def get_code(self, fullname=None):
267 fullname = self._fix_name(fullname)
268 if self.code is None:
269 mod_type = self.etc[2]
270 if mod_type==imp.PY_SOURCE:
271 source = self.get_source(fullname)
272 self.code = compile(source, self.filename, 'exec')
273 elif mod_type==imp.PY_COMPILED:
274 self._reopen()
275 try:
276 self.code = read_code(self.file)
277 finally:
278 self.file.close()
279 elif mod_type==imp.PKG_DIRECTORY:
280 self.code = self._get_delegate().get_code()
281 return self.code
282
283 def get_source(self, fullname=None):
284 fullname = self._fix_name(fullname)
285 if self.source is None:
286 mod_type = self.etc[2]
287 if mod_type==imp.PY_SOURCE:
288 self._reopen()
289 try:
290 self.source = self.file.read()
291 finally:
292 self.file.close()
293 elif mod_type==imp.PY_COMPILED:
294 if os.path.exists(self.filename[:-1]):
295 f = open(self.filename[:-1], 'rU')
296 self.source = f.read()
297 f.close()
298 elif mod_type==imp.PKG_DIRECTORY:
299 self.source = self._get_delegate().get_source()
300 return self.source
301
302
303 def _get_delegate(self):
304 return ImpImporter(self.filename).find_module('__init__')
305
306 def get_filename(self, fullname=None):
307 fullname = self._fix_name(fullname)
308 mod_type = self.etc[2]
309 if self.etc[2]==imp.PKG_DIRECTORY:
310 return self._get_delegate().get_filename()
311 elif self.etc[2] in (imp.PY_SOURCE, imp.PY_COMPILED, imp.C_EXTENSION):
312 return self.filename
313 return None
314
315
316try:
317 import zipimport
318 from zipimport import zipimporter
319
320 def iter_zipimport_modules(importer, prefix=''):
321 dirlist = zipimport._zip_directory_cache[importer.archive].keys()
322 dirlist.sort()
323 _prefix = importer.prefix
324 plen = len(_prefix)
325 yielded = {}
326 import inspect
327 for fn in dirlist:
328 if not fn.startswith(_prefix):
329 continue
330
331 fn = fn[plen:].split(os.sep)
332
333 if len(fn)==2 and fn[1].startswith('__init__.py'):
334 if fn[0] not in yielded:
335 yielded[fn[0]] = 1
336 yield fn[0], True
337
338 if len(fn)!=1:
339 continue
340
341 modname = inspect.getmodulename(fn[0])
342 if modname=='__init__':
343 continue
344
345 if modname and '.' not in modname and modname not in yielded:
346 yielded[modname] = 1
347 yield prefix + modname, False
348
349 iter_importer_modules.register(zipimporter, iter_zipimport_modules)
350
351except ImportError:
352 pass
353
354
355def get_importer(path_item):
356 """Retrieve a PEP 302 importer for the given path item
357
358 The returned importer is cached in sys.path_importer_cache
359 if it was newly created by a path hook.
360
361 If there is no importer, a wrapper around the basic import
362 machinery is returned. This wrapper is never inserted into
363 the importer cache (None is inserted instead).
364
365 The cache (or part of it) can be cleared manually if a
366 rescan of sys.path_hooks is necessary.
367 """
368 try:
369 importer = sys.path_importer_cache[path_item]
370 except KeyError:
371 for path_hook in sys.path_hooks:
372 try:
373 importer = path_hook(path_item)
374 break
375 except ImportError:
376 pass
377 else:
378 importer = None
Thomas Wouters477c8d52006-05-27 19:21:47 +0000379 sys.path_importer_cache.setdefault(path_item, importer)
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000380
Thomas Wouters0e3f5912006-08-11 14:57:12 +0000381 if importer is None:
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000382 try:
383 importer = ImpImporter(path_item)
384 except ImportError:
Thomas Wouters477c8d52006-05-27 19:21:47 +0000385 importer = None
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000386 return importer
387
388
389def iter_importers(fullname=""):
390 """Yield PEP 302 importers for the given module name
391
392 If fullname contains a '.', the importers will be for the package
393 containing fullname, otherwise they will be importers for sys.meta_path,
394 sys.path, and Python's "classic" import machinery, in that order. If
395 the named module is in a package, that package is imported as a side
396 effect of invoking this function.
397
398 Non PEP 302 mechanisms (e.g. the Windows registry) used by the
399 standard import machinery to find files in alternative locations
400 are partially supported, but are searched AFTER sys.path. Normally,
401 these locations are searched BEFORE sys.path, preventing sys.path
402 entries from shadowing them.
403
404 For this to cause a visible difference in behaviour, there must
405 be a module or package name that is accessible via both sys.path
406 and one of the non PEP 302 file system mechanisms. In this case,
407 the emulation will find the former version, while the builtin
408 import mechanism will find the latter.
409
410 Items of the following types can be affected by this discrepancy:
411 imp.C_EXTENSION, imp.PY_SOURCE, imp.PY_COMPILED, imp.PKG_DIRECTORY
412 """
413 if fullname.startswith('.'):
414 raise ImportError("Relative module names not supported")
415 if '.' in fullname:
416 # Get the containing package's __path__
417 pkg = '.'.join(fullname.split('.')[:-1])
418 if pkg not in sys.modules:
419 __import__(pkg)
Thomas Wouters477c8d52006-05-27 19:21:47 +0000420 path = getattr(sys.modules[pkg], '__path__', None) or []
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000421 else:
422 for importer in sys.meta_path:
423 yield importer
424 path = sys.path
425 for item in path:
426 yield get_importer(item)
427 if '.' not in fullname:
428 yield ImpImporter()
429
430def get_loader(module_or_name):
431 """Get a PEP 302 "loader" object for module_or_name
432
433 If the module or package is accessible via the normal import
434 mechanism, a wrapper around the relevant part of that machinery
435 is returned. Returns None if the module cannot be found or imported.
436 If the named module is not already imported, its containing package
437 (if any) is imported, in order to establish the package __path__.
438
439 This function uses iter_importers(), and is thus subject to the same
440 limitations regarding platform-specific special import locations such
441 as the Windows registry.
442 """
443 if module_or_name in sys.modules:
444 module_or_name = sys.modules[module_or_name]
445 if isinstance(module_or_name, ModuleType):
446 module = module_or_name
Thomas Wouters477c8d52006-05-27 19:21:47 +0000447 loader = getattr(module, '__loader__', None)
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000448 if loader is not None:
449 return loader
450 fullname = module.__name__
451 else:
452 fullname = module_or_name
453 return find_loader(fullname)
454
455def find_loader(fullname):
456 """Find a PEP 302 "loader" object for fullname
457
458 If fullname contains dots, path must be the containing package's __path__.
459 Returns None if the module cannot be found or imported. This function uses
460 iter_importers(), and is thus subject to the same limitations regarding
461 platform-specific special import locations such as the Windows registry.
462 """
463 for importer in iter_importers(fullname):
464 loader = importer.find_module(fullname)
465 if loader is not None:
466 return loader
467
468 return None
469
Guido van Rossuma4deda02002-12-23 16:30:00 +0000470
471def extend_path(path, name):
472 """Extend a package's path.
473
474 Intended use is to place the following code in a package's __init__.py:
475
476 from pkgutil import extend_path
477 __path__ = extend_path(__path__, __name__)
478
479 This will add to the package's __path__ all subdirectories of
480 directories on sys.path named after the package. This is useful
481 if one wants to distribute different parts of a single logical
482 package as multiple directories.
483
484 It also looks for *.pkg files beginning where * matches the name
485 argument. This feature is similar to *.pth files (see site.py),
486 except that it doesn't special-case lines starting with 'import'.
487 A *.pkg file is trusted at face value: apart from checking for
488 duplicates, all entries found in a *.pkg file are added to the
489 path, regardless of whether they are exist the filesystem. (This
490 is a feature.)
491
492 If the input path is not a list (as is the case for frozen
493 packages) it is returned unchanged. The input path is not
494 modified; an extended copy is returned. Items are only appended
495 to the copy at the end.
496
497 It is assumed that sys.path is a sequence. Items of sys.path that
498 are not (unicode or 8-bit) strings referring to existing
499 directories are ignored. Unicode items of sys.path that cause
500 errors when used as filenames may cause this function to raise an
501 exception (in line with os.path.isdir() behavior).
502 """
503
504 if not isinstance(path, list):
505 # This could happen e.g. when this is called from inside a
506 # frozen package. Return the path unchanged in that case.
507 return path
508
509 pname = os.path.join(*name.split('.')) # Reconstitute as relative path
Skip Montanaro7a98be22007-08-16 14:35:24 +0000510 sname_pkg = name + ".pkg"
511 init_py = "__init__.py"
Guido van Rossuma4deda02002-12-23 16:30:00 +0000512
513 path = path[:] # Start with a copy of the existing path
514
515 for dir in sys.path:
Guido van Rossum3172c5d2007-10-16 18:12:55 +0000516 if not isinstance(dir, str) or not os.path.isdir(dir):
Guido van Rossuma4deda02002-12-23 16:30:00 +0000517 continue
518 subdir = os.path.join(dir, pname)
519 # XXX This may still add duplicate entries to path on
520 # case-insensitive filesystems
521 initfile = os.path.join(subdir, init_py)
522 if subdir not in path and os.path.isfile(initfile):
523 path.append(subdir)
524 # XXX Is this the right thing for subpackages like zope.app?
525 # It looks for a file named "zope.app.pkg"
526 pkgfile = os.path.join(dir, sname_pkg)
527 if os.path.isfile(pkgfile):
528 try:
529 f = open(pkgfile)
Guido van Rossumb940e112007-01-10 16:19:56 +0000530 except IOError as msg:
Guido van Rossuma4deda02002-12-23 16:30:00 +0000531 sys.stderr.write("Can't open %s: %s\n" %
532 (pkgfile, msg))
533 else:
534 for line in f:
535 line = line.rstrip('\n')
536 if not line or line.startswith('#'):
537 continue
538 path.append(line) # Don't check for existence!
539 f.close()
540
541 return path
Christian Heimesdae2a892008-04-19 00:55:37 +0000542
543def get_data(package, resource):
544 """Get a resource from a package.
545
546 This is a wrapper round the PEP 302 loader get_data API. The package
547 argument should be the name of a package, in standard module format
548 (foo.bar). The resource argument should be in the form of a relative
549 filename, using '/' as the path separator. The parent directory name '..'
550 is not allowed, and nor is a rooted name (starting with a '/').
551
552 The function returns a binary string, which is the contents of the
553 specified resource.
554
555 For packages located in the filesystem, which have already been imported,
556 this is the rough equivalent of
557
558 d = os.path.dirname(sys.modules[package].__file__)
559 data = open(os.path.join(d, resource), 'rb').read()
560
561 If the package cannot be located or loaded, or it uses a PEP 302 loader
562 which does not support get_data(), then None is returned.
563 """
564
565 loader = get_loader(package)
566 if loader is None or not hasattr(loader, 'get_data'):
567 return None
568 mod = sys.modules.get(package) or loader.load_module(package)
569 if mod is None or not hasattr(mod, '__file__'):
570 return None
571
572 # Modify the resource name to be compatible with the loader.get_data
573 # signature - an os.path format "filename" starting with the dirname of
574 # the package's __file__
575 parts = resource.split('/')
576 parts.insert(0, os.path.dirname(mod.__file__))
577 resource_name = os.path.join(*parts)
578 return loader.get_data(resource_name)