| """distutils.command.dist |
| |
| Implements the Distutils 'dist' command (create a source distribution).""" |
| |
| # created 1999/09/22, Greg Ward |
| |
| __rcsid__ = "$Id$" |
| |
| import sys, os, string, re |
| import fnmatch |
| from types import * |
| from glob import glob |
| from distutils.core import Command |
| from distutils.text_file import TextFile |
| |
| |
| # Possible modes of operation: |
| # - require an explicit manifest that lists every single file (presumably |
| # along with a way to auto-generate the manifest) |
| # - require an explicit manifest, but allow it to have globs or |
| # filename patterns of some kind (and also have auto-generation) |
| # - allow an explict manifest, but automatically augment it at runtime |
| # with the source files mentioned in 'packages', 'py_modules', and |
| # 'ext_modules' (and any other such things that might come along) |
| |
| # I'm liking the third way. Possible gotchas: |
| # - redundant specification: 'packages' includes 'foo' and manifest |
| # includes 'foo/*.py' |
| # - obvious conflict: 'packages' includes 'foo' and manifest |
| # includes '! foo/*.py' (can't imagine why you'd want this) |
| # - subtle conflict: 'packages' includes 'foo' and manifest |
| # includes '! foo/bar.py' (this could well be desired: eg. exclude |
| # an experimental module from distribution) |
| |
| # Syntax for the manifest file: |
| # - if a line is just a Unix-style glob by itself, it's a "simple include |
| # pattern": go find all files that match and add them to the list |
| # of files |
| # - if a line is a glob preceded by "!", then it's a "simple exclude |
| # pattern": go over the current list of files and exclude any that |
| # match the glob pattern |
| # - if a line consists of a directory name followed by zero or more |
| # glob patterns, then we'll recursively explore that directory tree |
| # - the glob patterns can be include (no punctuation) or exclude |
| # (prefixed by "!", no space) |
| # - if no patterns given or the first pattern is not an include pattern, |
| # then assume "*" -- ie. find everything (and then start applying |
| # the rest of the patterns) |
| # - the patterns are given in order of increasing precedence, ie. |
| # the *last* one to match a given file applies to it |
| # |
| # example (ignoring auto-augmentation!): |
| # distutils/*.py |
| # distutils/command/*.py |
| # ! distutils/bleeding_edge.py |
| # examples/*.py |
| # examples/README |
| # |
| # smarter way (that *will* include distutils/command/bleeding_edge.py!) |
| # distutils *.py |
| # ! distutils/bleeding_edge.py |
| # examples !*~ !*.py[co] (same as: examples * !*~ !*.py[co]) |
| # test test_* *.txt !*~ !*.py[co] |
| # README |
| # setup.py |
| # |
| # The actual Distutils manifest (don't need to mention source files, |
| # README, setup.py -- they're automatically distributed!): |
| # examples !*~ !*.py[co] |
| # test !*~ !*.py[co] |
| |
| # The algorithm that will make it work: |
| # files = stuff from 'packages', 'py_modules', 'ext_modules', |
| # plus README, setup.py, ... ? |
| # foreach pattern in manifest file: |
| # if simple-include-pattern: # "distutils/*.py" |
| # files.append (glob (pattern)) |
| # elif simple-exclude-pattern: # "! distutils/foo*" |
| # xfiles = glob (pattern) |
| # remove all xfiles from files |
| # elif recursive-pattern: # "examples" (just a directory name) |
| # patterns = rest-of-words-on-line |
| # dir_files = list of all files under dir |
| # if patterns: |
| # if patterns[0] is an exclude-pattern: |
| # insert "*" at patterns[0] |
| # for file in dir_files: |
| # for dpattern in reverse (patterns): |
| # if file matches dpattern: |
| # if dpattern is an include-pattern: |
| # files.append (file) |
| # else: |
| # nothing, don't include it |
| # next file |
| # else: |
| # files.extend (dir_files) # ie. accept all of them |
| |
| |
| # Anyways, this is all implemented below -- BUT it is largely untested; I |
| # know it works for the simple case of distributing the Distutils, but |
| # haven't tried it on more complicated examples. Undoubtedly doing so will |
| # reveal bugs and cause delays, so I'm waiting until after I've released |
| # Distutils 0.1. |
| |
| |
| # Other things we need to look for in creating a source distribution: |
| # - make sure there's a README |
| # - make sure the distribution meta-info is supplied and non-empty |
| # (*must* have name, version, ((author and author_email) or |
| # (maintainer and maintainer_email)), url |
| # |
| # Frills: |
| # - make sure the setup script is called "setup.py" |
| # - make sure the README refers to "setup.py" (ie. has a line matching |
| # /^\s*python\s+setup\.py/) |
| |
| # A crazy idea that conflicts with having/requiring 'version' in setup.py: |
| # - make sure there's a version number in the "main file" (main file |
| # is __init__.py of first package, or the first module if no packages, |
| # or the first extension module if no pure Python modules) |
| # - XXX how do we look for __version__ in an extension module? |
| # - XXX do we import and look for __version__? or just scan source for |
| # /^__version__\s*=\s*"[^"]+"/ ? |
| # - what about 'version_from' as an alternative to 'version' -- then |
| # we know just where to search for the version -- no guessing about |
| # what the "main file" is |
| |
| |
| |
| class Dist (Command): |
| |
| options = [('formats=', 'f', |
| "formats for source distribution (tar, ztar, gztar, or zip)"), |
| ('manifest=', 'm', |
| "name of manifest file"), |
| ('list-only', 'l', |
| "just list files that would be distributed"), |
| ] |
| |
| default_format = { 'posix': 'gztar', |
| 'nt': 'zip' } |
| |
| exclude_re = re.compile (r'\s*!\s*(\S+)') # for manifest lines |
| |
| |
| def set_default_options (self): |
| self.formats = None |
| self.manifest = None |
| self.list_only = 0 |
| |
| |
| def set_final_options (self): |
| if self.formats is None: |
| try: |
| self.formats = [self.default_format[os.name]] |
| except KeyError: |
| raise DistutilsPlatformError, \ |
| "don't know how to build source distributions on " + \ |
| "%s platform" % os.name |
| elif type (self.formats) is StringType: |
| self.formats = string.split (self.formats, ',') |
| |
| if self.manifest is None: |
| self.manifest = "MANIFEST" |
| |
| |
| def run (self): |
| |
| self.check_metadata () |
| |
| self.files = [] |
| self.find_defaults () |
| self.read_manifest () |
| |
| if self.list_only: |
| for f in self.files: |
| print f |
| |
| else: |
| self.make_distribution () |
| |
| |
| def check_metadata (self): |
| |
| dist = self.distribution |
| |
| missing = [] |
| for attr in ('name', 'version', 'url'): |
| if not (hasattr (dist, attr) and getattr (dist, attr)): |
| missing.append (attr) |
| |
| if missing: |
| self.warn ("missing required meta-data: " + |
| string.join (missing, ", ")) |
| |
| if dist.author: |
| if not dist.author_email: |
| self.warn ("missing meta-data: if 'author' supplied, " + |
| "'author_email' must be supplied too") |
| elif dist.maintainer: |
| if not dist.maintainer_email: |
| self.warn ("missing meta-data: if 'maintainer' supplied, " + |
| "'maintainer_email' must be supplied too") |
| else: |
| self.warn ("missing meta-data: either author (and author_email) " + |
| "or maintainer (and maintainer_email) " + |
| "must be supplied") |
| |
| # check_metadata () |
| |
| |
| def find_defaults (self): |
| |
| standards = ['README', 'setup.py'] |
| for fn in standards: |
| if os.path.exists (fn): |
| self.files.append (fn) |
| else: |
| self.warn ("standard file %s not found" % fn) |
| |
| optional = ['test/test*.py'] |
| for pattern in optional: |
| files = glob (pattern) |
| if files: |
| self.files.extend (files) |
| |
| if self.distribution.packages or self.distribution.py_modules: |
| build_py = self.find_peer ('build_py') |
| build_py.ensure_ready () |
| self.files.extend (build_py.get_source_files ()) |
| |
| if self.distribution.ext_modules: |
| build_ext = self.find_peer ('build_ext') |
| build_ext.ensure_ready () |
| self.files.extend (build_ext.get_source_files ()) |
| |
| |
| |
| def open_manifest (self, filename): |
| return TextFile (filename, |
| strip_comments=1, |
| skip_blanks=1, |
| join_lines=1, |
| lstrip_ws=1, |
| rstrip_ws=1, |
| collapse_ws=1) |
| |
| |
| def search_dir (self, dir, patterns): |
| |
| allfiles = findall (dir) |
| if patterns: |
| if patterns[0][0] == "!": # starts with an exclude spec? |
| patterns.insert (0, "*")# then accept anything that isn't |
| # explicitly excluded |
| |
| act_patterns = [] # "action-patterns": (include,regexp) |
| # tuples where include is a boolean |
| for pattern in patterns: |
| if pattern[0] == '!': |
| act_patterns.append \ |
| ((0, re.compile (fnmatch.translate (pattern[1:])))) |
| else: |
| act_patterns.append \ |
| ((1, re.compile (fnmatch.translate (pattern)))) |
| act_patterns.reverse() |
| |
| |
| files = [] |
| for file in allfiles: |
| for (include,regexp) in act_patterns: |
| if regexp.match (file): |
| if include: |
| files.append (file) |
| break # continue to next file |
| else: |
| files = allfiles |
| |
| return files |
| |
| # search_dir () |
| |
| |
| def exclude_files (self, pattern): |
| |
| regexp = re.compile (fnmatch.translate (pattern)) |
| for i in range (len (self.files)-1, -1, -1): |
| if regexp.match (self.files[i]): |
| del self.files[i] |
| |
| |
| def read_manifest (self): |
| |
| # self.files had better already be defined (and hold the |
| # "automatically found" files -- Python modules and extensions, |
| # README, setup script, ...) |
| assert self.files is not None |
| |
| manifest = self.open_manifest (self.manifest) |
| while 1: |
| |
| pattern = manifest.readline() |
| if pattern is None: # end of file |
| break |
| |
| # Cases: |
| # 1) simple-include: "*.py", "foo/*.py", "doc/*.html", "FAQ" |
| # 2) simple-exclude: same, prefaced by ! |
| # 3) recursive: multi-word line, first word a directory |
| |
| exclude = self.exclude_re.match (pattern) |
| if exclude: |
| pattern = exclude.group (1) |
| |
| words = string.split (pattern) |
| assert words # must have something! |
| if os.name != 'posix': |
| words[0] = apply (os.path.join, string.split (words[0], '/')) |
| |
| # First word is a directory, possibly with include/exclude |
| # patterns making up the rest of the line: it's a recursive |
| # pattern |
| if os.path.isdir (words[0]): |
| if exclude: |
| file.warn ("exclude (!) doesn't apply to " + |
| "whole directory trees") |
| continue |
| |
| dir_files = self.search_dir (words[0], words[1:]) |
| self.files.extend (dir_files) |
| |
| # Multiple words in pattern: that's a no-no unless the first |
| # word is a directory name |
| elif len (words) > 1: |
| file.warn ("can't have multiple words unless first word " + |
| "('%s') is a directory name" % words[0]) |
| continue |
| |
| # Single word, no bang: it's a "simple include pattern" |
| elif not exclude: |
| matches = glob (pattern) |
| if matches: |
| self.files.extend (matches) |
| else: |
| manifest.warn ("no matches for '%s' found" % pattern) |
| |
| |
| # Single word prefixed with a bang: it's a "simple exclude pattern" |
| else: |
| if self.exclude_files (pattern) == 0: |
| file.warn ("no files excluded by '%s'" % pattern) |
| |
| # if/elif/.../else on 'pattern' |
| |
| # loop over lines of 'manifest' |
| |
| # read_manifest () |
| |
| |
| def make_release_tree (self, base_dir, files): |
| |
| # XXX this is Unix-specific |
| |
| # First get the list of directories to create |
| need_dir = {} |
| for file in files: |
| need_dir[os.path.join (base_dir, os.path.dirname (file))] = 1 |
| need_dirs = need_dir.keys() |
| need_dirs.sort() |
| |
| # Now create them |
| for dir in need_dirs: |
| self.mkpath (dir) |
| |
| # And walk over the list of files, making a hard link for |
| # each one that doesn't already exist in its corresponding |
| # location under 'base_dir' |
| |
| self.announce ("making hard links in %s..." % base_dir) |
| for file in files: |
| dest = os.path.join (base_dir, file) |
| if not os.path.exists (dest): |
| self.execute (os.link, (file, dest), |
| "linking %s -> %s" % (file, dest)) |
| # make_release_tree () |
| |
| |
| def make_tarball (self, base_dir): |
| |
| # XXX GNU tar 1.13 has a nifty option to add a prefix directory. |
| # It's pretty new, though, so we certainly can't require it -- but |
| # it would be nice to take advantage of it to skip the "create a |
| # tree of hardlinks" step! |
| |
| # But I am a lazy bastard, so I require GNU tar anyways. |
| |
| archive_name = base_dir + ".tar.gz" |
| self.spawn (["tar", "-czf", archive_name, base_dir]) |
| |
| |
| def make_zipfile (self, base_dir): |
| |
| # This assumes the Unix 'zip' utility -- it could be easily recast |
| # to use pkzip (or whatever the command-line zip creation utility |
| # on Redmond's archaic CP/M knockoff is nowadays), but I'll let |
| # someone who can actually test it do that. |
| |
| self.spawn (["zip", "-r", base_dir, base_dir]) |
| |
| |
| def make_distribution (self): |
| |
| # Don't warn about missing meta-data here -- should be done |
| # elsewhere. |
| name = self.distribution.name or "UNKNOWN" |
| version = self.distribution.version |
| |
| if version: |
| base_dir = "%s-%s" % (name, version) |
| else: |
| base_dir = name |
| |
| # Remove any files that match "base_dir" from the fileset -- we |
| # don't want to go distributing the distribution inside itself! |
| self.exclude_files (base_dir + "*") |
| |
| self.make_release_tree (base_dir, self.files) |
| if 'gztar' in self.formats: |
| self.make_tarball (base_dir) |
| if 'zip' in self.formats: |
| self.make_zipfile (base_dir) |
| |
| # class Dist |
| |
| |
| # ---------------------------------------------------------------------- |
| # Utility functions |
| |
| def findall (dir = os.curdir): |
| """Find all files under 'dir' and return the sorted list of full |
| filenames (relative to 'dir').""" |
| |
| list = [] |
| stack = [dir] |
| pop = stack.pop |
| push = stack.append |
| |
| while stack: |
| dir = pop() |
| names = os.listdir (dir) |
| |
| for name in names: |
| fullname = os.path.join (dir, name) |
| list.append (fullname) |
| if os.path.isdir (fullname) and not os.path.islink(fullname): |
| push (fullname) |
| |
| list.sort() |
| return list |
| |
| |
| |
| |
| |
| # ====================================================================== |
| # Here follows some extensive mental masturbation about how to |
| # make the manifest file and search algorithm even more complex. |
| # I think this is all gratuitous, really. |
| |
| # Hmm, something extra: want to apply an exclude pattern over a whole |
| # subtree without necessarily having to explicitly include files from it, |
| # ie. it should apply after gathering files by other means (simple |
| # include pattern) |
| # . !*~ !*.bak !#*# |
| # and we also want to prune at certain directories: |
| # . !RCS !CVS |
| # which again should apply globally. |
| # |
| # possible solution: |
| # - exclude pattern in a directory applies to all files found under that |
| # directory |
| # - subdirectories that match an exclude pattern will be pruned |
| # - hmmm, to be consistent, subdirectories that match an include |
| # pattern should be recursively included |
| # - and this should apply to "simple" patterns too |
| # |
| # thus: |
| # |
| # examples/ |
| # |
| # means get everything in examples/ and all subdirs; |
| # |
| # examples/ !*~ !#*# !*.py[co] |
| # |
| # means get everything under examples/ except files matching those three globs; |
| # |
| # ./ !RCS !CVS |
| # |
| # means get everything under current dir, but prune RCS/CVS directories; |
| # |
| # ./ !*~ !#*# !*.py[co] !RCS !CVS |
| # ! build/ |
| # ! experimental/ |
| # |
| # means get everything under the distribution directory except the usual |
| # excludes at all levels; exclude "build" and "experimental" under the |
| # distribution dir only. |
| # |
| # Do the former examples still work? |
| # |
| # distutils/ *.py |
| # ! distutils/bleeding_edge.py |
| # |
| # means all .py files recursively found under distutils, except for the one |
| # explicitly named. |
| # |
| # distutils/ *.py !bleeding_edge.py |
| # |
| # means the same, except bleeding_edge.py will be excluded wherever it's |
| # found -- thus this can exclude up to one file per directory under |
| # distutils. |
| # |
| # distutils/*.py |
| # ! distutils/bleeding_edge.py |
| # |
| # gets exactly distutils/*.py, minus the one explicitly mentioned exclude, and |
| # |
| # distutils/*.py |
| # distutils/ !bleeding_edge.py |
| # |
| # coincidentally does the same, but only because there can only be one file |
| # that matches the exclude pattern. Oh, we'd still like |
| # |
| # distutils *.py !bleeding*.py |
| # distutils/bleeding_ledge.py |
| # |
| # to include distutils/bleeding_ledge.py -- i.e. it should override the |
| # earlier exclude pattern by virtue of appearing later in the manifest. Does |
| # this conflict with the above requirements, ie. that "!RCS" and "!*~" should |
| # apply everywhere? Hmm, I think it doesn't have to, as long as we're smart |
| # about it. Consequence: |
| # |
| # . !RCS !CVS |
| # distutils * |
| # |
| # will go ahead and include RCS and CVS files under distutils, but |
| # |
| # distutils * |
| # . !RCS !CVS |
| # |
| # will do the right thing. Hmmm. I think that's OK, and an inevitable |
| # consequence of the ability to override exclusions. |
| |
| # OK, new crack at the search algorithm. |
| # |
| # for pattern in manifest: |
| # if dir-pattern: # ie. first word is a directory (incl. "."!) |
| # dir = first word on line |
| # patterns = rest of line |
| # if patterns: |
| # for dpattern in patterns: |
| # if exclude-pattern: |
| # remove from files anything matching dpattern (including pruning |
| # subtrees rooted at directories that match dpattern) |
| # else: |
| # files.append (recursive_glob (dir, dpattern)) |
| # else: |
| # files.append (recursive_glob (dir, '*') |
| # |
| # elif include-pattern: # it's a "simple include pattern" |
| # files.append (glob (pattern)) |
| # |
| # else: # it's a "simple exclude pattern" |
| # remove from files anything matching pattern |
| |
| # The two removal algorithms might be a bit tricky: |
| # |
| # "remove simple exclude pattern": |
| # for f in files: |
| # if f matches pattern: |
| # delete it |
| # |
| # "remove recursive exclude pattern": |
| # for f in files: |
| # |
| # t = tail (f) |
| # while t: |
| # if t matches pattern: |
| # delete current file |
| # continue |
| # t = tail (t) |
| # |
| # Well, that was an interesting mental exercise. I'm not completely |
| # convinced it will work, nor am I convinced this level of complexity |
| # is necessary. If you want to exclude RCS or CVS directories, just |
| # don't bloody include them! |
| |
| |