| """Class representing the list of files in a distribution. |
| |
| The Manifest class can be used to: |
| |
| - read or write a MANIFEST file |
| - read a template file and find out the file list |
| """ |
| # XXX todo: document + add tests |
| import re |
| import os |
| import fnmatch |
| |
| from packaging import logger |
| from packaging.util import write_file, convert_path |
| from packaging.errors import (PackagingTemplateError, |
| PackagingInternalError) |
| |
| __all__ = ['Manifest'] |
| |
| # a \ followed by some spaces + EOL |
| _COLLAPSE_PATTERN = re.compile('\\\w*\n', re.M) |
| _COMMENTED_LINE = re.compile('#.*?(?=\n)|\n(?=$)', re.M | re.S) |
| |
| |
| class Manifest(object): |
| """A list of files built by on exploring the filesystem and filtered by |
| applying various patterns to what we find there. |
| """ |
| |
| def __init__(self): |
| self.allfiles = None |
| self.files = [] |
| |
| # |
| # Public API |
| # |
| |
| def findall(self, dir=os.curdir): |
| self.allfiles = _findall(dir) |
| |
| def append(self, item): |
| self.files.append(item) |
| |
| def extend(self, items): |
| self.files.extend(items) |
| |
| def sort(self): |
| # Not a strict lexical sort! |
| self.files = [os.path.join(*path_tuple) for path_tuple in |
| sorted(os.path.split(path) for path in self.files)] |
| |
| def clear(self): |
| """Clear all collected files.""" |
| self.files = [] |
| if self.allfiles is not None: |
| self.allfiles = [] |
| |
| def remove_duplicates(self): |
| # Assumes list has been sorted! |
| for i in range(len(self.files) - 1, 0, -1): |
| if self.files[i] == self.files[i - 1]: |
| del self.files[i] |
| |
| def read_template(self, path_or_file): |
| """Read and parse a manifest template file. |
| 'path' can be a path or a file-like object. |
| |
| Updates the list accordingly. |
| """ |
| if isinstance(path_or_file, str): |
| f = open(path_or_file) |
| else: |
| f = path_or_file |
| |
| try: |
| content = f.read() |
| # first, let's unwrap collapsed lines |
| content = _COLLAPSE_PATTERN.sub('', content) |
| # next, let's remove commented lines and empty lines |
| content = _COMMENTED_LINE.sub('', content) |
| |
| # now we have our cleaned up lines |
| lines = [line.strip() for line in content.split('\n')] |
| finally: |
| f.close() |
| |
| for line in lines: |
| if line == '': |
| continue |
| try: |
| self._process_template_line(line) |
| except PackagingTemplateError as msg: |
| logger.warning("%s, %s", path_or_file, msg) |
| |
| def write(self, path): |
| """Write the file list in 'self.filelist' (presumably as filled in |
| by 'add_defaults()' and 'read_template()') to the manifest file |
| named by 'self.manifest'. |
| """ |
| if os.path.isfile(path): |
| with open(path) as fp: |
| first_line = fp.readline() |
| |
| if first_line != '# file GENERATED by packaging, do NOT edit\n': |
| logger.info("not writing to manually maintained " |
| "manifest file %r", path) |
| return |
| |
| self.sort() |
| self.remove_duplicates() |
| content = self.files[:] |
| content.insert(0, '# file GENERATED by packaging, do NOT edit') |
| logger.info("writing manifest file %r", path) |
| write_file(path, content) |
| |
| def read(self, path): |
| """Read the manifest file (named by 'self.manifest') and use it to |
| fill in 'self.filelist', the list of files to include in the source |
| distribution. |
| """ |
| logger.info("reading manifest file %r", path) |
| with open(path) as manifest: |
| for line in manifest.readlines(): |
| self.append(line) |
| |
| def exclude_pattern(self, pattern, anchor=True, prefix=None, |
| is_regex=False): |
| """Remove strings (presumably filenames) from 'files' that match |
| 'pattern'. |
| |
| Other parameters are the same as for 'include_pattern()', above. |
| The list 'self.files' is modified in place. Return True if files are |
| found. |
| """ |
| files_found = False |
| pattern_re = _translate_pattern(pattern, anchor, prefix, is_regex) |
| for i in range(len(self.files) - 1, -1, -1): |
| if pattern_re.search(self.files[i]): |
| del self.files[i] |
| files_found = True |
| |
| return files_found |
| |
| # |
| # Private API |
| # |
| |
| def _parse_template_line(self, line): |
| words = line.split() |
| if len(words) == 1 and words[0] not in ( |
| 'include', 'exclude', 'global-include', 'global-exclude', |
| 'recursive-include', 'recursive-exclude', 'graft', 'prune'): |
| # no action given, let's use the default 'include' |
| words.insert(0, 'include') |
| |
| action = words[0] |
| patterns = dir = dir_pattern = None |
| |
| if action in ('include', 'exclude', |
| 'global-include', 'global-exclude'): |
| if len(words) < 2: |
| raise PackagingTemplateError( |
| "%r expects <pattern1> <pattern2> ..." % action) |
| |
| patterns = [convert_path(word) for word in words[1:]] |
| |
| elif action in ('recursive-include', 'recursive-exclude'): |
| if len(words) < 3: |
| raise PackagingTemplateError( |
| "%r expects <dir> <pattern1> <pattern2> ..." % action) |
| |
| dir = convert_path(words[1]) |
| patterns = [convert_path(word) for word in words[2:]] |
| |
| elif action in ('graft', 'prune'): |
| if len(words) != 2: |
| raise PackagingTemplateError( |
| "%r expects a single <dir_pattern>" % action) |
| |
| dir_pattern = convert_path(words[1]) |
| |
| else: |
| raise PackagingTemplateError("unknown action %r" % action) |
| |
| return action, patterns, dir, dir_pattern |
| |
| def _process_template_line(self, line): |
| # Parse the line: split it up, make sure the right number of words |
| # is there, and return the relevant words. 'action' is always |
| # defined: it's the first word of the line. Which of the other |
| # three are defined depends on the action; it'll be either |
| # patterns, (dir and patterns), or (dir_pattern). |
| action, patterns, dir, dir_pattern = self._parse_template_line(line) |
| |
| # OK, now we know that the action is valid and we have the |
| # right number of words on the line for that action -- so we |
| # can proceed with minimal error-checking. |
| if action == 'include': |
| for pattern in patterns: |
| if not self._include_pattern(pattern, anchor=True): |
| logger.warning("no files found matching %r", pattern) |
| |
| elif action == 'exclude': |
| for pattern in patterns: |
| if not self.exclude_pattern(pattern, anchor=True): |
| logger.warning("no previously-included files " |
| "found matching %r", pattern) |
| |
| elif action == 'global-include': |
| for pattern in patterns: |
| if not self._include_pattern(pattern, anchor=False): |
| logger.warning("no files found matching %r " |
| "anywhere in distribution", pattern) |
| |
| elif action == 'global-exclude': |
| for pattern in patterns: |
| if not self.exclude_pattern(pattern, anchor=False): |
| logger.warning("no previously-included files " |
| "matching %r found anywhere in " |
| "distribution", pattern) |
| |
| elif action == 'recursive-include': |
| for pattern in patterns: |
| if not self._include_pattern(pattern, prefix=dir): |
| logger.warning("no files found matching %r " |
| "under directory %r", pattern, dir) |
| |
| elif action == 'recursive-exclude': |
| for pattern in patterns: |
| if not self.exclude_pattern(pattern, prefix=dir): |
| logger.warning("no previously-included files " |
| "matching %r found under directory %r", |
| pattern, dir) |
| |
| elif action == 'graft': |
| if not self._include_pattern(None, prefix=dir_pattern): |
| logger.warning("no directories found matching %r", |
| dir_pattern) |
| |
| elif action == 'prune': |
| if not self.exclude_pattern(None, prefix=dir_pattern): |
| logger.warning("no previously-included directories found " |
| "matching %r", dir_pattern) |
| else: |
| raise PackagingInternalError( |
| "this cannot happen: invalid action %r" % action) |
| |
| def _include_pattern(self, pattern, anchor=True, prefix=None, |
| is_regex=False): |
| """Select strings (presumably filenames) from 'self.files' that |
| match 'pattern', a Unix-style wildcard (glob) pattern. |
| |
| Patterns are not quite the same as implemented by the 'fnmatch' |
| module: '*' and '?' match non-special characters, where "special" |
| is platform-dependent: slash on Unix; colon, slash, and backslash on |
| DOS/Windows; and colon on Mac OS. |
| |
| If 'anchor' is true (the default), then the pattern match is more |
| stringent: "*.py" will match "foo.py" but not "foo/bar.py". If |
| 'anchor' is false, both of these will match. |
| |
| If 'prefix' is supplied, then only filenames starting with 'prefix' |
| (itself a pattern) and ending with 'pattern', with anything in between |
| them, will match. 'anchor' is ignored in this case. |
| |
| If 'is_regex' is true, 'anchor' and 'prefix' are ignored, and |
| 'pattern' is assumed to be either a string containing a regex or a |
| regex object -- no translation is done, the regex is just compiled |
| and used as-is. |
| |
| Selected strings will be added to self.files. |
| |
| Return True if files are found. |
| """ |
| files_found = False |
| pattern_re = _translate_pattern(pattern, anchor, prefix, is_regex) |
| |
| # delayed loading of allfiles list |
| if self.allfiles is None: |
| self.findall() |
| |
| for name in self.allfiles: |
| if pattern_re.search(name): |
| self.files.append(name) |
| files_found = True |
| |
| return files_found |
| |
| |
| # |
| # Utility functions |
| # |
| def _findall(dir=os.curdir): |
| """Find all files under 'dir' and return the list of full filenames |
| (relative to 'dir'). |
| """ |
| from stat import S_ISREG, S_ISDIR, S_ISLNK |
| |
| list = [] |
| stack = [dir] |
| pop = stack.pop |
| push = stack.append |
| |
| while stack: |
| dir = pop() |
| names = os.listdir(dir) |
| |
| for name in names: |
| if dir != os.curdir: # avoid the dreaded "./" syndrome |
| fullname = os.path.join(dir, name) |
| else: |
| fullname = name |
| |
| # Avoid excess stat calls -- just one will do, thank you! |
| stat = os.stat(fullname) |
| mode = stat.st_mode |
| if S_ISREG(mode): |
| list.append(fullname) |
| elif S_ISDIR(mode) and not S_ISLNK(mode): |
| push(fullname) |
| |
| return list |
| |
| |
| def _glob_to_re(pattern): |
| """Translate a shell-like glob pattern to a regular expression. |
| |
| Return a string containing the regex. Differs from |
| 'fnmatch.translate()' in that '*' does not match "special characters" |
| (which are platform-specific). |
| """ |
| pattern_re = fnmatch.translate(pattern) |
| |
| # '?' and '*' in the glob pattern become '.' and '.*' in the RE, which |
| # IMHO is wrong -- '?' and '*' aren't supposed to match slash in Unix, |
| # and by extension they shouldn't match such "special characters" under |
| # any OS. So change all non-escaped dots in the RE to match any |
| # character except the special characters. |
| # XXX currently the "special characters" are just slash -- i.e. this is |
| # Unix-only. |
| pattern_re = re.sub(r'((?<!\\)(\\\\)*)\.', r'\1[^/]', pattern_re) |
| |
| return pattern_re |
| |
| |
| def _translate_pattern(pattern, anchor=True, prefix=None, is_regex=False): |
| """Translate a shell-like wildcard pattern to a compiled regular |
| expression. |
| |
| Return the compiled regex. If 'is_regex' true, |
| then 'pattern' is directly compiled to a regex (if it's a string) |
| or just returned as-is (assumes it's a regex object). |
| """ |
| if is_regex: |
| if isinstance(pattern, str): |
| return re.compile(pattern) |
| else: |
| return pattern |
| |
| if pattern: |
| pattern_re = _glob_to_re(pattern) |
| else: |
| pattern_re = '' |
| |
| if prefix is not None: |
| # ditch end of pattern character |
| empty_pattern = _glob_to_re('') |
| prefix_re = _glob_to_re(prefix)[:-len(empty_pattern)] |
| # paths should always use / in manifest templates |
| pattern_re = "^%s/.*%s" % (prefix_re, pattern_re) |
| else: # no prefix -- respect anchor flag |
| if anchor: |
| pattern_re = "^" + pattern_re |
| |
| return re.compile(pattern_re) |