Greg Ward | 2689e3d | 1999-03-22 14:52:19 +0000 | [diff] [blame] | 1 | """distutils.util |
| 2 | |
Greg Ward | aebf706 | 2000-04-04 02:05:59 +0000 | [diff] [blame] | 3 | Miscellaneous utility functions -- anything that doesn't fit into |
Greg Ward | 4752769 | 2000-09-30 18:49:14 +0000 | [diff] [blame] | 4 | one of the other *util.py modules. |
| 5 | """ |
Greg Ward | 2689e3d | 1999-03-22 14:52:19 +0000 | [diff] [blame] | 6 | |
Éric Araujo | 47a4521 | 2011-10-08 00:34:13 +0200 | [diff] [blame] | 7 | import os |
| 8 | import re |
Brett Cannon | 61c3556 | 2013-06-15 12:59:53 -0400 | [diff] [blame] | 9 | import importlib.util |
Éric Araujo | 47a4521 | 2011-10-08 00:34:13 +0200 | [diff] [blame] | 10 | import string |
Victor Stinner | 9def284 | 2016-01-18 12:15:08 +0100 | [diff] [blame] | 11 | import sys |
Greg Ward | 1297b5c | 2000-09-30 20:37:56 +0000 | [diff] [blame] | 12 | from distutils.errors import DistutilsPlatformError |
| 13 | from distutils.dep_util import newer |
Tarek Ziadé | 3679727 | 2010-07-22 12:50:05 +0000 | [diff] [blame] | 14 | from distutils.spawn import spawn |
Jeremy Hylton | cd8a114 | 2002-06-04 20:14:43 +0000 | [diff] [blame] | 15 | from distutils import log |
Tarek Ziadé | 04fe7c0 | 2009-10-25 23:08:47 +0000 | [diff] [blame] | 16 | from distutils.errors import DistutilsByteCompileError |
Greg Ward | aa458bc | 2000-04-22 15:14:58 +0000 | [diff] [blame] | 17 | |
Tarek Ziadé | 3679727 | 2010-07-22 12:50:05 +0000 | [diff] [blame] | 18 | def get_platform (): |
Benjamin Peterson | 0693063 | 2017-09-04 16:36:05 -0700 | [diff] [blame] | 19 | """Return a string that identifies the current platform. This is used mainly to |
| 20 | distinguish platform-specific build directories and platform-specific built |
| 21 | distributions. Typically includes the OS name and version and the |
| 22 | architecture (as supplied by 'os.uname()'), although the exact information |
| 23 | included depends on the OS; eg. on Linux, the kernel version isn't |
| 24 | particularly important. |
Tarek Ziadé | 8b441d0 | 2010-01-29 11:46:31 +0000 | [diff] [blame] | 25 | |
Tarek Ziadé | 3679727 | 2010-07-22 12:50:05 +0000 | [diff] [blame] | 26 | Examples of returned values: |
| 27 | linux-i586 |
| 28 | linux-alpha (?) |
| 29 | solaris-2.6-sun4u |
Tarek Ziadé | 8b441d0 | 2010-01-29 11:46:31 +0000 | [diff] [blame] | 30 | |
Tarek Ziadé | 3679727 | 2010-07-22 12:50:05 +0000 | [diff] [blame] | 31 | Windows will return one of: |
| 32 | win-amd64 (64bit Windows on AMD64 (aka x86_64, Intel64, EM64T, etc) |
Tarek Ziadé | 3679727 | 2010-07-22 12:50:05 +0000 | [diff] [blame] | 33 | win32 (all others - specifically, sys.platform is returned) |
| 34 | |
| 35 | For other non-POSIX platforms, currently just returns 'sys.platform'. |
Benjamin Peterson | 0693063 | 2017-09-04 16:36:05 -0700 | [diff] [blame] | 36 | |
Tarek Ziadé | 8b441d0 | 2010-01-29 11:46:31 +0000 | [diff] [blame] | 37 | """ |
Tarek Ziadé | 3679727 | 2010-07-22 12:50:05 +0000 | [diff] [blame] | 38 | if os.name == 'nt': |
Zachary Ware | 49ce74e | 2017-09-06 15:45:25 -0700 | [diff] [blame] | 39 | if 'amd64' in sys.version.lower(): |
Tarek Ziadé | 3679727 | 2010-07-22 12:50:05 +0000 | [diff] [blame] | 40 | return 'win-amd64' |
Tarek Ziadé | 3679727 | 2010-07-22 12:50:05 +0000 | [diff] [blame] | 41 | return sys.platform |
Tarek Ziadé | 8b441d0 | 2010-01-29 11:46:31 +0000 | [diff] [blame] | 42 | |
doko@ubuntu.com | 1abe1c5 | 2012-06-30 20:42:45 +0200 | [diff] [blame] | 43 | # Set for cross builds explicitly |
| 44 | if "_PYTHON_HOST_PLATFORM" in os.environ: |
| 45 | return os.environ["_PYTHON_HOST_PLATFORM"] |
| 46 | |
Tarek Ziadé | 3679727 | 2010-07-22 12:50:05 +0000 | [diff] [blame] | 47 | if os.name != "posix" or not hasattr(os, 'uname'): |
| 48 | # XXX what about the architecture? NT is Intel or Alpha, |
| 49 | # Mac OS is M68k or PPC, etc. |
| 50 | return sys.platform |
Tarek Ziadé | 8b441d0 | 2010-01-29 11:46:31 +0000 | [diff] [blame] | 51 | |
Tarek Ziadé | 3679727 | 2010-07-22 12:50:05 +0000 | [diff] [blame] | 52 | # Try to distinguish various flavours of Unix |
Greg Ward | 5091929 | 2000-03-07 03:27:08 +0000 | [diff] [blame] | 53 | |
Tarek Ziadé | 3679727 | 2010-07-22 12:50:05 +0000 | [diff] [blame] | 54 | (osname, host, release, version, machine) = os.uname() |
Greg Ward | 5091929 | 2000-03-07 03:27:08 +0000 | [diff] [blame] | 55 | |
Benjamin Peterson | 288d1da | 2017-09-28 22:44:27 -0700 | [diff] [blame] | 56 | # Convert the OS name to lowercase, remove '/' characters, and translate |
| 57 | # spaces (for "Power Macintosh") |
Tarek Ziadé | 3679727 | 2010-07-22 12:50:05 +0000 | [diff] [blame] | 58 | osname = osname.lower().replace('/', '') |
| 59 | machine = machine.replace(' ', '_') |
| 60 | machine = machine.replace('/', '-') |
| 61 | |
| 62 | if osname[:5] == "linux": |
| 63 | # At least on Linux/Intel, 'machine' is the processor -- |
| 64 | # i386, etc. |
| 65 | # XXX what about Alpha, SPARC, etc? |
| 66 | return "%s-%s" % (osname, machine) |
| 67 | elif osname[:5] == "sunos": |
| 68 | if release[0] >= "5": # SunOS 5 == Solaris 2 |
| 69 | osname = "solaris" |
| 70 | release = "%d.%s" % (int(release[0]) - 3, release[2:]) |
Jesus Cea | 6e35d41 | 2012-01-18 04:27:37 +0100 | [diff] [blame] | 71 | # We can't use "platform.architecture()[0]" because a |
| 72 | # bootstrap problem. We use a dict to get an error |
| 73 | # if some suspicious happens. |
| 74 | bitness = {2147483647:"32bit", 9223372036854775807:"64bit"} |
Jesus Cea | 031605a | 2012-01-18 05:04:49 +0100 | [diff] [blame] | 75 | machine += ".%s" % bitness[sys.maxsize] |
Tarek Ziadé | 3679727 | 2010-07-22 12:50:05 +0000 | [diff] [blame] | 76 | # fall through to standard osname-release-machine representation |
Tarek Ziadé | 3679727 | 2010-07-22 12:50:05 +0000 | [diff] [blame] | 77 | elif osname[:3] == "aix": |
| 78 | return "%s-%s.%s" % (osname, version, release) |
| 79 | elif osname[:6] == "cygwin": |
| 80 | osname = "cygwin" |
| 81 | rel_re = re.compile (r'[\d.]+', re.ASCII) |
| 82 | m = rel_re.match(release) |
| 83 | if m: |
| 84 | release = m.group() |
| 85 | elif osname[:6] == "darwin": |
Ned Deily | df8aa2b | 2012-07-21 05:36:30 -0700 | [diff] [blame] | 86 | import _osx_support, distutils.sysconfig |
| 87 | osname, release, machine = _osx_support.get_platform_osx( |
| 88 | distutils.sysconfig.get_config_vars(), |
| 89 | osname, release, machine) |
Tarek Ziadé | 3679727 | 2010-07-22 12:50:05 +0000 | [diff] [blame] | 90 | |
| 91 | return "%s-%s-%s" % (osname, release, machine) |
| 92 | |
| 93 | # get_platform () |
| 94 | |
| 95 | |
| 96 | def convert_path (pathname): |
| 97 | """Return 'pathname' as a name that will work on the native filesystem, |
Greg Ward | b8b263b | 2000-09-30 18:40:42 +0000 | [diff] [blame] | 98 | i.e. split it on '/' and put it back together again using the current |
| 99 | directory separator. Needed because filenames in the setup script are |
| 100 | always supplied in Unix style, and have to be converted to the local |
| 101 | convention before we can actually use them in the filesystem. Raises |
Greg Ward | 4752769 | 2000-09-30 18:49:14 +0000 | [diff] [blame] | 102 | ValueError on non-Unix-ish systems if 'pathname' either starts or |
| 103 | ends with a slash. |
Greg Ward | b8b263b | 2000-09-30 18:40:42 +0000 | [diff] [blame] | 104 | """ |
Greg Ward | 7ec0535 | 2000-09-22 01:05:43 +0000 | [diff] [blame] | 105 | if os.sep == '/': |
| 106 | return pathname |
Neal Norwitz | b0df6a1 | 2002-08-13 17:42:57 +0000 | [diff] [blame] | 107 | if not pathname: |
| 108 | return pathname |
| 109 | if pathname[0] == '/': |
Collin Winter | 5b7e9d7 | 2007-08-30 03:52:21 +0000 | [diff] [blame] | 110 | raise ValueError("path '%s' cannot be absolute" % pathname) |
Neal Norwitz | b0df6a1 | 2002-08-13 17:42:57 +0000 | [diff] [blame] | 111 | if pathname[-1] == '/': |
Collin Winter | 5b7e9d7 | 2007-08-30 03:52:21 +0000 | [diff] [blame] | 112 | raise ValueError("path '%s' cannot end with '/'" % pathname) |
Greg Ward | 7ec0535 | 2000-09-22 01:05:43 +0000 | [diff] [blame] | 113 | |
Neal Norwitz | 9d72bb4 | 2007-04-17 08:48:32 +0000 | [diff] [blame] | 114 | paths = pathname.split('/') |
Jack Jansen | b4cd5c1 | 2001-01-28 12:23:32 +0000 | [diff] [blame] | 115 | while '.' in paths: |
| 116 | paths.remove('.') |
| 117 | if not paths: |
| 118 | return os.curdir |
Neal Norwitz | d910855 | 2006-03-17 08:00:19 +0000 | [diff] [blame] | 119 | return os.path.join(*paths) |
Greg Ward | 5091929 | 2000-03-07 03:27:08 +0000 | [diff] [blame] | 120 | |
Tarek Ziadé | 3679727 | 2010-07-22 12:50:05 +0000 | [diff] [blame] | 121 | # convert_path () |
Greg Ward | 1b4ede5 | 2000-03-22 00:22:44 +0000 | [diff] [blame] | 122 | |
| 123 | |
Tarek Ziadé | 3679727 | 2010-07-22 12:50:05 +0000 | [diff] [blame] | 124 | def change_root (new_root, pathname): |
| 125 | """Return 'pathname' with 'new_root' prepended. If 'pathname' is |
| 126 | relative, this is equivalent to "os.path.join(new_root,pathname)". |
Greg Ward | 67f75d4 | 2000-04-27 01:53:46 +0000 | [diff] [blame] | 127 | Otherwise, it requires making 'pathname' relative and then joining the |
Greg Ward | 4b46ef9 | 2000-05-31 02:14:32 +0000 | [diff] [blame] | 128 | two, which is tricky on DOS/Windows and Mac OS. |
| 129 | """ |
| 130 | if os.name == 'posix': |
Greg Ward | be86bde | 2000-09-26 01:56:15 +0000 | [diff] [blame] | 131 | if not os.path.isabs(pathname): |
| 132 | return os.path.join(new_root, pathname) |
Greg Ward | 4b46ef9 | 2000-05-31 02:14:32 +0000 | [diff] [blame] | 133 | else: |
Greg Ward | be86bde | 2000-09-26 01:56:15 +0000 | [diff] [blame] | 134 | return os.path.join(new_root, pathname[1:]) |
Greg Ward | 67f75d4 | 2000-04-27 01:53:46 +0000 | [diff] [blame] | 135 | |
| 136 | elif os.name == 'nt': |
Greg Ward | be86bde | 2000-09-26 01:56:15 +0000 | [diff] [blame] | 137 | (drive, path) = os.path.splitdrive(pathname) |
Greg Ward | 4b46ef9 | 2000-05-31 02:14:32 +0000 | [diff] [blame] | 138 | if path[0] == '\\': |
| 139 | path = path[1:] |
Greg Ward | be86bde | 2000-09-26 01:56:15 +0000 | [diff] [blame] | 140 | return os.path.join(new_root, path) |
Greg Ward | 67f75d4 | 2000-04-27 01:53:46 +0000 | [diff] [blame] | 141 | |
Greg Ward | 67f75d4 | 2000-04-27 01:53:46 +0000 | [diff] [blame] | 142 | else: |
Tarek Ziadé | 3679727 | 2010-07-22 12:50:05 +0000 | [diff] [blame] | 143 | raise DistutilsPlatformError("nothing known about platform '%s'" % os.name) |
| 144 | |
Greg Ward | 67f75d4 | 2000-04-27 01:53:46 +0000 | [diff] [blame] | 145 | |
Gregory P. Smith | e7e35ac | 2000-05-12 00:40:00 +0000 | [diff] [blame] | 146 | _environ_checked = 0 |
Tarek Ziadé | 3679727 | 2010-07-22 12:50:05 +0000 | [diff] [blame] | 147 | def check_environ (): |
| 148 | """Ensure that 'os.environ' has all the environment variables we |
| 149 | guarantee that users can use in config files, command-line options, |
Greg Ward | b8b263b | 2000-09-30 18:40:42 +0000 | [diff] [blame] | 150 | etc. Currently this includes: |
| 151 | HOME - user's home directory (Unix only) |
| 152 | PLAT - description of the current platform, including hardware |
| 153 | and OS (see 'get_platform()') |
Greg Ward | 1b4ede5 | 2000-03-22 00:22:44 +0000 | [diff] [blame] | 154 | """ |
Gregory P. Smith | e7e35ac | 2000-05-12 00:40:00 +0000 | [diff] [blame] | 155 | global _environ_checked |
| 156 | if _environ_checked: |
| 157 | return |
| 158 | |
Guido van Rossum | e2b70bc | 2006-08-18 22:13:04 +0000 | [diff] [blame] | 159 | if os.name == 'posix' and 'HOME' not in os.environ: |
Greg Ward | 1b4ede5 | 2000-03-22 00:22:44 +0000 | [diff] [blame] | 160 | import pwd |
Greg Ward | be86bde | 2000-09-26 01:56:15 +0000 | [diff] [blame] | 161 | os.environ['HOME'] = pwd.getpwuid(os.getuid())[5] |
Greg Ward | 1b4ede5 | 2000-03-22 00:22:44 +0000 | [diff] [blame] | 162 | |
Guido van Rossum | e2b70bc | 2006-08-18 22:13:04 +0000 | [diff] [blame] | 163 | if 'PLAT' not in os.environ: |
Tarek Ziadé | 3679727 | 2010-07-22 12:50:05 +0000 | [diff] [blame] | 164 | os.environ['PLAT'] = get_platform() |
Greg Ward | 1b4ede5 | 2000-03-22 00:22:44 +0000 | [diff] [blame] | 165 | |
Gregory P. Smith | e7e35ac | 2000-05-12 00:40:00 +0000 | [diff] [blame] | 166 | _environ_checked = 1 |
| 167 | |
Greg Ward | 1b4ede5 | 2000-03-22 00:22:44 +0000 | [diff] [blame] | 168 | |
Tarek Ziadé | 3679727 | 2010-07-22 12:50:05 +0000 | [diff] [blame] | 169 | def subst_vars (s, local_vars): |
| 170 | """Perform shell/Perl-style variable substitution on 'string'. Every |
| 171 | occurrence of '$' followed by a name is considered a variable, and |
Greg Ward | 4752769 | 2000-09-30 18:49:14 +0000 | [diff] [blame] | 172 | variable is substituted by the value found in the 'local_vars' |
| 173 | dictionary, or in 'os.environ' if it's not in 'local_vars'. |
| 174 | 'os.environ' is first checked/augmented to guarantee that it contains |
| 175 | certain values: see 'check_environ()'. Raise ValueError for any |
| 176 | variables not found in either 'local_vars' or 'os.environ'. |
Greg Ward | b8b263b | 2000-09-30 18:40:42 +0000 | [diff] [blame] | 177 | """ |
Greg Ward | be86bde | 2000-09-26 01:56:15 +0000 | [diff] [blame] | 178 | check_environ() |
Greg Ward | 1b4ede5 | 2000-03-22 00:22:44 +0000 | [diff] [blame] | 179 | def _subst (match, local_vars=local_vars): |
| 180 | var_name = match.group(1) |
Guido van Rossum | e2b70bc | 2006-08-18 22:13:04 +0000 | [diff] [blame] | 181 | if var_name in local_vars: |
Greg Ward | be86bde | 2000-09-26 01:56:15 +0000 | [diff] [blame] | 182 | return str(local_vars[var_name]) |
Greg Ward | 1b4ede5 | 2000-03-22 00:22:44 +0000 | [diff] [blame] | 183 | else: |
| 184 | return os.environ[var_name] |
| 185 | |
Greg Ward | 4752769 | 2000-09-30 18:49:14 +0000 | [diff] [blame] | 186 | try: |
Jeremy Hylton | 5e2d076 | 2001-01-25 20:10:32 +0000 | [diff] [blame] | 187 | return re.sub(r'\$([a-zA-Z_][a-zA-Z_0-9]*)', _subst, s) |
Guido van Rossum | b940e11 | 2007-01-10 16:19:56 +0000 | [diff] [blame] | 188 | except KeyError as var: |
Collin Winter | 5b7e9d7 | 2007-08-30 03:52:21 +0000 | [diff] [blame] | 189 | raise ValueError("invalid variable '$%s'" % var) |
Greg Ward | 1b4ede5 | 2000-03-22 00:22:44 +0000 | [diff] [blame] | 190 | |
Tarek Ziadé | 3679727 | 2010-07-22 12:50:05 +0000 | [diff] [blame] | 191 | # subst_vars () |
Greg Ward | 7c1a6d4 | 2000-03-29 02:48:40 +0000 | [diff] [blame] | 192 | |
Tarek Ziadé | 3679727 | 2010-07-22 12:50:05 +0000 | [diff] [blame] | 193 | |
| 194 | def grok_environment_error (exc, prefix="error: "): |
Éric Araujo | fc773a2 | 2014-03-12 03:34:02 -0400 | [diff] [blame] | 195 | # Function kept for backward compatibility. |
| 196 | # Used to try clever things with EnvironmentErrors, |
| 197 | # but nowadays str(exception) produces good messages. |
| 198 | return prefix + str(exc) |
Greg Ward | 6a2a3db | 2000-06-24 20:40:02 +0000 | [diff] [blame] | 199 | |
Tarek Ziadé | 3679727 | 2010-07-22 12:50:05 +0000 | [diff] [blame] | 200 | |
Greg Ward | 6a2a3db | 2000-06-24 20:40:02 +0000 | [diff] [blame] | 201 | # Needed by 'split_quoted()' |
Martin v. Löwis | 1c0f1f9 | 2004-03-25 14:58:19 +0000 | [diff] [blame] | 202 | _wordchars_re = _squote_re = _dquote_re = None |
| 203 | def _init_regex(): |
| 204 | global _wordchars_re, _squote_re, _dquote_re |
| 205 | _wordchars_re = re.compile(r'[^\\\'\"%s ]*' % string.whitespace) |
| 206 | _squote_re = re.compile(r"'(?:[^'\\]|\\.)*'") |
| 207 | _dquote_re = re.compile(r'"(?:[^"\\]|\\.)*"') |
Greg Ward | 6a2a3db | 2000-06-24 20:40:02 +0000 | [diff] [blame] | 208 | |
Tarek Ziadé | 3679727 | 2010-07-22 12:50:05 +0000 | [diff] [blame] | 209 | def split_quoted (s): |
Greg Ward | 6a2a3db | 2000-06-24 20:40:02 +0000 | [diff] [blame] | 210 | """Split a string up according to Unix shell-like rules for quotes and |
Tarek Ziadé | 3679727 | 2010-07-22 12:50:05 +0000 | [diff] [blame] | 211 | backslashes. In short: words are delimited by spaces, as long as those |
Greg Ward | 6a2a3db | 2000-06-24 20:40:02 +0000 | [diff] [blame] | 212 | spaces are not escaped by a backslash, or inside a quoted string. |
| 213 | Single and double quotes are equivalent, and the quote characters can |
| 214 | be backslash-escaped. The backslash is stripped from any two-character |
| 215 | escape sequence, leaving only the escaped character. The quote |
| 216 | characters are stripped from any quoted string. Returns a list of |
| 217 | words. |
| 218 | """ |
Tarek Ziadé | 3679727 | 2010-07-22 12:50:05 +0000 | [diff] [blame] | 219 | |
Greg Ward | 6a2a3db | 2000-06-24 20:40:02 +0000 | [diff] [blame] | 220 | # This is a nice algorithm for splitting up a single string, since it |
| 221 | # doesn't require character-by-character examination. It was a little |
| 222 | # bit of a brain-bender to get it working right, though... |
Martin v. Löwis | 1c0f1f9 | 2004-03-25 14:58:19 +0000 | [diff] [blame] | 223 | if _wordchars_re is None: _init_regex() |
Greg Ward | 6a2a3db | 2000-06-24 20:40:02 +0000 | [diff] [blame] | 224 | |
Neal Norwitz | 9d72bb4 | 2007-04-17 08:48:32 +0000 | [diff] [blame] | 225 | s = s.strip() |
Greg Ward | 6a2a3db | 2000-06-24 20:40:02 +0000 | [diff] [blame] | 226 | words = [] |
| 227 | pos = 0 |
| 228 | |
| 229 | while s: |
| 230 | m = _wordchars_re.match(s, pos) |
| 231 | end = m.end() |
| 232 | if end == len(s): |
| 233 | words.append(s[:end]) |
| 234 | break |
| 235 | |
Greg Ward | 2b042de | 2000-08-08 14:38:13 +0000 | [diff] [blame] | 236 | if s[end] in string.whitespace: # unescaped, unquoted whitespace: now |
Greg Ward | 6a2a3db | 2000-06-24 20:40:02 +0000 | [diff] [blame] | 237 | words.append(s[:end]) # we definitely have a word delimiter |
Neal Norwitz | 9d72bb4 | 2007-04-17 08:48:32 +0000 | [diff] [blame] | 238 | s = s[end:].lstrip() |
Greg Ward | 6a2a3db | 2000-06-24 20:40:02 +0000 | [diff] [blame] | 239 | pos = 0 |
| 240 | |
| 241 | elif s[end] == '\\': # preserve whatever is being escaped; |
| 242 | # will become part of the current word |
| 243 | s = s[:end] + s[end+1:] |
| 244 | pos = end+1 |
| 245 | |
| 246 | else: |
| 247 | if s[end] == "'": # slurp singly-quoted string |
| 248 | m = _squote_re.match(s, end) |
| 249 | elif s[end] == '"': # slurp doubly-quoted string |
| 250 | m = _dquote_re.match(s, end) |
| 251 | else: |
Collin Winter | 5b7e9d7 | 2007-08-30 03:52:21 +0000 | [diff] [blame] | 252 | raise RuntimeError("this can't happen (bad char '%c')" % s[end]) |
Greg Ward | 6a2a3db | 2000-06-24 20:40:02 +0000 | [diff] [blame] | 253 | |
| 254 | if m is None: |
Collin Winter | 5b7e9d7 | 2007-08-30 03:52:21 +0000 | [diff] [blame] | 255 | raise ValueError("bad string (mismatched %s quotes?)" % s[end]) |
Greg Ward | 6a2a3db | 2000-06-24 20:40:02 +0000 | [diff] [blame] | 256 | |
| 257 | (beg, end) = m.span() |
| 258 | s = s[:beg] + s[beg+1:end-1] + s[end:] |
| 259 | pos = m.end() - 2 |
| 260 | |
| 261 | if pos >= len(s): |
| 262 | words.append(s) |
| 263 | break |
| 264 | |
| 265 | return words |
| 266 | |
Tarek Ziadé | 3679727 | 2010-07-22 12:50:05 +0000 | [diff] [blame] | 267 | # split_quoted () |
Greg Ward | 1c16ac3 | 2000-08-02 01:37:30 +0000 | [diff] [blame] | 268 | |
| 269 | |
Tarek Ziadé | 3679727 | 2010-07-22 12:50:05 +0000 | [diff] [blame] | 270 | def execute (func, args, msg=None, verbose=0, dry_run=0): |
| 271 | """Perform some action that affects the outside world (eg. by |
| 272 | writing to the filesystem). Such actions are special because they |
| 273 | are disabled by the 'dry_run' flag. This method takes care of all |
Jeremy Hylton | cd8a114 | 2002-06-04 20:14:43 +0000 | [diff] [blame] | 274 | that bureaucracy for you; all you have to do is supply the |
| 275 | function to call and an argument tuple for it (to embody the |
| 276 | "external action" being performed), and an optional message to |
| 277 | print. |
Greg Ward | 1c16ac3 | 2000-08-02 01:37:30 +0000 | [diff] [blame] | 278 | """ |
Greg Ward | 1c16ac3 | 2000-08-02 01:37:30 +0000 | [diff] [blame] | 279 | if msg is None: |
Walter Dörwald | 70a6b49 | 2004-02-12 17:35:32 +0000 | [diff] [blame] | 280 | msg = "%s%r" % (func.__name__, args) |
Fred Drake | b94b849 | 2001-12-06 20:51:35 +0000 | [diff] [blame] | 281 | if msg[-2:] == ',)': # correct for singleton tuple |
Greg Ward | 1c16ac3 | 2000-08-02 01:37:30 +0000 | [diff] [blame] | 282 | msg = msg[0:-2] + ')' |
| 283 | |
Jeremy Hylton | cd8a114 | 2002-06-04 20:14:43 +0000 | [diff] [blame] | 284 | log.info(msg) |
Greg Ward | 1c16ac3 | 2000-08-02 01:37:30 +0000 | [diff] [blame] | 285 | if not dry_run: |
Neal Norwitz | d910855 | 2006-03-17 08:00:19 +0000 | [diff] [blame] | 286 | func(*args) |
Greg Ward | 1c16ac3 | 2000-08-02 01:37:30 +0000 | [diff] [blame] | 287 | |
Greg Ward | 817dc09 | 2000-09-25 01:25:06 +0000 | [diff] [blame] | 288 | |
Tarek Ziadé | 3679727 | 2010-07-22 12:50:05 +0000 | [diff] [blame] | 289 | def strtobool (val): |
Greg Ward | 817dc09 | 2000-09-25 01:25:06 +0000 | [diff] [blame] | 290 | """Convert a string representation of truth to true (1) or false (0). |
Tim Peters | 182b5ac | 2004-07-18 06:16:08 +0000 | [diff] [blame] | 291 | |
Greg Ward | 817dc09 | 2000-09-25 01:25:06 +0000 | [diff] [blame] | 292 | True values are 'y', 'yes', 't', 'true', 'on', and '1'; false values |
| 293 | are 'n', 'no', 'f', 'false', 'off', and '0'. Raises ValueError if |
| 294 | 'val' is anything else. |
| 295 | """ |
Neal Norwitz | 9d72bb4 | 2007-04-17 08:48:32 +0000 | [diff] [blame] | 296 | val = val.lower() |
Greg Ward | 817dc09 | 2000-09-25 01:25:06 +0000 | [diff] [blame] | 297 | if val in ('y', 'yes', 't', 'true', 'on', '1'): |
| 298 | return 1 |
| 299 | elif val in ('n', 'no', 'f', 'false', 'off', '0'): |
| 300 | return 0 |
| 301 | else: |
Collin Winter | 5b7e9d7 | 2007-08-30 03:52:21 +0000 | [diff] [blame] | 302 | raise ValueError("invalid truth value %r" % (val,)) |
Greg Ward | 1297b5c | 2000-09-30 20:37:56 +0000 | [diff] [blame] | 303 | |
| 304 | |
Tarek Ziadé | 3679727 | 2010-07-22 12:50:05 +0000 | [diff] [blame] | 305 | def byte_compile (py_files, |
| 306 | optimize=0, force=0, |
| 307 | prefix=None, base_dir=None, |
| 308 | verbose=1, dry_run=0, |
| 309 | direct=None): |
Brett Cannon | f299abd | 2015-04-13 14:21:02 -0400 | [diff] [blame] | 310 | """Byte-compile a collection of Python source files to .pyc |
| 311 | files in a __pycache__ subdirectory. 'py_files' is a list |
Éric Araujo | 04ea953 | 2011-10-08 03:02:37 +0200 | [diff] [blame] | 312 | of files to compile; any files that don't end in ".py" are silently |
| 313 | skipped. 'optimize' must be one of the following: |
Brett Cannon | f299abd | 2015-04-13 14:21:02 -0400 | [diff] [blame] | 314 | 0 - don't optimize |
Greg Ward | 1297b5c | 2000-09-30 20:37:56 +0000 | [diff] [blame] | 315 | 1 - normal optimization (like "python -O") |
| 316 | 2 - extra optimization (like "python -OO") |
| 317 | If 'force' is true, all files are recompiled regardless of |
| 318 | timestamps. |
| 319 | |
| 320 | The source filename encoded in each bytecode file defaults to the |
| 321 | filenames listed in 'py_files'; you can modify these with 'prefix' and |
| 322 | 'basedir'. 'prefix' is a string that will be stripped off of each |
| 323 | source filename, and 'base_dir' is a directory name that will be |
| 324 | prepended (after 'prefix' is stripped). You can supply either or both |
| 325 | (or neither) of 'prefix' and 'base_dir', as you wish. |
| 326 | |
Jeremy Hylton | cd8a114 | 2002-06-04 20:14:43 +0000 | [diff] [blame] | 327 | If 'dry_run' is true, doesn't actually do anything that would |
| 328 | affect the filesystem. |
Greg Ward | 1297b5c | 2000-09-30 20:37:56 +0000 | [diff] [blame] | 329 | |
| 330 | Byte-compilation is either done directly in this interpreter process |
| 331 | with the standard py_compile module, or indirectly by writing a |
| 332 | temporary script and executing it. Normally, you should let |
| 333 | 'byte_compile()' figure out to use direct compilation or not (see |
| 334 | the source for details). The 'direct' flag is used by the script |
| 335 | generated in indirect mode; unless you know what you're doing, leave |
| 336 | it set to None. |
| 337 | """ |
Victor Stinner | 9def284 | 2016-01-18 12:15:08 +0100 | [diff] [blame] | 338 | |
| 339 | # Late import to fix a bootstrap issue: _posixsubprocess is built by |
| 340 | # setup.py, but setup.py uses distutils. |
| 341 | import subprocess |
| 342 | |
Tarek Ziadé | 04fe7c0 | 2009-10-25 23:08:47 +0000 | [diff] [blame] | 343 | # nothing is done if sys.dont_write_bytecode is True |
| 344 | if sys.dont_write_bytecode: |
| 345 | raise DistutilsByteCompileError('byte-compiling is disabled.') |
| 346 | |
Greg Ward | 1297b5c | 2000-09-30 20:37:56 +0000 | [diff] [blame] | 347 | # First, if the caller didn't force us into direct or indirect mode, |
| 348 | # figure out which mode we should be in. We take a conservative |
| 349 | # approach: choose direct mode *only* if the current interpreter is |
| 350 | # in debug mode and optimize is 0. If we're not in debug mode (-O |
| 351 | # or -OO), we don't know which level of optimization this |
| 352 | # interpreter is running with, so we can't do direct |
| 353 | # byte-compilation and be certain that it's the right thing. Thus, |
| 354 | # always compile indirectly if the current interpreter is in either |
| 355 | # optimize mode, or if either optimization level was requested by |
| 356 | # the caller. |
| 357 | if direct is None: |
| 358 | direct = (__debug__ and optimize == 0) |
| 359 | |
| 360 | # "Indirect" byte-compilation: write a temporary script and then |
| 361 | # run it with the appropriate flags. |
| 362 | if not direct: |
Marc-André Lemburg | 0375079 | 2002-12-03 08:45:11 +0000 | [diff] [blame] | 363 | try: |
| 364 | from tempfile import mkstemp |
| 365 | (script_fd, script_name) = mkstemp(".py") |
Brett Cannon | cd171c8 | 2013-07-04 17:43:24 -0400 | [diff] [blame] | 366 | except ImportError: |
Marc-André Lemburg | 0375079 | 2002-12-03 08:45:11 +0000 | [diff] [blame] | 367 | from tempfile import mktemp |
| 368 | (script_fd, script_name) = None, mktemp(".py") |
Jeremy Hylton | cd8a114 | 2002-06-04 20:14:43 +0000 | [diff] [blame] | 369 | log.info("writing byte-compilation script '%s'", script_name) |
Greg Ward | 1297b5c | 2000-09-30 20:37:56 +0000 | [diff] [blame] | 370 | if not dry_run: |
Marc-André Lemburg | 0375079 | 2002-12-03 08:45:11 +0000 | [diff] [blame] | 371 | if script_fd is not None: |
| 372 | script = os.fdopen(script_fd, "w") |
| 373 | else: |
| 374 | script = open(script_name, "w") |
Greg Ward | 1297b5c | 2000-09-30 20:37:56 +0000 | [diff] [blame] | 375 | |
| 376 | script.write("""\ |
| 377 | from distutils.util import byte_compile |
| 378 | files = [ |
| 379 | """) |
Greg Ward | 9216cfe | 2000-10-03 03:31:05 +0000 | [diff] [blame] | 380 | |
| 381 | # XXX would be nice to write absolute filenames, just for |
| 382 | # safety's sake (script should be more robust in the face of |
| 383 | # chdir'ing before running it). But this requires abspath'ing |
| 384 | # 'prefix' as well, and that breaks the hack in build_lib's |
| 385 | # 'byte_compile()' method that carefully tacks on a trailing |
| 386 | # slash (os.sep really) to make sure the prefix here is "just |
| 387 | # right". This whole prefix business is rather delicate -- the |
| 388 | # problem is that it's really a directory, but I'm treating it |
| 389 | # as a dumb string, so trailing slashes and so forth matter. |
| 390 | |
| 391 | #py_files = map(os.path.abspath, py_files) |
| 392 | #if prefix: |
| 393 | # prefix = os.path.abspath(prefix) |
| 394 | |
Neal Norwitz | 9d72bb4 | 2007-04-17 08:48:32 +0000 | [diff] [blame] | 395 | script.write(",\n".join(map(repr, py_files)) + "]\n") |
Greg Ward | 1297b5c | 2000-09-30 20:37:56 +0000 | [diff] [blame] | 396 | script.write(""" |
Walter Dörwald | 70a6b49 | 2004-02-12 17:35:32 +0000 | [diff] [blame] | 397 | byte_compile(files, optimize=%r, force=%r, |
| 398 | prefix=%r, base_dir=%r, |
| 399 | verbose=%r, dry_run=0, |
Greg Ward | 1297b5c | 2000-09-30 20:37:56 +0000 | [diff] [blame] | 400 | direct=1) |
Walter Dörwald | 70a6b49 | 2004-02-12 17:35:32 +0000 | [diff] [blame] | 401 | """ % (optimize, force, prefix, base_dir, verbose)) |
Greg Ward | 1297b5c | 2000-09-30 20:37:56 +0000 | [diff] [blame] | 402 | |
| 403 | script.close() |
| 404 | |
Victor Stinner | 9def284 | 2016-01-18 12:15:08 +0100 | [diff] [blame] | 405 | cmd = [sys.executable] |
| 406 | cmd.extend(subprocess._optim_args_from_interpreter_flags()) |
| 407 | cmd.append(script_name) |
Jeremy Hylton | cd8a114 | 2002-06-04 20:14:43 +0000 | [diff] [blame] | 408 | spawn(cmd, dry_run=dry_run) |
Greg Ward | 9216cfe | 2000-10-03 03:31:05 +0000 | [diff] [blame] | 409 | execute(os.remove, (script_name,), "removing %s" % script_name, |
Jeremy Hylton | cd8a114 | 2002-06-04 20:14:43 +0000 | [diff] [blame] | 410 | dry_run=dry_run) |
Fred Drake | b94b849 | 2001-12-06 20:51:35 +0000 | [diff] [blame] | 411 | |
Greg Ward | 1297b5c | 2000-09-30 20:37:56 +0000 | [diff] [blame] | 412 | # "Direct" byte-compilation: use the py_compile module to compile |
| 413 | # right here, right now. Note that the script generated in indirect |
| 414 | # mode simply calls 'byte_compile()' in direct mode, a weird sort of |
| 415 | # cross-process recursion. Hey, it works! |
| 416 | else: |
| 417 | from py_compile import compile |
| 418 | |
| 419 | for file in py_files: |
| 420 | if file[-3:] != ".py": |
Greg Ward | f217e21 | 2000-10-01 23:49:30 +0000 | [diff] [blame] | 421 | # This lets us be lazy and not filter filenames in |
| 422 | # the "install_lib" command. |
| 423 | continue |
Greg Ward | 1297b5c | 2000-09-30 20:37:56 +0000 | [diff] [blame] | 424 | |
| 425 | # Terminology from the py_compile module: |
| 426 | # cfile - byte-compiled file |
| 427 | # dfile - purported source filename (same as 'file' by default) |
Éric Araujo | 47a4521 | 2011-10-08 00:34:13 +0200 | [diff] [blame] | 428 | if optimize >= 0: |
Brett Cannon | f299abd | 2015-04-13 14:21:02 -0400 | [diff] [blame] | 429 | opt = '' if optimize == 0 else optimize |
Brett Cannon | 61c3556 | 2013-06-15 12:59:53 -0400 | [diff] [blame] | 430 | cfile = importlib.util.cache_from_source( |
Brett Cannon | f299abd | 2015-04-13 14:21:02 -0400 | [diff] [blame] | 431 | file, optimization=opt) |
Éric Araujo | 47a4521 | 2011-10-08 00:34:13 +0200 | [diff] [blame] | 432 | else: |
Brett Cannon | 61c3556 | 2013-06-15 12:59:53 -0400 | [diff] [blame] | 433 | cfile = importlib.util.cache_from_source(file) |
Greg Ward | 1297b5c | 2000-09-30 20:37:56 +0000 | [diff] [blame] | 434 | dfile = file |
| 435 | if prefix: |
| 436 | if file[:len(prefix)] != prefix: |
Tarek Ziadé | 3679727 | 2010-07-22 12:50:05 +0000 | [diff] [blame] | 437 | raise ValueError("invalid prefix: filename %r doesn't start with %r" |
| 438 | % (file, prefix)) |
Greg Ward | 1297b5c | 2000-09-30 20:37:56 +0000 | [diff] [blame] | 439 | dfile = dfile[len(prefix):] |
| 440 | if base_dir: |
| 441 | dfile = os.path.join(base_dir, dfile) |
| 442 | |
| 443 | cfile_base = os.path.basename(cfile) |
| 444 | if direct: |
| 445 | if force or newer(file, cfile): |
Jeremy Hylton | cd8a114 | 2002-06-04 20:14:43 +0000 | [diff] [blame] | 446 | log.info("byte-compiling %s to %s", file, cfile_base) |
Greg Ward | 1297b5c | 2000-09-30 20:37:56 +0000 | [diff] [blame] | 447 | if not dry_run: |
| 448 | compile(file, cfile, dfile) |
| 449 | else: |
Jeremy Hylton | cd8a114 | 2002-06-04 20:14:43 +0000 | [diff] [blame] | 450 | log.debug("skipping byte-compilation of %s to %s", |
| 451 | file, cfile_base) |
Greg Ward | 1297b5c | 2000-09-30 20:37:56 +0000 | [diff] [blame] | 452 | |
Tarek Ziadé | 3679727 | 2010-07-22 12:50:05 +0000 | [diff] [blame] | 453 | # byte_compile () |
Andrew M. Kuchling | df66df0 | 2001-03-22 03:03:41 +0000 | [diff] [blame] | 454 | |
Tarek Ziadé | 3679727 | 2010-07-22 12:50:05 +0000 | [diff] [blame] | 455 | def rfc822_escape (header): |
Andrew M. Kuchling | df66df0 | 2001-03-22 03:03:41 +0000 | [diff] [blame] | 456 | """Return a version of the string escaped for inclusion in an |
Andrew M. Kuchling | 88b0884 | 2001-03-23 17:30:26 +0000 | [diff] [blame] | 457 | RFC-822 header, by ensuring there are 8 spaces space after each newline. |
Andrew M. Kuchling | df66df0 | 2001-03-22 03:03:41 +0000 | [diff] [blame] | 458 | """ |
Tarek Ziadé | df872d4 | 2009-12-06 09:28:17 +0000 | [diff] [blame] | 459 | lines = header.split('\n') |
| 460 | sep = '\n' + 8 * ' ' |
Neal Norwitz | 9d72bb4 | 2007-04-17 08:48:32 +0000 | [diff] [blame] | 461 | return sep.join(lines) |
Martin v. Löwis | 6178db6 | 2008-12-01 04:38:52 +0000 | [diff] [blame] | 462 | |
| 463 | # 2to3 support |
| 464 | |
| 465 | def run_2to3(files, fixer_names=None, options=None, explicit=None): |
| 466 | """Invoke 2to3 on a list of Python files. |
| 467 | The files should all come from the build area, as the |
| 468 | modification is done in-place. To reduce the build time, |
| 469 | only files modified since the last invocation of this |
| 470 | function should be passed in the files argument.""" |
| 471 | |
| 472 | if not files: |
| 473 | return |
| 474 | |
| 475 | # Make this class local, to delay import of 2to3 |
| 476 | from lib2to3.refactor import RefactoringTool, get_fixers_from_package |
| 477 | class DistutilsRefactoringTool(RefactoringTool): |
| 478 | def log_error(self, msg, *args, **kw): |
| 479 | log.error(msg, *args) |
| 480 | |
| 481 | def log_message(self, msg, *args): |
| 482 | log.info(msg, *args) |
| 483 | |
| 484 | def log_debug(self, msg, *args): |
| 485 | log.debug(msg, *args) |
| 486 | |
| 487 | if fixer_names is None: |
| 488 | fixer_names = get_fixers_from_package('lib2to3.fixes') |
| 489 | r = DistutilsRefactoringTool(fixer_names, options=options) |
| 490 | r.refactor(files, write=True) |
| 491 | |
Georg Brandl | 6d4a9cf | 2009-03-31 00:34:54 +0000 | [diff] [blame] | 492 | def copydir_run_2to3(src, dest, template=None, fixer_names=None, |
| 493 | options=None, explicit=None): |
| 494 | """Recursively copy a directory, only copying new and changed files, |
| 495 | running run_2to3 over all newly copied Python modules afterward. |
| 496 | |
| 497 | If you give a template string, it's parsed like a MANIFEST.in. |
| 498 | """ |
| 499 | from distutils.dir_util import mkpath |
| 500 | from distutils.file_util import copy_file |
| 501 | from distutils.filelist import FileList |
| 502 | filelist = FileList() |
| 503 | curdir = os.getcwd() |
| 504 | os.chdir(src) |
| 505 | try: |
| 506 | filelist.findall() |
| 507 | finally: |
| 508 | os.chdir(curdir) |
| 509 | filelist.files[:] = filelist.allfiles |
| 510 | if template: |
| 511 | for line in template.splitlines(): |
| 512 | line = line.strip() |
| 513 | if not line: continue |
| 514 | filelist.process_template_line(line) |
| 515 | copied = [] |
| 516 | for filename in filelist.files: |
| 517 | outname = os.path.join(dest, filename) |
| 518 | mkpath(os.path.dirname(outname)) |
| 519 | res = copy_file(os.path.join(src, filename), outname, update=1) |
| 520 | if res[1]: copied.append(outname) |
| 521 | run_2to3([fn for fn in copied if fn.lower().endswith('.py')], |
| 522 | fixer_names=fixer_names, options=options, explicit=explicit) |
| 523 | return copied |
| 524 | |
Martin v. Löwis | 6178db6 | 2008-12-01 04:38:52 +0000 | [diff] [blame] | 525 | class Mixin2to3: |
| 526 | '''Mixin class for commands that run 2to3. |
| 527 | To configure 2to3, setup scripts may either change |
| 528 | the class variables, or inherit from individual commands |
| 529 | to override how 2to3 is invoked.''' |
| 530 | |
| 531 | # provide list of fixers to run; |
| 532 | # defaults to all from lib2to3.fixers |
| 533 | fixer_names = None |
| 534 | |
| 535 | # options dictionary |
| 536 | options = None |
| 537 | |
| 538 | # list of fixers to invoke even though they are marked as explicit |
| 539 | explicit = None |
| 540 | |
| 541 | def run_2to3(self, files): |
| 542 | return run_2to3(files, self.fixer_names, self.options, self.explicit) |