Éric Araujo | 2e579f0 | 2010-11-20 21:53:02 +0000 | [diff] [blame] | 1 | """Module/script to byte-compile all .py files to .pyc (or .pyo) files. |
Guido van Rossum | c567b81 | 1998-01-19 23:07:55 +0000 | [diff] [blame] | 2 | |
| 3 | When called as a script with arguments, this compiles the directories |
| 4 | given as arguments recursively; the -l option prevents it from |
| 5 | recursing into directories. |
| 6 | |
| 7 | Without arguments, if compiles all modules on sys.path, without |
| 8 | recursing into subdirectories. (Even though it should do so for |
| 9 | packages -- for now, you'll have to deal with packages separately.) |
| 10 | |
| 11 | See module py_compile for details of the actual byte-compilation. |
Guido van Rossum | c567b81 | 1998-01-19 23:07:55 +0000 | [diff] [blame] | 12 | """ |
Guido van Rossum | 3bb5448 | 1994-08-29 10:52:58 +0000 | [diff] [blame] | 13 | import os |
| 14 | import sys |
Éric Araujo | 2e579f0 | 2010-11-20 21:53:02 +0000 | [diff] [blame] | 15 | import errno |
| 16 | import imp |
Guido van Rossum | 3bb5448 | 1994-08-29 10:52:58 +0000 | [diff] [blame] | 17 | import py_compile |
Brett Cannon | befb14f | 2009-02-10 02:10:16 +0000 | [diff] [blame] | 18 | import struct |
Guido van Rossum | 3bb5448 | 1994-08-29 10:52:58 +0000 | [diff] [blame] | 19 | |
Matthias Klose | c33b902 | 2010-03-16 00:36:26 +0000 | [diff] [blame] | 20 | __all__ = ["compile_dir","compile_file","compile_path"] |
Skip Montanaro | e99d5ea | 2001-01-20 19:54:20 +0000 | [diff] [blame] | 21 | |
Georg Brandl | 8334fd9 | 2010-12-04 10:26:46 +0000 | [diff] [blame] | 22 | def compile_dir(dir, maxlevels=10, ddir=None, force=False, rx=None, |
| 23 | quiet=False, legacy=False, optimize=-1): |
Guido van Rossum | c567b81 | 1998-01-19 23:07:55 +0000 | [diff] [blame] | 24 | """Byte-compile all modules in the given directory tree. |
Guido van Rossum | 3bb5448 | 1994-08-29 10:52:58 +0000 | [diff] [blame] | 25 | |
Guido van Rossum | c567b81 | 1998-01-19 23:07:55 +0000 | [diff] [blame] | 26 | Arguments (only dir is required): |
| 27 | |
| 28 | dir: the directory to byte-compile |
| 29 | maxlevels: maximum recursion level (default 10) |
R. David Murray | 94f58c3 | 2010-12-17 16:29:07 +0000 | [diff] [blame] | 30 | ddir: the directory that will be prepended to the path to the |
| 31 | file as it is compiled into each byte-code file. |
Barry Warsaw | 28a691b | 2010-04-17 00:19:56 +0000 | [diff] [blame] | 32 | force: if True, force compilation, even if timestamps are up-to-date |
| 33 | quiet: if True, be quiet during compilation |
| 34 | legacy: if True, produce legacy pyc paths instead of PEP 3147 paths |
Georg Brandl | 8334fd9 | 2010-12-04 10:26:46 +0000 | [diff] [blame] | 35 | optimize: optimization level or -1 for level of the interpreter |
Guido van Rossum | c567b81 | 1998-01-19 23:07:55 +0000 | [diff] [blame] | 36 | """ |
Martin v. Löwis | 5c137c2 | 2002-03-18 12:44:08 +0000 | [diff] [blame] | 37 | if not quiet: |
Victor Stinner | 5307126 | 2011-05-11 00:36:28 +0200 | [diff] [blame] | 38 | print('Listing {!r}...'.format(dir)) |
Guido van Rossum | c567b81 | 1998-01-19 23:07:55 +0000 | [diff] [blame] | 39 | try: |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 40 | names = os.listdir(dir) |
Andrew Svetlov | ad28c7f | 2012-12-18 22:02:39 +0200 | [diff] [blame^] | 41 | except OSError: |
Victor Stinner | 5307126 | 2011-05-11 00:36:28 +0200 | [diff] [blame] | 42 | print("Can't list {!r}".format(dir)) |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 43 | names = [] |
Guido van Rossum | c567b81 | 1998-01-19 23:07:55 +0000 | [diff] [blame] | 44 | names.sort() |
Fred Drake | 9065ea3 | 1999-03-29 20:25:40 +0000 | [diff] [blame] | 45 | success = 1 |
Guido van Rossum | c567b81 | 1998-01-19 23:07:55 +0000 | [diff] [blame] | 46 | for name in names: |
Barry Warsaw | c04317f | 2010-04-26 15:59:03 +0000 | [diff] [blame] | 47 | if name == '__pycache__': |
| 48 | continue |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 49 | fullname = os.path.join(dir, name) |
Raymond Hettinger | 8989ea6 | 2002-06-01 00:06:20 +0000 | [diff] [blame] | 50 | if ddir is not None: |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 51 | dfile = os.path.join(ddir, name) |
| 52 | else: |
| 53 | dfile = None |
Matthias Klose | c33b902 | 2010-03-16 00:36:26 +0000 | [diff] [blame] | 54 | if not os.path.isdir(fullname): |
Georg Brandl | 8334fd9 | 2010-12-04 10:26:46 +0000 | [diff] [blame] | 55 | if not compile_file(fullname, ddir, force, rx, quiet, |
| 56 | legacy, optimize): |
Matthias Klose | c33b902 | 2010-03-16 00:36:26 +0000 | [diff] [blame] | 57 | success = 0 |
Éric Araujo | 2e579f0 | 2010-11-20 21:53:02 +0000 | [diff] [blame] | 58 | elif (maxlevels > 0 and name != os.curdir and name != os.pardir and |
| 59 | os.path.isdir(fullname) and not os.path.islink(fullname)): |
Brett Cannon | befb14f | 2009-02-10 02:10:16 +0000 | [diff] [blame] | 60 | if not compile_dir(fullname, maxlevels - 1, dfile, force, rx, |
Georg Brandl | 4543846 | 2011-02-07 12:36:54 +0000 | [diff] [blame] | 61 | quiet, legacy, optimize): |
Jeremy Hylton | 12b6457 | 2001-04-18 01:20:21 +0000 | [diff] [blame] | 62 | success = 0 |
Fred Drake | 9065ea3 | 1999-03-29 20:25:40 +0000 | [diff] [blame] | 63 | return success |
Guido van Rossum | c567b81 | 1998-01-19 23:07:55 +0000 | [diff] [blame] | 64 | |
Éric Araujo | 413d7b4 | 2010-12-23 18:44:31 +0000 | [diff] [blame] | 65 | def compile_file(fullname, ddir=None, force=False, rx=None, quiet=False, |
Georg Brandl | 8334fd9 | 2010-12-04 10:26:46 +0000 | [diff] [blame] | 66 | legacy=False, optimize=-1): |
Éric Araujo | 413d7b4 | 2010-12-23 18:44:31 +0000 | [diff] [blame] | 67 | """Byte-compile one file. |
| 68 | |
| 69 | Arguments (only fullname is required): |
| 70 | |
Barry Warsaw | 28a691b | 2010-04-17 00:19:56 +0000 | [diff] [blame] | 71 | fullname: the file to byte-compile |
R. David Murray | 94f58c3 | 2010-12-17 16:29:07 +0000 | [diff] [blame] | 72 | ddir: if given, the directory name compiled in to the |
| 73 | byte-code file. |
Barry Warsaw | 28a691b | 2010-04-17 00:19:56 +0000 | [diff] [blame] | 74 | force: if True, force compilation, even if timestamps are up-to-date |
| 75 | quiet: if True, be quiet during compilation |
| 76 | legacy: if True, produce legacy pyc paths instead of PEP 3147 paths |
Georg Brandl | 8334fd9 | 2010-12-04 10:26:46 +0000 | [diff] [blame] | 77 | optimize: optimization level or -1 for level of the interpreter |
Matthias Klose | c33b902 | 2010-03-16 00:36:26 +0000 | [diff] [blame] | 78 | """ |
| 79 | success = 1 |
| 80 | name = os.path.basename(fullname) |
| 81 | if ddir is not None: |
| 82 | dfile = os.path.join(ddir, name) |
| 83 | else: |
| 84 | dfile = None |
| 85 | if rx is not None: |
| 86 | mo = rx.search(fullname) |
| 87 | if mo: |
| 88 | return success |
| 89 | if os.path.isfile(fullname): |
Barry Warsaw | 28a691b | 2010-04-17 00:19:56 +0000 | [diff] [blame] | 90 | if legacy: |
| 91 | cfile = fullname + ('c' if __debug__ else 'o') |
| 92 | else: |
Georg Brandl | 8334fd9 | 2010-12-04 10:26:46 +0000 | [diff] [blame] | 93 | if optimize >= 0: |
| 94 | cfile = imp.cache_from_source(fullname, |
| 95 | debug_override=not optimize) |
| 96 | else: |
| 97 | cfile = imp.cache_from_source(fullname) |
Barry Warsaw | 28a691b | 2010-04-17 00:19:56 +0000 | [diff] [blame] | 98 | cache_dir = os.path.dirname(cfile) |
Matthias Klose | c33b902 | 2010-03-16 00:36:26 +0000 | [diff] [blame] | 99 | head, tail = name[:-3], name[-3:] |
| 100 | if tail == '.py': |
| 101 | if not force: |
| 102 | try: |
| 103 | mtime = int(os.stat(fullname).st_mtime) |
| 104 | expect = struct.pack('<4sl', imp.get_magic(), mtime) |
Matthias Klose | c33b902 | 2010-03-16 00:36:26 +0000 | [diff] [blame] | 105 | with open(cfile, 'rb') as chandle: |
| 106 | actual = chandle.read(8) |
| 107 | if expect == actual: |
| 108 | return success |
| 109 | except IOError: |
| 110 | pass |
| 111 | if not quiet: |
Victor Stinner | 5307126 | 2011-05-11 00:36:28 +0200 | [diff] [blame] | 112 | print('Compiling {!r}...'.format(fullname)) |
Matthias Klose | c33b902 | 2010-03-16 00:36:26 +0000 | [diff] [blame] | 113 | try: |
Georg Brandl | 8334fd9 | 2010-12-04 10:26:46 +0000 | [diff] [blame] | 114 | ok = py_compile.compile(fullname, cfile, dfile, True, |
| 115 | optimize=optimize) |
Matthias Klose | c33b902 | 2010-03-16 00:36:26 +0000 | [diff] [blame] | 116 | except py_compile.PyCompileError as err: |
| 117 | if quiet: |
Victor Stinner | 5307126 | 2011-05-11 00:36:28 +0200 | [diff] [blame] | 118 | print('*** Error compiling {!r}...'.format(fullname)) |
Matthias Klose | c33b902 | 2010-03-16 00:36:26 +0000 | [diff] [blame] | 119 | else: |
| 120 | print('*** ', end='') |
Martin v. Löwis | 4b00307 | 2010-03-16 13:19:21 +0000 | [diff] [blame] | 121 | # escape non-printable characters in msg |
Barry Warsaw | 28a691b | 2010-04-17 00:19:56 +0000 | [diff] [blame] | 122 | msg = err.msg.encode(sys.stdout.encoding, |
| 123 | errors='backslashreplace') |
Martin v. Löwis | 4b00307 | 2010-03-16 13:19:21 +0000 | [diff] [blame] | 124 | msg = msg.decode(sys.stdout.encoding) |
| 125 | print(msg) |
Matthias Klose | c33b902 | 2010-03-16 00:36:26 +0000 | [diff] [blame] | 126 | success = 0 |
| 127 | except (SyntaxError, UnicodeError, IOError) as e: |
| 128 | if quiet: |
Victor Stinner | 5307126 | 2011-05-11 00:36:28 +0200 | [diff] [blame] | 129 | print('*** Error compiling {!r}...'.format(fullname)) |
Matthias Klose | c33b902 | 2010-03-16 00:36:26 +0000 | [diff] [blame] | 130 | else: |
| 131 | print('*** ', end='') |
| 132 | print(e.__class__.__name__ + ':', e) |
| 133 | success = 0 |
| 134 | else: |
| 135 | if ok == 0: |
| 136 | success = 0 |
| 137 | return success |
| 138 | |
Barry Warsaw | 28a691b | 2010-04-17 00:19:56 +0000 | [diff] [blame] | 139 | def compile_path(skip_curdir=1, maxlevels=0, force=False, quiet=False, |
Georg Brandl | 8334fd9 | 2010-12-04 10:26:46 +0000 | [diff] [blame] | 140 | legacy=False, optimize=-1): |
Guido van Rossum | c567b81 | 1998-01-19 23:07:55 +0000 | [diff] [blame] | 141 | """Byte-compile all module on sys.path. |
| 142 | |
| 143 | Arguments (all optional): |
| 144 | |
Éric Araujo | 3b371cf | 2011-09-01 20:00:33 +0200 | [diff] [blame] | 145 | skip_curdir: if true, skip current directory (default True) |
Guido van Rossum | c567b81 | 1998-01-19 23:07:55 +0000 | [diff] [blame] | 146 | maxlevels: max recursion level (default 0) |
Barry Warsaw | 28a691b | 2010-04-17 00:19:56 +0000 | [diff] [blame] | 147 | force: as for compile_dir() (default False) |
| 148 | quiet: as for compile_dir() (default False) |
| 149 | legacy: as for compile_dir() (default False) |
Georg Brandl | 8334fd9 | 2010-12-04 10:26:46 +0000 | [diff] [blame] | 150 | optimize: as for compile_dir() (default -1) |
Guido van Rossum | c567b81 | 1998-01-19 23:07:55 +0000 | [diff] [blame] | 151 | """ |
Fred Drake | 9065ea3 | 1999-03-29 20:25:40 +0000 | [diff] [blame] | 152 | success = 1 |
Guido van Rossum | c567b81 | 1998-01-19 23:07:55 +0000 | [diff] [blame] | 153 | for dir in sys.path: |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 154 | if (not dir or dir == os.curdir) and skip_curdir: |
Guido van Rossum | be19ed7 | 2007-02-09 05:37:30 +0000 | [diff] [blame] | 155 | print('Skipping current directory') |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 156 | else: |
Martin v. Löwis | 5c137c2 | 2002-03-18 12:44:08 +0000 | [diff] [blame] | 157 | success = success and compile_dir(dir, maxlevels, None, |
Barry Warsaw | 28a691b | 2010-04-17 00:19:56 +0000 | [diff] [blame] | 158 | force, quiet=quiet, |
Georg Brandl | 8334fd9 | 2010-12-04 10:26:46 +0000 | [diff] [blame] | 159 | legacy=legacy, optimize=optimize) |
Fred Drake | 9065ea3 | 1999-03-29 20:25:40 +0000 | [diff] [blame] | 160 | return success |
Guido van Rossum | 3bb5448 | 1994-08-29 10:52:58 +0000 | [diff] [blame] | 161 | |
Matthias Klose | c33b902 | 2010-03-16 00:36:26 +0000 | [diff] [blame] | 162 | |
Guido van Rossum | 3bb5448 | 1994-08-29 10:52:58 +0000 | [diff] [blame] | 163 | def main(): |
Guido van Rossum | c567b81 | 1998-01-19 23:07:55 +0000 | [diff] [blame] | 164 | """Script main program.""" |
R. David Murray | 650f147 | 2010-11-20 21:18:51 +0000 | [diff] [blame] | 165 | import argparse |
| 166 | |
| 167 | parser = argparse.ArgumentParser( |
| 168 | description='Utilities to support installing Python libraries.') |
R. David Murray | 94f58c3 | 2010-12-17 16:29:07 +0000 | [diff] [blame] | 169 | parser.add_argument('-l', action='store_const', const=0, |
| 170 | default=10, dest='maxlevels', |
| 171 | help="don't recurse into subdirectories") |
R. David Murray | 650f147 | 2010-11-20 21:18:51 +0000 | [diff] [blame] | 172 | parser.add_argument('-f', action='store_true', dest='force', |
| 173 | help='force rebuild even if timestamps are up to date') |
| 174 | parser.add_argument('-q', action='store_true', dest='quiet', |
R. David Murray | 94f58c3 | 2010-12-17 16:29:07 +0000 | [diff] [blame] | 175 | help='output only error messages') |
R. David Murray | 650f147 | 2010-11-20 21:18:51 +0000 | [diff] [blame] | 176 | parser.add_argument('-b', action='store_true', dest='legacy', |
R. David Murray | 94f58c3 | 2010-12-17 16:29:07 +0000 | [diff] [blame] | 177 | help='use legacy (pre-PEP3147) compiled file locations') |
R. David Murray | 650f147 | 2010-11-20 21:18:51 +0000 | [diff] [blame] | 178 | parser.add_argument('-d', metavar='DESTDIR', dest='ddir', default=None, |
R. David Murray | 94f58c3 | 2010-12-17 16:29:07 +0000 | [diff] [blame] | 179 | help=('directory to prepend to file paths for use in ' |
Éric Araujo | 3b371cf | 2011-09-01 20:00:33 +0200 | [diff] [blame] | 180 | 'compile-time tracebacks and in runtime ' |
R. David Murray | 94f58c3 | 2010-12-17 16:29:07 +0000 | [diff] [blame] | 181 | 'tracebacks in cases where the source file is ' |
| 182 | 'unavailable')) |
R. David Murray | 650f147 | 2010-11-20 21:18:51 +0000 | [diff] [blame] | 183 | parser.add_argument('-x', metavar='REGEXP', dest='rx', default=None, |
Éric Araujo | 3b371cf | 2011-09-01 20:00:33 +0200 | [diff] [blame] | 184 | help=('skip files matching the regular expression; ' |
| 185 | 'the regexp is searched for in the full path ' |
| 186 | 'of each file considered for compilation')) |
R. David Murray | 650f147 | 2010-11-20 21:18:51 +0000 | [diff] [blame] | 187 | parser.add_argument('-i', metavar='FILE', dest='flist', |
R. David Murray | 94f58c3 | 2010-12-17 16:29:07 +0000 | [diff] [blame] | 188 | help=('add all the files and directories listed in ' |
Éric Araujo | 3b371cf | 2011-09-01 20:00:33 +0200 | [diff] [blame] | 189 | 'FILE to the list considered for compilation; ' |
| 190 | 'if "-", names are read from stdin')) |
R. David Murray | 94f58c3 | 2010-12-17 16:29:07 +0000 | [diff] [blame] | 191 | parser.add_argument('compile_dest', metavar='FILE|DIR', nargs='*', |
| 192 | help=('zero or more file and directory names ' |
| 193 | 'to compile; if no arguments given, defaults ' |
| 194 | 'to the equivalent of -l sys.path')) |
R. David Murray | 650f147 | 2010-11-20 21:18:51 +0000 | [diff] [blame] | 195 | args = parser.parse_args() |
| 196 | |
R. David Murray | 95333e3 | 2010-12-14 22:32:50 +0000 | [diff] [blame] | 197 | compile_dests = args.compile_dest |
| 198 | |
| 199 | if (args.ddir and (len(compile_dests) != 1 |
| 200 | or not os.path.isdir(compile_dests[0]))): |
| 201 | parser.exit('-d destdir requires exactly one directory argument') |
R. David Murray | 650f147 | 2010-11-20 21:18:51 +0000 | [diff] [blame] | 202 | if args.rx: |
| 203 | import re |
| 204 | args.rx = re.compile(args.rx) |
| 205 | |
| 206 | # if flist is provided then load it |
R. David Murray | 650f147 | 2010-11-20 21:18:51 +0000 | [diff] [blame] | 207 | if args.flist: |
R. David Murray | 95333e3 | 2010-12-14 22:32:50 +0000 | [diff] [blame] | 208 | try: |
| 209 | with (sys.stdin if args.flist=='-' else open(args.flist)) as f: |
| 210 | for line in f: |
| 211 | compile_dests.append(line.strip()) |
Andrew Svetlov | 3438fa4 | 2012-12-17 23:35:18 +0200 | [diff] [blame] | 212 | except OSError: |
R. David Murray | 95333e3 | 2010-12-14 22:32:50 +0000 | [diff] [blame] | 213 | print("Error reading file list {}".format(args.flist)) |
| 214 | return False |
R. David Murray | 650f147 | 2010-11-20 21:18:51 +0000 | [diff] [blame] | 215 | |
R. David Murray | 95333e3 | 2010-12-14 22:32:50 +0000 | [diff] [blame] | 216 | success = True |
Guido van Rossum | c567b81 | 1998-01-19 23:07:55 +0000 | [diff] [blame] | 217 | try: |
R. David Murray | 650f147 | 2010-11-20 21:18:51 +0000 | [diff] [blame] | 218 | if compile_dests: |
| 219 | for dest in compile_dests: |
R. David Murray | 5317e9c | 2010-12-16 19:08:51 +0000 | [diff] [blame] | 220 | if os.path.isfile(dest): |
| 221 | if not compile_file(dest, args.ddir, args.force, args.rx, |
| 222 | args.quiet, args.legacy): |
| 223 | success = False |
| 224 | else: |
R. David Murray | 650f147 | 2010-11-20 21:18:51 +0000 | [diff] [blame] | 225 | if not compile_dir(dest, args.maxlevels, args.ddir, |
| 226 | args.force, args.rx, args.quiet, |
| 227 | args.legacy): |
R. David Murray | 95333e3 | 2010-12-14 22:32:50 +0000 | [diff] [blame] | 228 | success = False |
R. David Murray | 95333e3 | 2010-12-14 22:32:50 +0000 | [diff] [blame] | 229 | return success |
Guido van Rossum | 45e2fbc | 1998-03-26 21:13:24 +0000 | [diff] [blame] | 230 | else: |
R. David Murray | 650f147 | 2010-11-20 21:18:51 +0000 | [diff] [blame] | 231 | return compile_path(legacy=args.legacy) |
Guido van Rossum | c567b81 | 1998-01-19 23:07:55 +0000 | [diff] [blame] | 232 | except KeyboardInterrupt: |
Éric Araujo | 2e579f0 | 2010-11-20 21:53:02 +0000 | [diff] [blame] | 233 | print("\n[interrupted]") |
R. David Murray | 95333e3 | 2010-12-14 22:32:50 +0000 | [diff] [blame] | 234 | return False |
| 235 | return True |
R. David Murray | 650f147 | 2010-11-20 21:18:51 +0000 | [diff] [blame] | 236 | |
Guido van Rossum | 3bb5448 | 1994-08-29 10:52:58 +0000 | [diff] [blame] | 237 | |
| 238 | if __name__ == '__main__': |
Raymond Hettinger | 7b4b788 | 2004-12-20 00:29:29 +0000 | [diff] [blame] | 239 | exit_status = int(not main()) |
Jeremy Hylton | 12b6457 | 2001-04-18 01:20:21 +0000 | [diff] [blame] | 240 | sys.exit(exit_status) |