Tarek Ziade | 1231a4e | 2011-05-19 13:07:25 +0200 | [diff] [blame] | 1 | """Build pure Python modules (just copy to build directory).""" |
| 2 | |
| 3 | import os |
| 4 | import sys |
| 5 | from glob import glob |
| 6 | |
| 7 | from packaging import logger |
| 8 | from packaging.command.cmd import Command |
| 9 | from packaging.errors import PackagingOptionError, PackagingFileError |
| 10 | from packaging.util import convert_path |
| 11 | from packaging.compat import Mixin2to3 |
| 12 | |
| 13 | # marking public APIs |
| 14 | __all__ = ['build_py'] |
| 15 | |
| 16 | class build_py(Command, Mixin2to3): |
| 17 | |
| 18 | description = "build pure Python modules (copy to build directory)" |
| 19 | |
| 20 | user_options = [ |
| 21 | ('build-lib=', 'd', "directory to build (copy) to"), |
| 22 | ('compile', 'c', "compile .py to .pyc"), |
| 23 | ('no-compile', None, "don't compile .py files [default]"), |
| 24 | ('optimize=', 'O', |
| 25 | "also compile with optimization: -O1 for \"python -O\", " |
| 26 | "-O2 for \"python -OO\", and -O0 to disable [default: -O0]"), |
| 27 | ('force', 'f', "forcibly build everything (ignore file timestamps)"), |
| 28 | ('use-2to3', None, |
| 29 | "use 2to3 to make source python 3.x compatible"), |
| 30 | ('convert-2to3-doctests', None, |
| 31 | "use 2to3 to convert doctests in seperate text files"), |
| 32 | ('use-2to3-fixers', None, |
| 33 | "list additional fixers opted for during 2to3 conversion"), |
| 34 | ] |
| 35 | |
| 36 | boolean_options = ['compile', 'force'] |
| 37 | negative_opt = {'no-compile' : 'compile'} |
| 38 | |
| 39 | def initialize_options(self): |
| 40 | self.build_lib = None |
| 41 | self.py_modules = None |
| 42 | self.package = None |
| 43 | self.package_data = None |
| 44 | self.package_dir = None |
| 45 | self.compile = False |
| 46 | self.optimize = 0 |
| 47 | self.force = None |
| 48 | self._updated_files = [] |
| 49 | self._doctests_2to3 = [] |
| 50 | self.use_2to3 = False |
| 51 | self.convert_2to3_doctests = None |
| 52 | self.use_2to3_fixers = None |
| 53 | |
| 54 | def finalize_options(self): |
| 55 | self.set_undefined_options('build', |
| 56 | 'use_2to3', 'use_2to3_fixers', |
| 57 | 'convert_2to3_doctests', 'build_lib', |
| 58 | 'force') |
| 59 | |
| 60 | # Get the distribution options that are aliases for build_py |
| 61 | # options -- list of packages and list of modules. |
| 62 | self.packages = self.distribution.packages |
| 63 | self.py_modules = self.distribution.py_modules |
| 64 | self.package_data = self.distribution.package_data |
| 65 | self.package_dir = None |
| 66 | if self.distribution.package_dir is not None: |
| 67 | self.package_dir = convert_path(self.distribution.package_dir) |
| 68 | self.data_files = self.get_data_files() |
| 69 | |
| 70 | # Ick, copied straight from install_lib.py (fancy_getopt needs a |
| 71 | # type system! Hell, *everything* needs a type system!!!) |
| 72 | if not isinstance(self.optimize, int): |
| 73 | try: |
| 74 | self.optimize = int(self.optimize) |
| 75 | assert 0 <= self.optimize <= 2 |
| 76 | except (ValueError, AssertionError): |
| 77 | raise PackagingOptionError("optimize must be 0, 1, or 2") |
| 78 | |
| 79 | def run(self): |
| 80 | # XXX copy_file by default preserves atime and mtime. IMHO this is |
| 81 | # the right thing to do, but perhaps it should be an option -- in |
| 82 | # particular, a site administrator might want installed files to |
| 83 | # reflect the time of installation rather than the last |
| 84 | # modification time before the installed release. |
| 85 | |
| 86 | # XXX copy_file by default preserves mode, which appears to be the |
| 87 | # wrong thing to do: if a file is read-only in the working |
| 88 | # directory, we want it to be installed read/write so that the next |
| 89 | # installation of the same module distribution can overwrite it |
| 90 | # without problems. (This might be a Unix-specific issue.) Thus |
| 91 | # we turn off 'preserve_mode' when copying to the build directory, |
| 92 | # since the build directory is supposed to be exactly what the |
| 93 | # installation will look like (ie. we preserve mode when |
| 94 | # installing). |
| 95 | |
| 96 | # Two options control which modules will be installed: 'packages' |
| 97 | # and 'py_modules'. The former lets us work with whole packages, not |
| 98 | # specifying individual modules at all; the latter is for |
| 99 | # specifying modules one-at-a-time. |
| 100 | |
| 101 | if self.py_modules: |
| 102 | self.build_modules() |
| 103 | if self.packages: |
| 104 | self.build_packages() |
| 105 | self.build_package_data() |
| 106 | |
| 107 | if self.use_2to3 and self._updated_files: |
| 108 | self.run_2to3(self._updated_files, self._doctests_2to3, |
| 109 | self.use_2to3_fixers) |
| 110 | |
| 111 | self.byte_compile(self.get_outputs(include_bytecode=False)) |
| 112 | |
| 113 | # -- Top-level worker functions ------------------------------------ |
| 114 | |
| 115 | def get_data_files(self): |
| 116 | """Generate list of '(package,src_dir,build_dir,filenames)' tuples. |
| 117 | |
| 118 | Helper function for `finalize_options()`. |
| 119 | """ |
| 120 | data = [] |
| 121 | if not self.packages: |
| 122 | return data |
| 123 | for package in self.packages: |
| 124 | # Locate package source directory |
| 125 | src_dir = self.get_package_dir(package) |
| 126 | |
| 127 | # Compute package build directory |
| 128 | build_dir = os.path.join(*([self.build_lib] + package.split('.'))) |
| 129 | |
| 130 | # Length of path to strip from found files |
| 131 | plen = 0 |
| 132 | if src_dir: |
| 133 | plen = len(src_dir)+1 |
| 134 | |
| 135 | # Strip directory from globbed filenames |
| 136 | filenames = [ |
| 137 | file[plen:] for file in self.find_data_files(package, src_dir) |
| 138 | ] |
| 139 | data.append((package, src_dir, build_dir, filenames)) |
| 140 | return data |
| 141 | |
| 142 | def find_data_files(self, package, src_dir): |
| 143 | """Return filenames for package's data files in 'src_dir'. |
| 144 | |
| 145 | Helper function for `get_data_files()`. |
| 146 | """ |
| 147 | globs = (self.package_data.get('', []) |
| 148 | + self.package_data.get(package, [])) |
| 149 | files = [] |
| 150 | for pattern in globs: |
| 151 | # Each pattern has to be converted to a platform-specific path |
| 152 | filelist = glob(os.path.join(src_dir, convert_path(pattern))) |
| 153 | # Files that match more than one pattern are only added once |
| 154 | files.extend(fn for fn in filelist if fn not in files) |
| 155 | return files |
| 156 | |
| 157 | def build_package_data(self): |
| 158 | """Copy data files into build directory. |
| 159 | |
| 160 | Helper function for `run()`. |
| 161 | """ |
| 162 | # FIXME add tests for this method |
| 163 | for package, src_dir, build_dir, filenames in self.data_files: |
| 164 | for filename in filenames: |
| 165 | target = os.path.join(build_dir, filename) |
| 166 | srcfile = os.path.join(src_dir, filename) |
| 167 | self.mkpath(os.path.dirname(target)) |
| 168 | outf, copied = self.copy_file(srcfile, |
| 169 | target, preserve_mode=False) |
| 170 | if copied and srcfile in self.distribution.convert_2to3.doctests: |
| 171 | self._doctests_2to3.append(outf) |
| 172 | |
| 173 | # XXX - this should be moved to the Distribution class as it is not |
| 174 | # only needed for build_py. It also has no dependencies on this class. |
| 175 | def get_package_dir(self, package): |
| 176 | """Return the directory, relative to the top of the source |
| 177 | distribution, where package 'package' should be found |
| 178 | (at least according to the 'package_dir' option, if any).""" |
| 179 | |
| 180 | path = package.split('.') |
| 181 | if self.package_dir is not None: |
| 182 | path.insert(0, self.package_dir) |
| 183 | |
| 184 | if len(path) > 0: |
| 185 | return os.path.join(*path) |
| 186 | |
| 187 | return '' |
| 188 | |
| 189 | def check_package(self, package, package_dir): |
| 190 | """Helper function for `find_package_modules()` and `find_modules()'. |
| 191 | """ |
| 192 | # Empty dir name means current directory, which we can probably |
| 193 | # assume exists. Also, os.path.exists and isdir don't know about |
| 194 | # my "empty string means current dir" convention, so we have to |
| 195 | # circumvent them. |
| 196 | if package_dir != "": |
| 197 | if not os.path.exists(package_dir): |
| 198 | raise PackagingFileError( |
| 199 | "package directory '%s' does not exist" % package_dir) |
| 200 | if not os.path.isdir(package_dir): |
| 201 | raise PackagingFileError( |
| 202 | "supposed package directory '%s' exists, " |
| 203 | "but is not a directory" % package_dir) |
| 204 | |
| 205 | # Require __init__.py for all but the "root package" |
| 206 | if package: |
| 207 | init_py = os.path.join(package_dir, "__init__.py") |
| 208 | if os.path.isfile(init_py): |
| 209 | return init_py |
| 210 | else: |
| 211 | logger.warning(("package init file '%s' not found " + |
| 212 | "(or not a regular file)"), init_py) |
| 213 | |
| 214 | # Either not in a package at all (__init__.py not expected), or |
| 215 | # __init__.py doesn't exist -- so don't return the filename. |
| 216 | return None |
| 217 | |
| 218 | def check_module(self, module, module_file): |
| 219 | if not os.path.isfile(module_file): |
| 220 | logger.warning("file %s (for module %s) not found", |
| 221 | module_file, module) |
| 222 | return False |
| 223 | else: |
| 224 | return True |
| 225 | |
| 226 | def find_package_modules(self, package, package_dir): |
| 227 | self.check_package(package, package_dir) |
| 228 | module_files = glob(os.path.join(package_dir, "*.py")) |
| 229 | modules = [] |
| 230 | if self.distribution.script_name is not None: |
| 231 | setup_script = os.path.abspath(self.distribution.script_name) |
| 232 | else: |
| 233 | setup_script = None |
| 234 | |
| 235 | for f in module_files: |
| 236 | abs_f = os.path.abspath(f) |
| 237 | if abs_f != setup_script: |
| 238 | module = os.path.splitext(os.path.basename(f))[0] |
| 239 | modules.append((package, module, f)) |
| 240 | else: |
| 241 | logger.debug("excluding %s", setup_script) |
| 242 | return modules |
| 243 | |
| 244 | def find_modules(self): |
| 245 | """Finds individually-specified Python modules, ie. those listed by |
| 246 | module name in 'self.py_modules'. Returns a list of tuples (package, |
| 247 | module_base, filename): 'package' is a tuple of the path through |
| 248 | package-space to the module; 'module_base' is the bare (no |
| 249 | packages, no dots) module name, and 'filename' is the path to the |
| 250 | ".py" file (relative to the distribution root) that implements the |
| 251 | module. |
| 252 | """ |
| 253 | # Map package names to tuples of useful info about the package: |
| 254 | # (package_dir, checked) |
| 255 | # package_dir - the directory where we'll find source files for |
| 256 | # this package |
| 257 | # checked - true if we have checked that the package directory |
| 258 | # is valid (exists, contains __init__.py, ... ?) |
| 259 | packages = {} |
| 260 | |
| 261 | # List of (package, module, filename) tuples to return |
| 262 | modules = [] |
| 263 | |
| 264 | # We treat modules-in-packages almost the same as toplevel modules, |
| 265 | # just the "package" for a toplevel is empty (either an empty |
| 266 | # string or empty list, depending on context). Differences: |
| 267 | # - don't check for __init__.py in directory for empty package |
| 268 | for module in self.py_modules: |
| 269 | path = module.split('.') |
| 270 | package = '.'.join(path[0:-1]) |
| 271 | module_base = path[-1] |
| 272 | |
| 273 | try: |
| 274 | package_dir, checked = packages[package] |
| 275 | except KeyError: |
| 276 | package_dir = self.get_package_dir(package) |
| 277 | checked = False |
| 278 | |
| 279 | if not checked: |
| 280 | init_py = self.check_package(package, package_dir) |
| 281 | packages[package] = (package_dir, 1) |
| 282 | if init_py: |
| 283 | modules.append((package, "__init__", init_py)) |
| 284 | |
| 285 | # XXX perhaps we should also check for just .pyc files |
| 286 | # (so greedy closed-source bastards can distribute Python |
| 287 | # modules too) |
| 288 | module_file = os.path.join(package_dir, module_base + ".py") |
| 289 | if not self.check_module(module, module_file): |
| 290 | continue |
| 291 | |
| 292 | modules.append((package, module_base, module_file)) |
| 293 | |
| 294 | return modules |
| 295 | |
| 296 | def find_all_modules(self): |
| 297 | """Compute the list of all modules that will be built, whether |
| 298 | they are specified one-module-at-a-time ('self.py_modules') or |
| 299 | by whole packages ('self.packages'). Return a list of tuples |
| 300 | (package, module, module_file), just like 'find_modules()' and |
| 301 | 'find_package_modules()' do.""" |
| 302 | modules = [] |
| 303 | if self.py_modules: |
| 304 | modules.extend(self.find_modules()) |
| 305 | if self.packages: |
| 306 | for package in self.packages: |
| 307 | package_dir = self.get_package_dir(package) |
| 308 | m = self.find_package_modules(package, package_dir) |
| 309 | modules.extend(m) |
| 310 | return modules |
| 311 | |
| 312 | def get_source_files(self): |
| 313 | sources = [module[-1] for module in self.find_all_modules()] |
| 314 | sources += [ |
| 315 | os.path.join(src_dir, filename) |
| 316 | for package, src_dir, build_dir, filenames in self.data_files |
| 317 | for filename in filenames] |
| 318 | return sources |
| 319 | |
| 320 | def get_module_outfile(self, build_dir, package, module): |
| 321 | outfile_path = [build_dir] + list(package) + [module + ".py"] |
| 322 | return os.path.join(*outfile_path) |
| 323 | |
| 324 | def get_outputs(self, include_bytecode=True): |
| 325 | modules = self.find_all_modules() |
| 326 | outputs = [] |
| 327 | for package, module, module_file in modules: |
| 328 | package = package.split('.') |
| 329 | filename = self.get_module_outfile(self.build_lib, package, module) |
| 330 | outputs.append(filename) |
| 331 | if include_bytecode: |
| 332 | if self.compile: |
| 333 | outputs.append(filename + "c") |
| 334 | if self.optimize > 0: |
| 335 | outputs.append(filename + "o") |
| 336 | |
| 337 | outputs += [ |
| 338 | os.path.join(build_dir, filename) |
| 339 | for package, src_dir, build_dir, filenames in self.data_files |
| 340 | for filename in filenames] |
| 341 | |
| 342 | return outputs |
| 343 | |
| 344 | def build_module(self, module, module_file, package): |
| 345 | if isinstance(package, str): |
| 346 | package = package.split('.') |
| 347 | elif not isinstance(package, (list, tuple)): |
| 348 | raise TypeError( |
| 349 | "'package' must be a string (dot-separated), list, or tuple") |
| 350 | |
| 351 | # Now put the module source file into the "build" area -- this is |
| 352 | # easy, we just copy it somewhere under self.build_lib (the build |
| 353 | # directory for Python source). |
| 354 | outfile = self.get_module_outfile(self.build_lib, package, module) |
| 355 | dir = os.path.dirname(outfile) |
| 356 | self.mkpath(dir) |
| 357 | return self.copy_file(module_file, outfile, preserve_mode=False) |
| 358 | |
| 359 | def build_modules(self): |
| 360 | modules = self.find_modules() |
| 361 | for package, module, module_file in modules: |
| 362 | |
| 363 | # Now "build" the module -- ie. copy the source file to |
| 364 | # self.build_lib (the build directory for Python source). |
| 365 | # (Actually, it gets copied to the directory for this package |
| 366 | # under self.build_lib.) |
| 367 | self.build_module(module, module_file, package) |
| 368 | |
| 369 | def build_packages(self): |
| 370 | for package in self.packages: |
| 371 | |
| 372 | # Get list of (package, module, module_file) tuples based on |
| 373 | # scanning the package directory. 'package' is only included |
| 374 | # in the tuple so that 'find_modules()' and |
| 375 | # 'find_package_tuples()' have a consistent interface; it's |
| 376 | # ignored here (apart from a sanity check). Also, 'module' is |
| 377 | # the *unqualified* module name (ie. no dots, no package -- we |
| 378 | # already know its package!), and 'module_file' is the path to |
| 379 | # the .py file, relative to the current directory |
| 380 | # (ie. including 'package_dir'). |
| 381 | package_dir = self.get_package_dir(package) |
| 382 | modules = self.find_package_modules(package, package_dir) |
| 383 | |
| 384 | # Now loop over the modules we found, "building" each one (just |
| 385 | # copy it to self.build_lib). |
| 386 | for package_, module, module_file in modules: |
| 387 | assert package == package_ |
| 388 | self.build_module(module, module_file, package) |
| 389 | |
| 390 | def byte_compile(self, files): |
| 391 | if hasattr(sys, 'dont_write_bytecode') and sys.dont_write_bytecode: |
| 392 | logger.warning('%s: byte-compiling is disabled, skipping.', |
| 393 | self.get_command_name()) |
| 394 | return |
| 395 | |
Éric Araujo | 95fc53f | 2011-09-01 05:11:29 +0200 | [diff] [blame] | 396 | from packaging.util import byte_compile # FIXME use compileall |
Tarek Ziade | 1231a4e | 2011-05-19 13:07:25 +0200 | [diff] [blame] | 397 | prefix = self.build_lib |
| 398 | if prefix[-1] != os.sep: |
| 399 | prefix = prefix + os.sep |
| 400 | |
| 401 | # XXX this code is essentially the same as the 'byte_compile() |
| 402 | # method of the "install_lib" command, except for the determination |
| 403 | # of the 'prefix' string. Hmmm. |
| 404 | |
| 405 | if self.compile: |
| 406 | byte_compile(files, optimize=0, |
| 407 | force=self.force, prefix=prefix, dry_run=self.dry_run) |
| 408 | if self.optimize > 0: |
| 409 | byte_compile(files, optimize=self.optimize, |
| 410 | force=self.force, prefix=prefix, dry_run=self.dry_run) |