Ben Murdoch | 097c5b2 | 2016-05-18 11:27:45 +0100 | [diff] [blame^] | 1 | # Copyright 2013 The Chromium Authors. All rights reserved. |
| 2 | # Use of this source code is governed by a BSD-style license that can be |
| 3 | # found in the LICENSE file. |
| 4 | |
| 5 | import ast |
| 6 | import contextlib |
| 7 | import fnmatch |
| 8 | import json |
| 9 | import os |
| 10 | import pipes |
| 11 | import re |
| 12 | import shlex |
| 13 | import shutil |
| 14 | import stat |
| 15 | import subprocess |
| 16 | import sys |
| 17 | import tempfile |
| 18 | import zipfile |
| 19 | |
| 20 | # Some clients do not add //build/android/gyp to PYTHONPATH. |
| 21 | import md5_check # pylint: disable=relative-import |
| 22 | |
| 23 | sys.path.append(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)) |
| 24 | from pylib.constants import host_paths |
| 25 | |
| 26 | COLORAMA_ROOT = os.path.join(host_paths.DIR_SOURCE_ROOT, |
| 27 | 'third_party', 'colorama', 'src') |
| 28 | # aapt should ignore OWNERS files in addition the default ignore pattern. |
| 29 | AAPT_IGNORE_PATTERN = ('!OWNERS:!.svn:!.git:!.ds_store:!*.scc:.*:<dir>_*:' + |
| 30 | '!CVS:!thumbs.db:!picasa.ini:!*~:!*.d.stamp') |
| 31 | _HERMETIC_TIMESTAMP = (2001, 1, 1, 0, 0, 0) |
| 32 | _HERMETIC_FILE_ATTR = (0644 << 16L) |
| 33 | |
| 34 | |
| 35 | @contextlib.contextmanager |
| 36 | def TempDir(): |
| 37 | dirname = tempfile.mkdtemp() |
| 38 | try: |
| 39 | yield dirname |
| 40 | finally: |
| 41 | shutil.rmtree(dirname) |
| 42 | |
| 43 | |
| 44 | def MakeDirectory(dir_path): |
| 45 | try: |
| 46 | os.makedirs(dir_path) |
| 47 | except OSError: |
| 48 | pass |
| 49 | |
| 50 | |
| 51 | def DeleteDirectory(dir_path): |
| 52 | if os.path.exists(dir_path): |
| 53 | shutil.rmtree(dir_path) |
| 54 | |
| 55 | |
| 56 | def Touch(path, fail_if_missing=False): |
| 57 | if fail_if_missing and not os.path.exists(path): |
| 58 | raise Exception(path + ' doesn\'t exist.') |
| 59 | |
| 60 | MakeDirectory(os.path.dirname(path)) |
| 61 | with open(path, 'a'): |
| 62 | os.utime(path, None) |
| 63 | |
| 64 | |
| 65 | def FindInDirectory(directory, filename_filter): |
| 66 | files = [] |
| 67 | for root, _dirnames, filenames in os.walk(directory): |
| 68 | matched_files = fnmatch.filter(filenames, filename_filter) |
| 69 | files.extend((os.path.join(root, f) for f in matched_files)) |
| 70 | return files |
| 71 | |
| 72 | |
| 73 | def FindInDirectories(directories, filename_filter): |
| 74 | all_files = [] |
| 75 | for directory in directories: |
| 76 | all_files.extend(FindInDirectory(directory, filename_filter)) |
| 77 | return all_files |
| 78 | |
| 79 | |
| 80 | def ParseGnList(gn_string): |
| 81 | # TODO(brettw) bug 573132: This doesn't handle GN escaping properly, so any |
| 82 | # weird characters like $ or \ in the strings will be corrupted. |
| 83 | # |
| 84 | # The code should import build/gn_helpers.py and then do: |
| 85 | # parser = gn_helpers.GNValueParser(gn_string) |
| 86 | # return return parser.ParseList() |
| 87 | # As of this writing, though, there is a CastShell build script that sends |
| 88 | # JSON through this function, and using correct GN parsing corrupts that. |
| 89 | # |
| 90 | # We need to be consistent about passing either JSON or GN lists through |
| 91 | # this function. |
| 92 | return ast.literal_eval(gn_string) |
| 93 | |
| 94 | |
| 95 | def ParseGypList(gyp_string): |
| 96 | # The ninja generator doesn't support $ in strings, so use ## to |
| 97 | # represent $. |
| 98 | # TODO(cjhopman): Remove when |
| 99 | # https://code.google.com/p/gyp/issues/detail?id=327 |
| 100 | # is addressed. |
| 101 | gyp_string = gyp_string.replace('##', '$') |
| 102 | |
| 103 | if gyp_string.startswith('['): |
| 104 | return ParseGnList(gyp_string) |
| 105 | return shlex.split(gyp_string) |
| 106 | |
| 107 | |
| 108 | def CheckOptions(options, parser, required=None): |
| 109 | if not required: |
| 110 | return |
| 111 | for option_name in required: |
| 112 | if getattr(options, option_name) is None: |
| 113 | parser.error('--%s is required' % option_name.replace('_', '-')) |
| 114 | |
| 115 | |
| 116 | def WriteJson(obj, path, only_if_changed=False): |
| 117 | old_dump = None |
| 118 | if os.path.exists(path): |
| 119 | with open(path, 'r') as oldfile: |
| 120 | old_dump = oldfile.read() |
| 121 | |
| 122 | new_dump = json.dumps(obj, sort_keys=True, indent=2, separators=(',', ': ')) |
| 123 | |
| 124 | if not only_if_changed or old_dump != new_dump: |
| 125 | with open(path, 'w') as outfile: |
| 126 | outfile.write(new_dump) |
| 127 | |
| 128 | |
| 129 | def ReadJson(path): |
| 130 | with open(path, 'r') as jsonfile: |
| 131 | return json.load(jsonfile) |
| 132 | |
| 133 | |
| 134 | class CalledProcessError(Exception): |
| 135 | """This exception is raised when the process run by CheckOutput |
| 136 | exits with a non-zero exit code.""" |
| 137 | |
| 138 | def __init__(self, cwd, args, output): |
| 139 | super(CalledProcessError, self).__init__() |
| 140 | self.cwd = cwd |
| 141 | self.args = args |
| 142 | self.output = output |
| 143 | |
| 144 | def __str__(self): |
| 145 | # A user should be able to simply copy and paste the command that failed |
| 146 | # into their shell. |
| 147 | copyable_command = '( cd {}; {} )'.format(os.path.abspath(self.cwd), |
| 148 | ' '.join(map(pipes.quote, self.args))) |
| 149 | return 'Command failed: {}\n{}'.format(copyable_command, self.output) |
| 150 | |
| 151 | |
| 152 | # This can be used in most cases like subprocess.check_output(). The output, |
| 153 | # particularly when the command fails, better highlights the command's failure. |
| 154 | # If the command fails, raises a build_utils.CalledProcessError. |
| 155 | def CheckOutput(args, cwd=None, env=None, |
| 156 | print_stdout=False, print_stderr=True, |
| 157 | stdout_filter=None, |
| 158 | stderr_filter=None, |
| 159 | fail_func=lambda returncode, stderr: returncode != 0): |
| 160 | if not cwd: |
| 161 | cwd = os.getcwd() |
| 162 | |
| 163 | child = subprocess.Popen(args, |
| 164 | stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=cwd, env=env) |
| 165 | stdout, stderr = child.communicate() |
| 166 | |
| 167 | if stdout_filter is not None: |
| 168 | stdout = stdout_filter(stdout) |
| 169 | |
| 170 | if stderr_filter is not None: |
| 171 | stderr = stderr_filter(stderr) |
| 172 | |
| 173 | if fail_func(child.returncode, stderr): |
| 174 | raise CalledProcessError(cwd, args, stdout + stderr) |
| 175 | |
| 176 | if print_stdout: |
| 177 | sys.stdout.write(stdout) |
| 178 | if print_stderr: |
| 179 | sys.stderr.write(stderr) |
| 180 | |
| 181 | return stdout |
| 182 | |
| 183 | |
| 184 | def GetModifiedTime(path): |
| 185 | # For a symlink, the modified time should be the greater of the link's |
| 186 | # modified time and the modified time of the target. |
| 187 | return max(os.lstat(path).st_mtime, os.stat(path).st_mtime) |
| 188 | |
| 189 | |
| 190 | def IsTimeStale(output, inputs): |
| 191 | if not os.path.exists(output): |
| 192 | return True |
| 193 | |
| 194 | output_time = GetModifiedTime(output) |
| 195 | for i in inputs: |
| 196 | if GetModifiedTime(i) > output_time: |
| 197 | return True |
| 198 | return False |
| 199 | |
| 200 | |
| 201 | def IsDeviceReady(): |
| 202 | device_state = CheckOutput(['adb', 'get-state']) |
| 203 | return device_state.strip() == 'device' |
| 204 | |
| 205 | |
| 206 | def CheckZipPath(name): |
| 207 | if os.path.normpath(name) != name: |
| 208 | raise Exception('Non-canonical zip path: %s' % name) |
| 209 | if os.path.isabs(name): |
| 210 | raise Exception('Absolute zip path: %s' % name) |
| 211 | |
| 212 | |
| 213 | def IsSymlink(zip_file, name): |
| 214 | zi = zip_file.getinfo(name) |
| 215 | |
| 216 | # The two high-order bytes of ZipInfo.external_attr represent |
| 217 | # UNIX permissions and file type bits. |
| 218 | return stat.S_ISLNK(zi.external_attr >> 16L) |
| 219 | |
| 220 | |
| 221 | def ExtractAll(zip_path, path=None, no_clobber=True, pattern=None, |
| 222 | predicate=None): |
| 223 | if path is None: |
| 224 | path = os.getcwd() |
| 225 | elif not os.path.exists(path): |
| 226 | MakeDirectory(path) |
| 227 | |
| 228 | with zipfile.ZipFile(zip_path) as z: |
| 229 | for name in z.namelist(): |
| 230 | if name.endswith('/'): |
| 231 | continue |
| 232 | if pattern is not None: |
| 233 | if not fnmatch.fnmatch(name, pattern): |
| 234 | continue |
| 235 | if predicate and not predicate(name): |
| 236 | continue |
| 237 | CheckZipPath(name) |
| 238 | if no_clobber: |
| 239 | output_path = os.path.join(path, name) |
| 240 | if os.path.exists(output_path): |
| 241 | raise Exception( |
| 242 | 'Path already exists from zip: %s %s %s' |
| 243 | % (zip_path, name, output_path)) |
| 244 | if IsSymlink(z, name): |
| 245 | dest = os.path.join(path, name) |
| 246 | MakeDirectory(os.path.dirname(dest)) |
| 247 | os.symlink(z.read(name), dest) |
| 248 | else: |
| 249 | z.extract(name, path) |
| 250 | |
| 251 | |
| 252 | def AddToZipHermetic(zip_file, zip_path, src_path=None, data=None, |
| 253 | compress=None): |
| 254 | """Adds a file to the given ZipFile with a hard-coded modified time. |
| 255 | |
| 256 | Args: |
| 257 | zip_file: ZipFile instance to add the file to. |
| 258 | zip_path: Destination path within the zip file. |
| 259 | src_path: Path of the source file. Mutually exclusive with |data|. |
| 260 | data: File data as a string. |
| 261 | compress: Whether to enable compression. Default is take from ZipFile |
| 262 | constructor. |
| 263 | """ |
| 264 | assert (src_path is None) != (data is None), ( |
| 265 | '|src_path| and |data| are mutually exclusive.') |
| 266 | CheckZipPath(zip_path) |
| 267 | zipinfo = zipfile.ZipInfo(filename=zip_path, date_time=_HERMETIC_TIMESTAMP) |
| 268 | zipinfo.external_attr = _HERMETIC_FILE_ATTR |
| 269 | |
| 270 | if src_path and os.path.islink(src_path): |
| 271 | zipinfo.filename = zip_path |
| 272 | zipinfo.external_attr |= stat.S_IFLNK << 16L # mark as a symlink |
| 273 | zip_file.writestr(zipinfo, os.readlink(src_path)) |
| 274 | return |
| 275 | |
| 276 | if src_path: |
| 277 | with file(src_path) as f: |
| 278 | data = f.read() |
| 279 | |
| 280 | # zipfile will deflate even when it makes the file bigger. To avoid |
| 281 | # growing files, disable compression at an arbitrary cut off point. |
| 282 | if len(data) < 16: |
| 283 | compress = False |
| 284 | |
| 285 | # None converts to ZIP_STORED, when passed explicitly rather than the |
| 286 | # default passed to the ZipFile constructor. |
| 287 | compress_type = zip_file.compression |
| 288 | if compress is not None: |
| 289 | compress_type = zipfile.ZIP_DEFLATED if compress else zipfile.ZIP_STORED |
| 290 | zip_file.writestr(zipinfo, data, compress_type) |
| 291 | |
| 292 | |
| 293 | def DoZip(inputs, output, base_dir=None): |
| 294 | """Creates a zip file from a list of files. |
| 295 | |
| 296 | Args: |
| 297 | inputs: A list of paths to zip, or a list of (zip_path, fs_path) tuples. |
| 298 | output: Destination .zip file. |
| 299 | base_dir: Prefix to strip from inputs. |
| 300 | """ |
| 301 | input_tuples = [] |
| 302 | for tup in inputs: |
| 303 | if isinstance(tup, basestring): |
| 304 | tup = (os.path.relpath(tup, base_dir), tup) |
| 305 | input_tuples.append(tup) |
| 306 | |
| 307 | # Sort by zip path to ensure stable zip ordering. |
| 308 | input_tuples.sort(key=lambda tup: tup[0]) |
| 309 | with zipfile.ZipFile(output, 'w') as outfile: |
| 310 | for zip_path, fs_path in input_tuples: |
| 311 | AddToZipHermetic(outfile, zip_path, src_path=fs_path) |
| 312 | |
| 313 | |
| 314 | def ZipDir(output, base_dir): |
| 315 | """Creates a zip file from a directory.""" |
| 316 | inputs = [] |
| 317 | for root, _, files in os.walk(base_dir): |
| 318 | for f in files: |
| 319 | inputs.append(os.path.join(root, f)) |
| 320 | DoZip(inputs, output, base_dir) |
| 321 | |
| 322 | |
| 323 | def MatchesGlob(path, filters): |
| 324 | """Returns whether the given path matches any of the given glob patterns.""" |
| 325 | return filters and any(fnmatch.fnmatch(path, f) for f in filters) |
| 326 | |
| 327 | |
| 328 | def MergeZips(output, inputs, exclude_patterns=None, path_transform=None): |
| 329 | path_transform = path_transform or (lambda p, z: p) |
| 330 | added_names = set() |
| 331 | |
| 332 | with zipfile.ZipFile(output, 'w') as out_zip: |
| 333 | for in_file in inputs: |
| 334 | with zipfile.ZipFile(in_file, 'r') as in_zip: |
| 335 | in_zip._expected_crc = None |
| 336 | for info in in_zip.infolist(): |
| 337 | # Ignore directories. |
| 338 | if info.filename[-1] == '/': |
| 339 | continue |
| 340 | dst_name = path_transform(info.filename, in_file) |
| 341 | already_added = dst_name in added_names |
| 342 | if not already_added and not MatchesGlob(dst_name, exclude_patterns): |
| 343 | AddToZipHermetic(out_zip, dst_name, data=in_zip.read(info)) |
| 344 | added_names.add(dst_name) |
| 345 | |
| 346 | |
| 347 | def PrintWarning(message): |
| 348 | print 'WARNING: ' + message |
| 349 | |
| 350 | |
| 351 | def PrintBigWarning(message): |
| 352 | print '***** ' * 8 |
| 353 | PrintWarning(message) |
| 354 | print '***** ' * 8 |
| 355 | |
| 356 | |
| 357 | def GetSortedTransitiveDependencies(top, deps_func): |
| 358 | """Gets the list of all transitive dependencies in sorted order. |
| 359 | |
| 360 | There should be no cycles in the dependency graph. |
| 361 | |
| 362 | Args: |
| 363 | top: a list of the top level nodes |
| 364 | deps_func: A function that takes a node and returns its direct dependencies. |
| 365 | Returns: |
| 366 | A list of all transitive dependencies of nodes in top, in order (a node will |
| 367 | appear in the list at a higher index than all of its dependencies). |
| 368 | """ |
| 369 | def Node(dep): |
| 370 | return (dep, deps_func(dep)) |
| 371 | |
| 372 | # First: find all deps |
| 373 | unchecked_deps = list(top) |
| 374 | all_deps = set(top) |
| 375 | while unchecked_deps: |
| 376 | dep = unchecked_deps.pop() |
| 377 | new_deps = deps_func(dep).difference(all_deps) |
| 378 | unchecked_deps.extend(new_deps) |
| 379 | all_deps = all_deps.union(new_deps) |
| 380 | |
| 381 | # Then: simple, slow topological sort. |
| 382 | sorted_deps = [] |
| 383 | unsorted_deps = dict(map(Node, all_deps)) |
| 384 | while unsorted_deps: |
| 385 | for library, dependencies in unsorted_deps.items(): |
| 386 | if not dependencies.intersection(unsorted_deps.keys()): |
| 387 | sorted_deps.append(library) |
| 388 | del unsorted_deps[library] |
| 389 | |
| 390 | return sorted_deps |
| 391 | |
| 392 | |
| 393 | def GetPythonDependencies(): |
| 394 | """Gets the paths of imported non-system python modules. |
| 395 | |
| 396 | A path is assumed to be a "system" import if it is outside of chromium's |
| 397 | src/. The paths will be relative to the current directory. |
| 398 | """ |
| 399 | module_paths = (m.__file__ for m in sys.modules.itervalues() |
| 400 | if m is not None and hasattr(m, '__file__')) |
| 401 | |
| 402 | abs_module_paths = map(os.path.abspath, module_paths) |
| 403 | |
| 404 | assert os.path.isabs(host_paths.DIR_SOURCE_ROOT) |
| 405 | non_system_module_paths = [ |
| 406 | p for p in abs_module_paths if p.startswith(host_paths.DIR_SOURCE_ROOT)] |
| 407 | def ConvertPycToPy(s): |
| 408 | if s.endswith('.pyc'): |
| 409 | return s[:-1] |
| 410 | return s |
| 411 | |
| 412 | non_system_module_paths = map(ConvertPycToPy, non_system_module_paths) |
| 413 | non_system_module_paths = map(os.path.relpath, non_system_module_paths) |
| 414 | return sorted(set(non_system_module_paths)) |
| 415 | |
| 416 | |
| 417 | def AddDepfileOption(parser): |
| 418 | # TODO(agrieve): Get rid of this once we've moved to argparse. |
| 419 | if hasattr(parser, 'add_option'): |
| 420 | func = parser.add_option |
| 421 | else: |
| 422 | func = parser.add_argument |
| 423 | func('--depfile', |
| 424 | help='Path to depfile. Must be specified as the action\'s first output.') |
| 425 | |
| 426 | |
| 427 | def WriteDepfile(path, dependencies): |
| 428 | with open(path, 'w') as depfile: |
| 429 | depfile.write(path) |
| 430 | depfile.write(': ') |
| 431 | depfile.write(' '.join(dependencies)) |
| 432 | depfile.write('\n') |
| 433 | |
| 434 | |
| 435 | def ExpandFileArgs(args): |
| 436 | """Replaces file-arg placeholders in args. |
| 437 | |
| 438 | These placeholders have the form: |
| 439 | @FileArg(filename:key1:key2:...:keyn) |
| 440 | |
| 441 | The value of such a placeholder is calculated by reading 'filename' as json. |
| 442 | And then extracting the value at [key1][key2]...[keyn]. |
| 443 | |
| 444 | Note: This intentionally does not return the list of files that appear in such |
| 445 | placeholders. An action that uses file-args *must* know the paths of those |
| 446 | files prior to the parsing of the arguments (typically by explicitly listing |
| 447 | them in the action's inputs in build files). |
| 448 | """ |
| 449 | new_args = list(args) |
| 450 | file_jsons = dict() |
| 451 | r = re.compile('@FileArg\((.*?)\)') |
| 452 | for i, arg in enumerate(args): |
| 453 | match = r.search(arg) |
| 454 | if not match: |
| 455 | continue |
| 456 | |
| 457 | if match.end() != len(arg): |
| 458 | raise Exception('Unexpected characters after FileArg: ' + arg) |
| 459 | |
| 460 | lookup_path = match.group(1).split(':') |
| 461 | file_path = lookup_path[0] |
| 462 | if not file_path in file_jsons: |
| 463 | file_jsons[file_path] = ReadJson(file_path) |
| 464 | |
| 465 | expansion = file_jsons[file_path] |
| 466 | for k in lookup_path[1:]: |
| 467 | expansion = expansion[k] |
| 468 | |
| 469 | new_args[i] = arg[:match.start()] + str(expansion) |
| 470 | |
| 471 | return new_args |
| 472 | |
| 473 | |
| 474 | def CallAndWriteDepfileIfStale(function, options, record_path=None, |
| 475 | input_paths=None, input_strings=None, |
| 476 | output_paths=None, force=False, |
| 477 | pass_changes=False, |
| 478 | depfile_deps=None): |
| 479 | """Wraps md5_check.CallAndRecordIfStale() and also writes dep & stamp files. |
| 480 | |
| 481 | Depfiles and stamp files are automatically added to output_paths when present |
| 482 | in the |options| argument. They are then created after |function| is called. |
| 483 | |
| 484 | By default, only python dependencies are added to the depfile. If there are |
| 485 | other input paths that are not captured by GN deps, then they should be listed |
| 486 | in depfile_deps. It's important to write paths to the depfile that are already |
| 487 | captured by GN deps since GN args can cause GN deps to change, and such |
| 488 | changes are not immediately reflected in depfiles (http://crbug.com/589311). |
| 489 | """ |
| 490 | if not output_paths: |
| 491 | raise Exception('At least one output_path must be specified.') |
| 492 | input_paths = list(input_paths or []) |
| 493 | input_strings = list(input_strings or []) |
| 494 | output_paths = list(output_paths or []) |
| 495 | |
| 496 | python_deps = None |
| 497 | if hasattr(options, 'depfile') and options.depfile: |
| 498 | python_deps = GetPythonDependencies() |
| 499 | # List python deps in input_strings rather than input_paths since the |
| 500 | # contents of them does not change what gets written to the depfile. |
| 501 | input_strings += python_deps |
| 502 | output_paths += [options.depfile] |
| 503 | |
| 504 | stamp_file = hasattr(options, 'stamp') and options.stamp |
| 505 | if stamp_file: |
| 506 | output_paths += [stamp_file] |
| 507 | |
| 508 | def on_stale_md5(changes): |
| 509 | args = (changes,) if pass_changes else () |
| 510 | function(*args) |
| 511 | if python_deps is not None: |
| 512 | all_depfile_deps = list(python_deps) |
| 513 | if depfile_deps: |
| 514 | all_depfile_deps.extend(depfile_deps) |
| 515 | WriteDepfile(options.depfile, all_depfile_deps) |
| 516 | if stamp_file: |
| 517 | Touch(stamp_file) |
| 518 | |
| 519 | md5_check.CallAndRecordIfStale( |
| 520 | on_stale_md5, |
| 521 | record_path=record_path, |
| 522 | input_paths=input_paths, |
| 523 | input_strings=input_strings, |
| 524 | output_paths=output_paths, |
| 525 | force=force, |
| 526 | pass_changes=True) |
| 527 | |