| #!/usr/bin/env python |
| # Copyright (C) 2017 The Android Open Source Project |
| # |
| # Licensed under the Apache License, Version 2.0 (the "License"); |
| # you may not use this file except in compliance with the License. |
| # You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| |
| import argparse |
| import hashlib |
| import logging |
| import os |
| import shutil |
| import subprocess |
| import sys |
| import tempfile |
| import zipfile |
| |
| from collections import namedtuple |
| from platform import system |
| |
| # The format for the deps below is the following: |
| # (target_folder, source_url, sha1, target_platform) |
| # |source_url| can be either a git repo or a http url. |
| # If a git repo, |checksum| is the SHA1 committish that will be checked out. |
| # If a http url, |checksum| is the SHA256 of the downloaded file. |
| # If the url is a .zip or .tgz file it will be automatically deflated under |
| # |target_folder|, taking care of stripping the root folder if it's a single |
| # root (to avoid ending up with buildtools/protobuf/protobuf-1.2.3/... and have |
| # instead just buildtools/protobuf). |
| # |target_platform| is either 'darwin', 'linux' or 'all' and applies the dep |
| # only on the given platform |
| Dependency = namedtuple( |
| 'Dependency', |
| ['target_folder', 'source_url', 'checksum', 'target_platform']) |
| |
| # Dependencies required to build code on the host or when targeting desktop OS. |
| BUILD_DEPS_HOST = [ |
| # GN. From https://chrome-infra-packages.appspot.com/dl/gn/gn/. |
| Dependency( |
| 'buildtools/mac/gn', |
| 'https://storage.googleapis.com/perfetto/gn-mac-1695-83dad00a', |
| '513d3adeb56b745e62af4e3ccb76b76f023c6aaa25d6a2be9a89e44cd10a4c1a', |
| 'darwin'), |
| Dependency( |
| 'buildtools/linux64/gn', |
| 'https://storage.googleapis.com/perfetto/gn-linux64-1695-83dad00a', |
| '4f589364153f182b05cd845e93407489d6ce8acc03290c897928a7bd22b20cce', |
| 'linux'), |
| |
| # clang-format |
| # From https://chromium.googlesource.com/chromium/src/buildtools/+/refs/heads/master/mac/clang-format.sha1 |
| Dependency( |
| 'buildtools/mac/clang-format', |
| 'https://storage.googleapis.com/chromium-clang-format/62bde1baa7196ad9df969fc1f06b66360b1a927b', |
| '6df686a937443cbe6efc013467a7ba5f98d3f187eb7765bb7abc6ce47626cf66', |
| 'darwin'), |
| # From https://chromium.googlesource.com/chromium/src/buildtools/+/refs/heads/master/linux64/clang-format.sha1 |
| Dependency( |
| 'buildtools/linux64/clang-format', |
| 'https://storage.googleapis.com/chromium-clang-format/1baf0089e895c989a311b6a38ed94d0e8be4c0a7', |
| 'd02a97a87e8c28898033aaf5986967b24dc47ebd5b376e1cd93e5009f22cd75e', |
| 'linux'), |
| |
| # Keep the SHA1 in sync with |clang_format_rev| in chromium //buildtools/DEPS. |
| Dependency( |
| 'buildtools/clang_format/script', |
| 'https://chromium.googlesource.com/chromium/llvm-project/cfe/tools/clang-format.git', |
| '96636aa0e9f047f17447f2d45a094d0b59ed7917', 'all'), |
| |
| # Ninja |
| Dependency( |
| 'buildtools/mac/ninja', |
| 'https://storage.googleapis.com/perfetto/ninja-mac-c15b0698da038b2bd2e8970c14c75fadc06b1add', |
| '4224b90734590b0148ad8ee63ee7b295e88e0652e4d1f4271ef2b91d880b0e19', |
| 'darwin'), |
| Dependency( |
| 'buildtools/linux64/ninja', |
| 'https://storage.googleapis.com/perfetto/ninja-linux64-c866952bda50c29a669222477309287119bbb7e8', |
| '54ac6a01362190aaabf4cf276f9c8982cdf11b225438940fdde3339be0f2ecdc', |
| 'linux'), |
| |
| # Keep in sync with Android's //external/googletest/README.version. |
| Dependency( |
| 'buildtools/googletest', |
| 'https://android.googlesource.com/platform/external/googletest.git', |
| '3f05f651ae3621db58468153e32016bc1397800b', 'all'), |
| |
| # Keep in sync with Chromium's //third_party/protobuf. |
| Dependency( |
| 'buildtools/protobuf', |
| 'https://chromium.googlesource.com/external/github.com/google/protobuf.git', |
| '6a59a2ad1f61d9696092f79b6d74368b4d7970a3', # refs/tags/v3.9.0 |
| 'all'), |
| |
| # libc++, libc++abi and libunwind for Linux where we need to rebuild the C++ |
| # lib from sources. Keep the SHA1s in sync with Chrome's src/buildtools/DEPS. |
| Dependency( |
| 'buildtools/libcxx', |
| 'https://chromium.googlesource.com/external/github.com/llvm/llvm-project/libcxx.git', |
| 'd9040c75cfea5928c804ab7c235fed06a63f743a', 'all'), |
| Dependency( |
| 'buildtools/libcxxabi', |
| 'https://chromium.googlesource.com/external/github.com/llvm/llvm-project/libcxxabi.git', |
| '196ba1aaa8ac285d94f4ea8d9836390a45360533', 'all'), |
| Dependency( |
| 'buildtools/libunwind', |
| 'https://chromium.googlesource.com/external/github.com/llvm/llvm-project/libunwind.git', |
| 'd999d54f4bca789543a2eb6c995af2d9b5a1f3ed', 'all'), |
| |
| # Keep the revision in sync with Chrome's PACKAGE_VERSION in |
| # tools/clang/scripts/update.py. |
| Dependency( |
| 'buildtools/clang.tgz', |
| 'https://commondatastorage.googleapis.com/chromium-browser-clang/Linux_x64/clang-llvmorg-12-init-5035-gd0abc757-3.tgz', |
| 'b0c3015209b6d624844ad230064eb5c9b4429a2eafd4854981e73217c563d93d', |
| 'linux'), |
| |
| # Keep in sync with chromium DEPS. |
| Dependency( |
| 'buildtools/libfuzzer', |
| 'https://chromium.googlesource.com/chromium/llvm-project/compiler-rt/lib/fuzzer.git', |
| 'debe7d2d1982e540fbd6bd78604bf001753f9e74', 'linux'), |
| |
| # Benchmarking tool. |
| Dependency( |
| 'buildtools/benchmark', |
| 'https://chromium.googlesource.com/external/github.com/google/benchmark.git', |
| '090faecb454fbd6e6e17a75ef8146acb037118d4', 'all'), |
| |
| # Libbacktrace, for stacktraces in Linux/Android debug builds. |
| # From https://github.com/ianlancetaylor/libbacktrace/archive/177940370e4a6b2509e92a0aaa9749184e64af43.zip |
| Dependency( |
| 'buildtools/libbacktrace.zip', |
| 'https://storage.googleapis.com/perfetto/libbacktrace-177940370e4a6b2509e92a0aaa9749184e64af43.zip', |
| '21ac9a4209f7aeef766c482be53a7fa365063c031c7077e2070b491202983b31', |
| 'all'), |
| |
| # Sqlite for the trace processing library. |
| # This is the amalgamated source whose compiled output is meant to be faster. |
| # We still pull the full source for the extensions (which are not available |
| # in the amalgamation). |
| Dependency( |
| 'buildtools/sqlite.zip', |
| 'https://storage.googleapis.com/perfetto/sqlite-amalgamation-3320300.zip', |
| 'e9cec01d4519e2d49b3810615237325263fe1feaceae390ee12b4a29bd73dbe2', |
| 'all'), |
| Dependency( |
| 'buildtools/sqlite_src', |
| 'https://chromium.googlesource.com/external/github.com/sqlite/sqlite.git', |
| 'ee3686eb50c0e3dbb087c9a0976f7e37e1b014ae', # refs/tags/version-3.32.3. |
| 'all'), |
| |
| # JsonCpp for legacy json import. Used only by the trace processor in |
| # standalone builds. |
| Dependency( |
| 'buildtools/jsoncpp', |
| 'https://chromium.googlesource.com/external/github.com/open-source-parsers/jsoncpp.git', |
| '6aba23f4a8628d599a9ef7fa4811c4ff6e4070e2', # refs/tags/1.9.3. |
| 'all'), |
| |
| # These dependencies are for libunwindstack, which is used by src/profiling. |
| Dependency('buildtools/android-core', |
| 'https://android.googlesource.com/platform/system/core.git', |
| '278f11b57425d3a14369049e866c597980677988', 'all'), |
| Dependency('buildtools/libbase', |
| 'https://android.googlesource.com/platform/system/libbase.git', |
| '78f1c2f83e625bdf66d55b48bdb3a301c20d2fb3', 'all'), |
| Dependency('buildtools/lzma', |
| 'https://android.googlesource.com/platform/external/lzma.git', |
| '7851dce6f4ca17f5caa1c93a4e0a45686b1d56c3', 'all'), |
| Dependency('buildtools/zlib', |
| 'https://android.googlesource.com/platform/external/zlib.git', |
| 'dfa0646a03b4e1707469e04dc931b09774968fe6', 'all'), |
| Dependency('buildtools/bionic', |
| 'https://android.googlesource.com/platform/bionic.git', |
| 'a60488109cda997dfd83832731c8527feaa2825e', 'all'), |
| |
| # Example traces for regression tests. |
| Dependency( |
| 'buildtools/test_data.zip', |
| 'https://storage.googleapis.com/perfetto/test-data-20200910-141649.zip', |
| '543b03a2ad77a056426800a610bf4e3ce6cad670dc55e12e68bc216425dd42f6', |
| 'all', |
| ), |
| |
| # Linenoise, used only by trace_processor in standalone builds. |
| Dependency('buildtools/linenoise', |
| 'https://fuchsia.googlesource.com/third_party/linenoise.git', |
| 'c894b9e59f02203dbe4e2be657572cf88c4230c3', 'all'), |
| ] |
| |
| # Dependencies required to build Android code. |
| # URLs and SHA1s taken from: |
| # - https://dl.google.com/android/repository/repository-11.xml |
| # - https://dl.google.com/android/repository/sys-img/android/sys-img.xml |
| BUILD_DEPS_ANDROID = [ |
| # Android NDK |
| Dependency( |
| 'buildtools/ndk.zip', |
| 'https://dl.google.com/android/repository/android-ndk-r17b-darwin-x86_64.zip', |
| 'd21072c04ffcf8a723a4dba3837c886bd30c18c0623a4d0ddc53850e2222d27f', |
| 'darwin'), |
| Dependency( |
| 'buildtools/ndk.zip', |
| 'https://dl.google.com/android/repository/android-ndk-r17b-linux-x86_64.zip', |
| '5dfbbdc2d3ba859fed90d0e978af87c71a91a5be1f6e1c40ba697503d48ccecd', |
| 'linux'), |
| ] |
| |
| # Dependencies required to run Android tests. |
| TEST_DEPS_ANDROID = [ |
| # Android emulator images. |
| Dependency( |
| 'buildtools/aosp-arm.zip', |
| 'https://storage.googleapis.com/perfetto/aosp-02022018-arm.zip', |
| 'f5c7a3a22ad7aa0bd14ba467e8697e1e917d306699bd25622aa4419a413b9b67', |
| 'all'), |
| |
| # platform-tools.zip contains adb binaries. |
| Dependency( |
| 'buildtools/android_sdk/platform-tools.zip', |
| 'https://dl.google.com/android/repository/platform-tools_r26.0.0-darwin.zip', |
| '98d392cbd21ca20d643c7e1605760cc49075611e317c534096b5564053f4ac8e', |
| 'darwin'), |
| Dependency( |
| 'buildtools/android_sdk/platform-tools.zip', |
| 'https://dl.google.com/android/repository/platform-tools_r26.0.0-linux.zip', |
| '90208207521d85abf0d46e3374aa4e04b7aff74e4f355c792ac334de7a77e50b', |
| 'linux'), |
| |
| # Android emulator binaries. |
| Dependency( |
| 'buildtools/emulator', |
| 'https://android.googlesource.com/platform/prebuilts/android-emulator.git', |
| '4b260028dc27bc92c39bee9129cb2ba839970956', 'all'), |
| ] |
| |
| # This variable is updated by tools/roll-catapult-trace-viewer. |
| CATAPULT_SHA256 = 'b30108e05268ce6c65bb4126b65f6bfac165d17f5c1fd285046e7e6fd76c209f' |
| |
| TYPEFACES_SHA256 = 'b3f0f14eeecd4555ae94f897ec246b2c6e046ce0ea417407553f5767e7812575' |
| |
| UI_DEPS = [ |
| Dependency( |
| 'buildtools/nodejs.tgz', |
| 'https://storage.googleapis.com/perfetto/node-v12.18.3-darwin-x64.tar.gz', |
| 'af376caf114bdd5d7e566dbf7590e9077ffc01f9b2692eb2651f31d7219a30bb', |
| 'darwin'), |
| Dependency( |
| 'buildtools/nodejs.tgz', |
| 'https://storage.googleapis.com/perfetto/node-v12.18.3-linux-x64.tar.gz', |
| '0be428afce5b24f799f3b1ab8902d9d91094b929ac58d2b1cec29436ea2d742c', |
| 'linux'), |
| Dependency( |
| 'buildtools/emsdk/emscripten.tgz', |
| 'https://storage.googleapis.com/perfetto/emscripten-1.37.40.tar.gz', |
| '40d8a095c510e5e5e2032131e0dc5d38f996172deb0415588b702146d1ea8bd5', |
| 'all'), |
| Dependency( |
| 'buildtools/emsdk/llvm.tgz', |
| 'https://storage.googleapis.com/perfetto/emscripten-llvm-e1.37.40-darwin.tar.gz', |
| 'ed51f3a467c0a5af5365abb448794734affcc2932fa19d62be233dee435c89ea', |
| 'darwin'), |
| Dependency( |
| 'buildtools/emsdk/llvm.tgz', |
| 'https://storage.googleapis.com/perfetto/emscripten-llvm-e1.37.40-static-linux.tar.gz', |
| '257da419f92d305025777f340b79ee9f3d3077af38f5e7efb0fac1060d3ea6d9', |
| 'linux'), |
| Dependency( |
| 'buildtools/catapult_trace_viewer.tgz', |
| 'https://storage.googleapis.com/perfetto/catapult_trace_viewer-%s.tar.gz' |
| % CATAPULT_SHA256, CATAPULT_SHA256, 'all'), |
| Dependency( |
| 'buildtools/typefaces.tgz', |
| 'https://storage.googleapis.com/perfetto/typefaces-%s.tar.gz' % |
| TYPEFACES_SHA256, TYPEFACES_SHA256, 'all') |
| ] |
| |
| ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) |
| UI_DIR = os.path.join(ROOT_DIR, 'ui') |
| NODE_MODULES_STATUS_FILE = os.path.join(UI_DIR, 'node_modules', '.last_install') |
| |
| |
| def DownloadURL(url, out_file): |
| subprocess.check_call(['curl', '-L', '-#', '-o', out_file, url]) |
| |
| |
| def ReadFile(path): |
| if not os.path.exists(path): |
| return None |
| with open(path) as f: |
| return f.read().strip() |
| |
| |
| def MkdirRecursive(path): |
| # Works with both relative and absolute paths |
| cwd = '/' if path.startswith('/') else ROOT_DIR |
| for part in path.split('/'): |
| cwd = os.path.join(cwd, part) |
| if not os.path.exists(cwd): |
| os.makedirs(cwd) |
| else: |
| assert (os.path.isdir(cwd)) |
| |
| |
| def HashLocalFile(path): |
| if not os.path.exists(path): |
| return None |
| with open(path, 'rb') as f: |
| return hashlib.sha256(f.read()).hexdigest() |
| |
| |
| def ExtractZipfilePreservePermissions(zf, info, path): |
| zf.extract(info.filename, path=path) |
| target_path = os.path.join(path, info.filename) |
| min_acls = 0o755 if info.filename.endswith('/') else 0o644 |
| os.chmod(target_path, (info.external_attr >> 16) | min_acls) |
| |
| |
| def IsGitRepoCheckoutOutAtRevision(path, revision): |
| return ReadFile(os.path.join(path, '.git', 'HEAD')) == revision |
| |
| |
| def CheckoutGitRepo(path, git_url, revision, check_only): |
| if IsGitRepoCheckoutOutAtRevision(path, revision): |
| return False |
| if check_only: |
| return True |
| if os.path.exists(path): |
| shutil.rmtree(path) |
| MkdirRecursive(path) |
| logging.info('Fetching %s @ %s into %s', git_url, revision, path) |
| subprocess.check_call(['git', 'init', path], cwd=path) |
| subprocess.check_call( |
| ['git', 'fetch', '--quiet', '--depth', '1', git_url, revision], cwd=path) |
| subprocess.check_call(['git', 'checkout', revision, '--quiet'], cwd=path) |
| assert (IsGitRepoCheckoutOutAtRevision(path, revision)) |
| return True |
| |
| |
| def InstallNodeModules(force_clean=False): |
| if force_clean: |
| node_modules = os.path.join(UI_DIR, 'node_modules') |
| logging.info('Clearing %s', node_modules) |
| subprocess.check_call(['git', 'clean', '-qxffd', node_modules], |
| cwd=ROOT_DIR) |
| logging.info("Running npm install in {0}".format(UI_DIR)) |
| subprocess.check_call([os.path.join(UI_DIR, 'npm'), 'install', '--no-save'], |
| cwd=UI_DIR) |
| with open(NODE_MODULES_STATUS_FILE, 'w') as f: |
| f.write(HashLocalFile(os.path.join(UI_DIR, 'package-lock.json'))) |
| |
| |
| def CheckNodeModules(): |
| """Returns True if the modules are up-to-date. |
| |
| There doesn't seem to be an easy way to check node modules versions. Instead |
| just check if package-lock.json changed since the last `npm install` call. |
| """ |
| if not os.path.exists(NODE_MODULES_STATUS_FILE): |
| return False |
| with open(NODE_MODULES_STATUS_FILE, 'r') as f: |
| actual = f.read() |
| expected = HashLocalFile(os.path.join(UI_DIR, 'package-lock.json')) |
| return expected == actual |
| |
| |
| def CheckHashes(): |
| for deps in [BUILD_DEPS_HOST, BUILD_DEPS_ANDROID, TEST_DEPS_ANDROID, UI_DEPS]: |
| for dep in deps: |
| if dep.source_url.endswith('.git'): |
| continue |
| logging.info('Downloading %s from %s', dep.target_platform, |
| dep.source_url) |
| with tempfile.NamedTemporaryFile(delete=False) as f: |
| f.close() |
| DownloadURL(dep.source_url, f.name) |
| actual_checksum = HashLocalFile(f.name) |
| os.unlink(f.name) |
| if (actual_checksum != dep.checksum): |
| logging.fatal('SHA-256 mismatch for {} expected {} was {}'.format( |
| dep.source_url, dep.checksum, actual_checksum)) |
| |
| |
| def Main(): |
| parser = argparse.ArgumentParser() |
| parser.add_argument('--android', action='store_true') |
| parser.add_argument('--ui', action='store_true') |
| parser.add_argument('--check-only') |
| parser.add_argument('--verify', help='Check all URLs', action='store_true') |
| args = parser.parse_args() |
| if args.verify: |
| CheckHashes() |
| return 0 |
| deps = BUILD_DEPS_HOST |
| if args.android: |
| deps += BUILD_DEPS_ANDROID + TEST_DEPS_ANDROID |
| if args.ui: |
| deps += UI_DEPS |
| deps_updated = False |
| nodejs_updated = False |
| |
| for dep in deps: |
| if (dep.target_platform != 'all' and |
| dep.target_platform != system().lower()): |
| continue |
| local_path = os.path.join(ROOT_DIR, dep.target_folder) |
| if dep.source_url.endswith('.git'): |
| deps_updated |= CheckoutGitRepo(local_path, dep.source_url, dep.checksum, |
| args.check_only) |
| continue |
| is_zip = local_path.endswith('.zip') or local_path.endswith('.tgz') |
| zip_target_dir = local_path[:-4] if is_zip else None |
| zip_dir_stamp = os.path.join(zip_target_dir, '.stamp') if is_zip else None |
| |
| if ((not is_zip and HashLocalFile(local_path) == dep.checksum) or |
| (is_zip and ReadFile(zip_dir_stamp) == dep.checksum)): |
| continue |
| deps_updated = True |
| if args.check_only: |
| continue |
| MkdirRecursive(os.path.dirname(dep.target_folder)) |
| if HashLocalFile(local_path) != dep.checksum: |
| download_path = local_path + '.tmp' |
| logging.info('Downloading %s from %s', local_path, dep.source_url) |
| DownloadURL(dep.source_url, download_path) |
| os.chmod(download_path, 0o755) |
| actual_checksum = HashLocalFile(download_path) |
| if (actual_checksum != dep.checksum): |
| os.remove(download_path) |
| logging.fatal('SHA-256 mismatch for {} expected {} was {}'.format( |
| download_path, dep.checksum, actual_checksum)) |
| return 1 |
| os.rename(download_path, local_path) |
| if 'nodejs' in dep.target_folder: |
| nodejs_updated = True |
| |
| assert (HashLocalFile(local_path) == dep.checksum) |
| |
| if is_zip: |
| logging.info('Extracting %s into %s' % (local_path, zip_target_dir)) |
| assert (os.path.commonprefix((ROOT_DIR, zip_target_dir)) == ROOT_DIR) |
| if os.path.exists(zip_target_dir): |
| logging.info('Deleting stale dir %s' % zip_target_dir) |
| shutil.rmtree(zip_target_dir) |
| |
| # Decompress the archive. |
| if local_path.endswith('.tgz'): |
| MkdirRecursive(zip_target_dir) |
| subprocess.check_call(['tar', '-xf', local_path], cwd=zip_target_dir) |
| elif local_path.endswith('.zip'): |
| with zipfile.ZipFile(local_path, 'r') as zf: |
| for info in zf.infolist(): |
| ExtractZipfilePreservePermissions(zf, info, zip_target_dir) |
| |
| # If the zip contains one root folder, rebase one level up moving all |
| # its sub files and folders inside |target_dir|. |
| subdir = os.listdir(zip_target_dir) |
| if len(subdir) == 1: |
| subdir = os.path.join(zip_target_dir, subdir[0]) |
| if os.path.isdir(subdir): |
| for subf in os.listdir(subdir): |
| shutil.move(os.path.join(subdir, subf), zip_target_dir) |
| os.rmdir(subdir) |
| |
| # Create stamp and remove the archive. |
| with open(zip_dir_stamp, 'w') as stamp_file: |
| stamp_file.write(dep.checksum) |
| os.remove(local_path) |
| |
| if args.ui: |
| # Needs to happen after nodejs is installed above. |
| if args.check_only: |
| deps_updated = not CheckNodeModules() |
| else: |
| InstallNodeModules(force_clean=nodejs_updated) |
| |
| if args.check_only: |
| if not deps_updated: |
| with open(args.check_only, 'w') as f: |
| f.write('OK') # The content is irrelevant, just keep GN happy. |
| return 0 |
| argz = ' '.join([x for x in sys.argv[1:] if '--check-only' not in x]) |
| sys.stderr.write('\033[91mBuild deps are stale. ' + |
| 'Please run tools/install-build-deps %s\033[0m' % argz) |
| return 1 |
| |
| if deps_updated: |
| # Stale binary files may be compiled against old sysroot headers that aren't |
| # tracked by gn. |
| logging.warning('Remember to run "gn clean <output_directory>" ' + |
| 'to avoid stale binary files.') |
| |
| |
| if __name__ == '__main__': |
| logging.basicConfig(level=logging.INFO) |
| sys.exit(Main()) |