Primiano Tucci | ae2879e | 2017-09-27 11:02:09 +0900 | [diff] [blame] | 1 | #!/usr/bin/env python |
| 2 | # Copyright (C) 2017 The Android Open Source Project |
| 3 | # |
| 4 | # Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | # you may not use this file except in compliance with the License. |
| 6 | # You may obtain a copy of the License at |
| 7 | # |
| 8 | # http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | # |
| 10 | # Unless required by applicable law or agreed to in writing, software |
| 11 | # distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | # See the License for the specific language governing permissions and |
| 14 | # limitations under the License. |
| 15 | |
| 16 | import argparse |
| 17 | import hashlib |
| 18 | import logging |
| 19 | import os |
| 20 | import shutil |
Primiano Tucci | 0825bc8 | 2017-09-28 18:50:23 +0100 | [diff] [blame] | 21 | import subprocess |
Primiano Tucci | ae2879e | 2017-09-27 11:02:09 +0900 | [diff] [blame] | 22 | import sys |
| 23 | import urllib |
| 24 | import zipfile |
| 25 | |
Primiano Tucci | d775045 | 2017-09-29 14:38:51 +0100 | [diff] [blame] | 26 | from collections import namedtuple |
| 27 | |
Primiano Tucci | 636ede1 | 2017-10-30 16:03:07 +0000 | [diff] [blame] | 28 | # When adding a new git dependency here please also add a corresponding entry in |
| 29 | # .travis.yml under the "cache:" section. |
| 30 | |
Primiano Tucci | b60d4b0 | 2017-11-10 11:03:00 +0000 | [diff] [blame] | 31 | # The format for the deps below is the following: |
| 32 | # (target_folder, source_url, sha1, target_platform) |
| 33 | # |source_url| can be either a git repo or a http url. |
| 34 | # If a git repo, |sha1| is the committish that will be checked out. |
| 35 | # If a http url, |sha1| is the shasum of the original file. |
| 36 | # If the url is a .zip or .tgz file it will be automatically deflated under |
| 37 | # |target_folder|, taking care of stripping the root folder if it's a single |
| 38 | # root (to avoid ending up with buildtools/protobuf/protobuf-1.2.3/... and have |
| 39 | # instead just buildtools/protobuf). |
| 40 | # |target_platform| is either 'darwin', 'linux2' or 'all' and applies the dep |
| 41 | # only on the given platform (ask python why linux2 and not just linux). |
| 42 | |
Primiano Tucci | d775045 | 2017-09-29 14:38:51 +0100 | [diff] [blame] | 43 | # Dependencies required to build code on the host or when targeting desktop OS. |
| 44 | BUILD_DEPS_HOST = [ |
Primiano Tucci | ae2879e | 2017-09-27 11:02:09 +0900 | [diff] [blame] | 45 | # GN |
| 46 | ('buildtools/mac/gn', |
Primiano Tucci | c2eb510 | 2018-05-15 10:40:01 +0100 | [diff] [blame] | 47 | 'https://storage.googleapis.com/chromium-gn/9be792dd9010ce303a9c3a497a67bcc5ac8c7666', |
| 48 | '9be792dd9010ce303a9c3a497a67bcc5ac8c7666', |
Primiano Tucci | ae2879e | 2017-09-27 11:02:09 +0900 | [diff] [blame] | 49 | 'darwin' |
| 50 | ), |
| 51 | ('buildtools/linux64/gn', |
Primiano Tucci | c2eb510 | 2018-05-15 10:40:01 +0100 | [diff] [blame] | 52 | 'https://storage.googleapis.com/chromium-gn/2f27ff0b6118e5886df976da5effa6003d19d1ce', |
| 53 | '2f27ff0b6118e5886df976da5effa6003d19d1ce', |
Primiano Tucci | ae2879e | 2017-09-27 11:02:09 +0900 | [diff] [blame] | 54 | 'linux2' |
| 55 | ), |
| 56 | |
Primiano Tucci | 104bd6d | 2017-10-12 00:10:24 +0200 | [diff] [blame] | 57 | # clang-format |
| 58 | ('buildtools/mac/clang-format', |
| 59 | 'https://storage.googleapis.com/chromium-clang-format/0679b295e2ce2fce7919d1e8d003e497475f24a3', |
| 60 | '0679b295e2ce2fce7919d1e8d003e497475f24a3', |
| 61 | 'darwin' |
| 62 | ), |
| 63 | ('buildtools/linux64/clang-format', |
| 64 | 'https://storage.googleapis.com/chromium-clang-format/5349d1954e17f6ccafb6e6663b0f13cdb2bb33c8', |
| 65 | '5349d1954e17f6ccafb6e6663b0f13cdb2bb33c8', |
| 66 | 'linux2' |
| 67 | ), |
| 68 | # Keep the SHA1 in sync with |clang_format_rev| in chromium //buildtools/DEPS. |
| 69 | ('buildtools/clang_format/script', |
| 70 | 'https://chromium.googlesource.com/chromium/llvm-project/cfe/tools/clang-format.git', |
| 71 | '0653eee0c81ea04715c635dd0885e8096ff6ba6d', |
| 72 | 'all' |
| 73 | ), |
| 74 | |
Primiano Tucci | ae2879e | 2017-09-27 11:02:09 +0900 | [diff] [blame] | 75 | # Ninja |
| 76 | ('buildtools/mac/ninja', |
| 77 | 'https://storage.googleapis.com/fuchsia-build/fuchsia/ninja/mac/a1db595e824c50cf565fbf0af2437fd91b7babf4', |
| 78 | 'a1db595e824c50cf565fbf0af2437fd91b7babf4', |
| 79 | 'darwin' |
| 80 | ), |
| 81 | ('buildtools/linux64/ninja', |
| 82 | 'https://storage.googleapis.com/fuchsia-build/fuchsia/ninja/linux64/d35b36c84a09f7e38b25947cafada10e8bf835bc', |
| 83 | 'd35b36c84a09f7e38b25947cafada10e8bf835bc', |
| 84 | 'linux2' |
| 85 | ), |
| 86 | |
Primiano Tucci | d775045 | 2017-09-29 14:38:51 +0100 | [diff] [blame] | 87 | # Keep in sync with Android's //external/googletest/README.version. |
| 88 | ('buildtools/googletest.zip', |
| 89 | 'https://github.com/google/googletest/archive/ff07a5de0e81580547f1685e101194ed1a4fcd56.zip', |
| 90 | 'c7edec7d7e6db1fc37a20710de9c4d89e3a3893b', |
| 91 | 'all' |
| 92 | ), |
| 93 | |
| 94 | # Keep in sync with Android's //external/protobuf/README.version. |
| 95 | ('buildtools/protobuf.zip', |
| 96 | 'https://github.com/google/protobuf/releases/download/v3.0.0-beta-3/protobuf-cpp-3.0.0-beta-3.zip', |
| 97 | '3caec60aa9d8eefc8c3c3201b6b8ca19935edb89', |
| 98 | 'all' |
| 99 | ), |
| 100 | |
Primiano Tucci | b60d4b0 | 2017-11-10 11:03:00 +0000 | [diff] [blame] | 101 | # libc++, libc++abi and libunwind for Linux where we need to rebuild the C++ |
| 102 | # lib from sources. Keep the SHA1s in sync with Chrome's src/buildtools/DEPS. |
Primiano Tucci | d775045 | 2017-09-29 14:38:51 +0100 | [diff] [blame] | 103 | ('buildtools/libcxx', |
| 104 | 'https://chromium.googlesource.com/chromium/llvm-project/libcxx.git', |
Primiano Tucci | 5403e4f | 2018-11-27 10:07:03 +0000 | [diff] [blame] | 105 | '2199647acb904b91eea0a5e045f5b227c87d6e85', |
Primiano Tucci | d775045 | 2017-09-29 14:38:51 +0100 | [diff] [blame] | 106 | 'all' |
| 107 | ), |
| 108 | ('buildtools/libcxxabi', |
| 109 | 'https://chromium.googlesource.com/chromium/llvm-project/libcxxabi.git', |
Primiano Tucci | 5403e4f | 2018-11-27 10:07:03 +0000 | [diff] [blame] | 110 | 'c3f4753f7139c73063304235781e4f7788a94c06', |
Primiano Tucci | d775045 | 2017-09-29 14:38:51 +0100 | [diff] [blame] | 111 | 'all' |
| 112 | ), |
Primiano Tucci | 7278dea | 2017-10-31 11:50:32 +0000 | [diff] [blame] | 113 | ('buildtools/libunwind', |
| 114 | 'https://chromium.googlesource.com/external/llvm.org/libunwind.git', |
Primiano Tucci | 5403e4f | 2018-11-27 10:07:03 +0000 | [diff] [blame] | 115 | '94edd59b16d2084a62699290f9cfdf120d74eedf', |
Primiano Tucci | 7278dea | 2017-10-31 11:50:32 +0000 | [diff] [blame] | 116 | 'all' |
| 117 | ), |
Hector Dearman | 88a1011 | 2017-10-12 11:07:10 +0100 | [diff] [blame] | 118 | |
Primiano Tucci | b60d4b0 | 2017-11-10 11:03:00 +0000 | [diff] [blame] | 119 | # Keep the revision in sync with Chrome's CLANG_REVISION in |
| 120 | # tools/clang/scripts/update.py. |
| 121 | ('buildtools/clang.tgz', |
Primiano Tucci | 5403e4f | 2018-11-27 10:07:03 +0000 | [diff] [blame] | 122 | 'https://commondatastorage.googleapis.com/chromium-browser-clang/Linux_x64/clang-346388-1.tgz', |
| 123 | 'c2998d67a9c623fe12e01a33e8b7cf437b396099', |
Primiano Tucci | a65497e | 2018-09-26 19:53:58 +0100 | [diff] [blame] | 124 | 'linux2' |
| 125 | ), |
| 126 | |
| 127 | # Keep in sync with chromium DEPS. |
| 128 | ('buildtools/libfuzzer', |
| 129 | 'https://chromium.googlesource.com/chromium/llvm-project/compiler-rt/lib/fuzzer.git', |
Primiano Tucci | 5403e4f | 2018-11-27 10:07:03 +0000 | [diff] [blame] | 130 | '2a53098584c48af50aec3fb51febe5e651489774', |
Primiano Tucci | b60d4b0 | 2017-11-10 11:03:00 +0000 | [diff] [blame] | 131 | 'linux2' |
| 132 | ), |
| 133 | |
Hector Dearman | 88a1011 | 2017-10-12 11:07:10 +0100 | [diff] [blame] | 134 | # Benchmarking tool. |
| 135 | ('buildtools/benchmark.zip', |
Lalit Maganti | c99d93c | 2018-03-22 15:09:30 +0000 | [diff] [blame] | 136 | 'https://github.com/google/benchmark/archive/v1.3.0.zip', |
| 137 | 'f387e0df37d54bfd5be239e8d0d3ea2e2c3e34f4', |
Hector Dearman | 88a1011 | 2017-10-12 11:07:10 +0100 | [diff] [blame] | 138 | 'all' |
| 139 | ), |
Primiano Tucci | 38faa6f | 2018-04-01 20:12:08 +0200 | [diff] [blame] | 140 | |
| 141 | # Libbacktrace, for stacktraces in Linux/Android debug builds. |
| 142 | ('buildtools/libbacktrace.zip', |
| 143 | 'https://github.com/ianlancetaylor/libbacktrace/archive/177940370e4a6b2509e92a0aaa9749184e64af43.zip', |
| 144 | 'b723fe9d671d1ab54df1297f6afbf2893a41c3ea', |
| 145 | 'all' |
| 146 | ), |
Lalit Maganti | 6fe26e6 | 2018-05-23 12:14:38 +0100 | [diff] [blame] | 147 | |
| 148 | # Sqlite for the trace processing library. |
Ioannis Ilkos | 178535e | 2018-11-05 17:32:45 +0000 | [diff] [blame] | 149 | # This is the amalgamated source whose compiled output is meant to be faster. |
| 150 | # We still pull the full source for the extensions (not amalgamated). |
Lalit Maganti | 6fe26e6 | 2018-05-23 12:14:38 +0100 | [diff] [blame] | 151 | ('buildtools/sqlite.zip', |
Lalit Maganti | 10d6deb | 2018-11-20 15:15:50 +0000 | [diff] [blame] | 152 | 'https://storage.googleapis.com/perfetto/sqlite-amalgamation-3250300.zip', |
| 153 | 'b78c2cb0d2c9182686c582312479f96a82bf5380', |
Lalit Maganti | 6fe26e6 | 2018-05-23 12:14:38 +0100 | [diff] [blame] | 154 | 'all' |
| 155 | ), |
Ioannis Ilkos | 178535e | 2018-11-05 17:32:45 +0000 | [diff] [blame] | 156 | ('buildtools/sqlite_src.zip', |
Lalit Maganti | 10d6deb | 2018-11-20 15:15:50 +0000 | [diff] [blame] | 157 | 'https://storage.googleapis.com/perfetto/sqlite-src-3250300.zip', |
| 158 | 'd1af2883bb800852946f9bf8ab6055e7698e18ee', |
Ioannis Ilkos | 178535e | 2018-11-05 17:32:45 +0000 | [diff] [blame] | 159 | 'all' |
| 160 | ), |
Primiano Tucci | 0d72a31 | 2018-08-07 14:42:45 +0100 | [diff] [blame] | 161 | |
| 162 | # JsonCpp for legacy json import. Used only by the trace processor in |
| 163 | # standalone builds. |
| 164 | ('buildtools/jsoncpp.zip', |
| 165 | 'https://github.com/open-source-parsers/jsoncpp/archive/1.0.0.zip', |
| 166 | '3219e26f2e249bb46b7d688478208c7ec138fea4', |
| 167 | 'all' |
| 168 | ), |
Florian Mayer | 475bd7e | 2018-08-07 20:04:03 +0100 | [diff] [blame] | 169 | |
Florian Mayer | f833566 | 2018-08-08 11:30:32 +0100 | [diff] [blame] | 170 | # These dependencies are for libunwindstack, which is used by src/profiling. |
| 171 | ('buildtools/android-core', |
Florian Mayer | 475bd7e | 2018-08-07 20:04:03 +0100 | [diff] [blame] | 172 | 'https://android.googlesource.com/platform/system/core.git', |
Florian Mayer | 6140f32 | 2019-01-21 15:10:17 +0000 | [diff] [blame] | 173 | 'ad6a5c565c4908e3ae292f26510a2cc3c8bdf046', |
Florian Mayer | 475bd7e | 2018-08-07 20:04:03 +0100 | [diff] [blame] | 174 | 'all' |
| 175 | ), |
| 176 | |
| 177 | ('buildtools/lzma', |
| 178 | 'https://android.googlesource.com/platform/external/lzma.git', |
| 179 | '7851dce6f4ca17f5caa1c93a4e0a45686b1d56c3', |
| 180 | 'all' |
| 181 | ), |
Hector Dearman | 7f71d0e | 2018-08-09 11:26:10 +0100 | [diff] [blame] | 182 | |
Florian Mayer | b64d6b1 | 2018-08-30 10:46:30 -0700 | [diff] [blame] | 183 | ('buildtools/bionic', |
| 184 | 'https://android.googlesource.com/platform/bionic.git', |
Primiano Tucci | fb012ab | 2018-10-05 00:48:33 +0100 | [diff] [blame] | 185 | '4b7c5cca7fbd0330cdfef41c97f1401824e78fba', |
Florian Mayer | b64d6b1 | 2018-08-30 10:46:30 -0700 | [diff] [blame] | 186 | 'all' |
| 187 | ), |
| 188 | |
Hector Dearman | 7f71d0e | 2018-08-09 11:26:10 +0100 | [diff] [blame] | 189 | # Example traces for regression tests. |
| 190 | ('buildtools/test_data.zip', |
Lalit Maganti | a824c33 | 2019-01-23 17:55:58 +0000 | [diff] [blame] | 191 | 'https://storage.googleapis.com/perfetto/test-data-20190123.zip', |
Lalit Maganti | 41c9892 | 2019-01-23 19:35:58 +0000 | [diff] [blame] | 192 | '3761a0501669fc070022025678a8651f4a9495ad', |
Hector Dearman | 7f71d0e | 2018-08-09 11:26:10 +0100 | [diff] [blame] | 193 | 'all', |
| 194 | ), |
Hector Dearman | e44ad45 | 2018-09-21 11:51:57 +0100 | [diff] [blame] | 195 | |
Primiano Tucci | 21d41a7 | 2018-09-24 23:12:04 +0100 | [diff] [blame] | 196 | # Linenoise, used only by trace_processor in standalone builds. |
Hector Dearman | e44ad45 | 2018-09-21 11:51:57 +0100 | [diff] [blame] | 197 | ('buildtools/linenoise', |
Primiano Tucci | 21d41a7 | 2018-09-24 23:12:04 +0100 | [diff] [blame] | 198 | 'https://fuchsia.googlesource.com/third_party/linenoise.git', |
| 199 | 'c894b9e59f02203dbe4e2be657572cf88c4230c3', |
Hector Dearman | e44ad45 | 2018-09-21 11:51:57 +0100 | [diff] [blame] | 200 | 'all' |
| 201 | ), |
Primiano Tucci | d775045 | 2017-09-29 14:38:51 +0100 | [diff] [blame] | 202 | ] |
| 203 | |
| 204 | # Dependencies required to build Android code. |
| 205 | # URLs and SHA1s taken from: |
| 206 | # - https://dl.google.com/android/repository/repository-11.xml |
| 207 | # - https://dl.google.com/android/repository/sys-img/android/sys-img.xml |
| 208 | BUILD_DEPS_ANDROID = [ |
Primiano Tucci | ae2879e | 2017-09-27 11:02:09 +0900 | [diff] [blame] | 209 | # Android NDK |
| 210 | ('buildtools/ndk.zip', |
Florian Mayer | aa5316b | 2018-08-20 17:45:12 -0700 | [diff] [blame] | 211 | 'https://dl.google.com/android/repository/android-ndk-r17b-darwin-x86_64.zip', |
| 212 | 'f990aafaffec0b583d2c5420bfa622e52ac14248', |
Primiano Tucci | ae2879e | 2017-09-27 11:02:09 +0900 | [diff] [blame] | 213 | 'darwin' |
| 214 | ), |
| 215 | ('buildtools/ndk.zip', |
Florian Mayer | aa5316b | 2018-08-20 17:45:12 -0700 | [diff] [blame] | 216 | 'https://dl.google.com/android/repository/android-ndk-r17b-linux-x86_64.zip', |
| 217 | 'dd5762ee7ef4995ad04fe0c45a608c344d99ca9f', |
Primiano Tucci | ae2879e | 2017-09-27 11:02:09 +0900 | [diff] [blame] | 218 | 'linux2' |
| 219 | ), |
Primiano Tucci | d775045 | 2017-09-29 14:38:51 +0100 | [diff] [blame] | 220 | ] |
Primiano Tucci | ae2879e | 2017-09-27 11:02:09 +0900 | [diff] [blame] | 221 | |
Primiano Tucci | d775045 | 2017-09-29 14:38:51 +0100 | [diff] [blame] | 222 | # Dependencies required to run Android tests. |
| 223 | TEST_DEPS_ANDROID = [ |
Lalit Maganti | 367fcd5 | 2018-02-05 16:06:13 +0000 | [diff] [blame] | 224 | # Android emulator images. |
| 225 | ('buildtools/aosp-arm.zip', |
| 226 | 'https://storage.googleapis.com/perfetto/aosp-02022018-arm.zip', |
| 227 | 'a480d5e7d3ca888b0a58fe15ce76b1791537429a', |
Primiano Tucci | e7ca7c6 | 2018-04-07 08:28:03 +0200 | [diff] [blame] | 228 | 'all' |
Primiano Tucci | ae2879e | 2017-09-27 11:02:09 +0900 | [diff] [blame] | 229 | ), |
| 230 | |
Primiano Tucci | d775045 | 2017-09-29 14:38:51 +0100 | [diff] [blame] | 231 | # platform-tools.zip contains adb binaries. |
| 232 | ('buildtools/android_sdk/platform-tools.zip', |
| 233 | 'https://dl.google.com/android/repository/platform-tools_r26.0.0-darwin.zip', |
| 234 | 'e75b6137dc444f777eb02f44a6d9819b3aabff82', |
| 235 | 'darwin' |
| 236 | ), |
| 237 | ('buildtools/android_sdk/platform-tools.zip', |
| 238 | 'https://dl.google.com/android/repository/platform-tools_r26.0.0-linux.zip', |
| 239 | '00de8a6631405b617c10f68cd11ff2e1cd528e23', |
| 240 | 'linux2' |
Primiano Tucci | ae2879e | 2017-09-27 11:02:09 +0900 | [diff] [blame] | 241 | ), |
Primiano Tucci | 0825bc8 | 2017-09-28 18:50:23 +0100 | [diff] [blame] | 242 | |
Lalit Maganti | 367fcd5 | 2018-02-05 16:06:13 +0000 | [diff] [blame] | 243 | # Android emulator binaries. |
| 244 | ('buildtools/emulator', |
| 245 | 'https://android.googlesource.com/platform/prebuilts/android-emulator.git', |
| 246 | '4b260028dc27bc92c39bee9129cb2ba839970956', |
| 247 | 'all' |
Primiano Tucci | 0825bc8 | 2017-09-28 18:50:23 +0100 | [diff] [blame] | 248 | ), |
Primiano Tucci | d775045 | 2017-09-29 14:38:51 +0100 | [diff] [blame] | 249 | ] |
Primiano Tucci | ae2879e | 2017-09-27 11:02:09 +0900 | [diff] [blame] | 250 | |
Primiano Tucci | 1c752c1 | 2018-10-23 09:27:19 +0100 | [diff] [blame] | 251 | # This variable is updated by tools/roll-catapult-trace-viewer. |
| 252 | CATAPULT_SHA1 = 'ff5d8fd7244680b4d4456c25d5fdc04c76f9ef66' |
| 253 | |
Hector Dearman | 6177b75 | 2019-01-24 10:17:32 +0000 | [diff] [blame^] | 254 | TYPEFACES_SHA1 = '756b0a015b8f99f5718f7fdf967d052c1ec55ab3' |
| 255 | |
Primiano Tucci | 4bdc4c4 | 2018-05-10 15:52:33 +0100 | [diff] [blame] | 256 | UI_DEPS = [ |
| 257 | ('buildtools/nodejs.tgz', |
Primiano Tucci | dca9609 | 2018-05-31 11:47:16 +0100 | [diff] [blame] | 258 | 'https://storage.googleapis.com/perfetto/node-v10.3.0-darwin-x64.tar.gz', |
| 259 | '6d9a122785f38c256add3b25f74adf125497861a', |
Primiano Tucci | 4bdc4c4 | 2018-05-10 15:52:33 +0100 | [diff] [blame] | 260 | 'darwin' |
| 261 | ), |
| 262 | ('buildtools/nodejs.tgz', |
Primiano Tucci | dca9609 | 2018-05-31 11:47:16 +0100 | [diff] [blame] | 263 | 'https://storage.googleapis.com/perfetto/node-v10.3.0-linux-x64.tar.xz', |
| 264 | '118f6ea19f75089b3f12ac2ddfce357bff872b5e', |
Primiano Tucci | 4bdc4c4 | 2018-05-10 15:52:33 +0100 | [diff] [blame] | 265 | 'linux2' |
| 266 | ), |
| 267 | ('buildtools/emsdk/emscripten.tgz', |
| 268 | 'https://storage.googleapis.com/perfetto/emscripten-1.37.40.tar.gz', |
| 269 | '588c28221321ebbdfc8e3a6f47ea6106f589669b', |
| 270 | 'all' |
| 271 | ), |
| 272 | ('buildtools/emsdk/llvm.tgz', |
| 273 | 'https://storage.googleapis.com/perfetto/emscripten-llvm-e1.37.40-darwin.tar.gz', |
| 274 | '7a894ef0a52821c62f6abaac552dc4ce5d424607', |
| 275 | 'darwin' |
| 276 | ), |
| 277 | ('buildtools/emsdk/llvm.tgz', |
Primiano Tucci | fe19990 | 2018-06-04 12:33:04 +0200 | [diff] [blame] | 278 | 'https://storage.googleapis.com/perfetto/emscripten-llvm-e1.37.40-static-linux.tar.gz', |
| 279 | '478501b9b7a14884e546c84efe209a90052cbb07', |
Primiano Tucci | 4bdc4c4 | 2018-05-10 15:52:33 +0100 | [diff] [blame] | 280 | 'linux2' |
| 281 | ), |
Hector Dearman | d9628d3 | 2018-10-17 14:38:26 +0100 | [diff] [blame] | 282 | ('buildtools/d8.tgz', |
| 283 | 'https://storage.googleapis.com/perfetto/d8-linux2-5.7.492.65.tar.gz', |
| 284 | '95e82ad7faf0a6f74d950c2aa65e3858b7bdb6c6', |
| 285 | 'linux2' |
| 286 | ), |
| 287 | ('buildtools/d8.tgz', |
| 288 | 'https://storage.googleapis.com/perfetto/d8-darwin-6.6.346.32.tar.gz', |
| 289 | '1abd630619bb1977ab62095570a113d782a1545d', |
| 290 | 'darwin' |
| 291 | ), |
Primiano Tucci | 1c752c1 | 2018-10-23 09:27:19 +0100 | [diff] [blame] | 292 | ('buildtools/catapult_trace_viewer.tgz', |
| 293 | 'https://storage.googleapis.com/perfetto/catapult_trace_viewer-%s.tar.gz' % CATAPULT_SHA1, |
| 294 | CATAPULT_SHA1, |
| 295 | 'all' |
| 296 | ), |
Hector Dearman | 6177b75 | 2019-01-24 10:17:32 +0000 | [diff] [blame^] | 297 | ('buildtools/typefaces.tgz', |
| 298 | 'https://storage.googleapis.com/perfetto/typefaces-%s.tar.gz' % TYPEFACES_SHA1, |
| 299 | TYPEFACES_SHA1, |
| 300 | 'all' |
| 301 | ) |
Primiano Tucci | 4bdc4c4 | 2018-05-10 15:52:33 +0100 | [diff] [blame] | 302 | ] |
| 303 | |
Primiano Tucci | ae2879e | 2017-09-27 11:02:09 +0900 | [diff] [blame] | 304 | ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) |
| 305 | |
| 306 | |
| 307 | def ReadFile(path): |
| 308 | if not os.path.exists(path): |
| 309 | return None |
| 310 | with open(path) as f: |
| 311 | return f.read().strip() |
| 312 | |
| 313 | |
Primiano Tucci | 0825bc8 | 2017-09-28 18:50:23 +0100 | [diff] [blame] | 314 | def MkdirRecursive(path): |
| 315 | # Works with both relative and absolute paths |
| 316 | cwd = '/' if path.startswith('/') else ROOT_DIR |
| 317 | for part in path.split('/'): |
Primiano Tucci | ae2879e | 2017-09-27 11:02:09 +0900 | [diff] [blame] | 318 | cwd = os.path.join(cwd, part) |
| 319 | if not os.path.exists(cwd): |
| 320 | os.makedirs(cwd) |
| 321 | else: |
| 322 | assert(os.path.isdir(cwd)) |
| 323 | |
| 324 | |
| 325 | def HashLocalFile(path): |
| 326 | if not os.path.exists(path): |
| 327 | return None |
| 328 | with open(path, 'rb') as f: |
| 329 | return hashlib.sha1(f.read()).hexdigest() |
| 330 | |
| 331 | |
| 332 | def ExtractZipfilePreservePermissions(zf, info, path): |
| 333 | zf.extract(info.filename, path=path) |
| 334 | target_path = os.path.join(path, info.filename) |
| 335 | min_acls = 0o755 if info.filename.endswith('/') else 0o644 |
| 336 | os.chmod(target_path, (info.external_attr >> 16L) | min_acls) |
| 337 | |
| 338 | |
Primiano Tucci | 0825bc8 | 2017-09-28 18:50:23 +0100 | [diff] [blame] | 339 | def IsGitRepoCheckoutOutAtRevision(path, revision): |
| 340 | return ReadFile(os.path.join(path, '.git', 'HEAD')) == revision |
| 341 | |
| 342 | |
| 343 | def CheckoutGitRepo(path, git_url, revision): |
| 344 | if IsGitRepoCheckoutOutAtRevision(path, revision): |
| 345 | return |
| 346 | if os.path.exists(path): |
| 347 | shutil.rmtree(path) |
| 348 | MkdirRecursive(path) |
| 349 | logging.info('Fetching %s @ %s into %s', git_url, revision, path) |
Lalit Maganti | 367fcd5 | 2018-02-05 16:06:13 +0000 | [diff] [blame] | 350 | subprocess.check_call(['git', 'init', path], cwd=path) |
| 351 | subprocess.check_call( |
Ryan Savitski | e957ce8 | 2018-11-06 14:53:33 +0000 | [diff] [blame] | 352 | ['git', 'fetch', '--quiet', '--depth', '1', git_url, revision], cwd=path) |
Primiano Tucci | 0825bc8 | 2017-09-28 18:50:23 +0100 | [diff] [blame] | 353 | subprocess.check_call(['git', 'checkout', revision, '--quiet'], cwd=path) |
| 354 | assert(IsGitRepoCheckoutOutAtRevision(path, revision)) |
| 355 | |
Deepanjan Roy | acf8c07 | 2018-07-13 11:37:04 -0400 | [diff] [blame] | 356 | def InstallNodeModules(): |
| 357 | ui_dir = os.path.join(ROOT_DIR, 'ui') |
| 358 | logging.info("Running npm install in {0}".format(ui_dir)) |
| 359 | subprocess.check_call( |
| 360 | [os.path.join(ui_dir, 'npm'), 'install', '--no-save'], |
| 361 | cwd=os.path.join(ROOT_DIR, 'ui')) |
Primiano Tucci | 0825bc8 | 2017-09-28 18:50:23 +0100 | [diff] [blame] | 362 | |
Primiano Tucci | ae2879e | 2017-09-27 11:02:09 +0900 | [diff] [blame] | 363 | def Main(): |
| 364 | parser = argparse.ArgumentParser() |
Primiano Tucci | d775045 | 2017-09-29 14:38:51 +0100 | [diff] [blame] | 365 | parser.add_argument('--no-android', action='store_true') |
Primiano Tucci | 4bdc4c4 | 2018-05-10 15:52:33 +0100 | [diff] [blame] | 366 | parser.add_argument('--ui', action='store_true') |
Primiano Tucci | ae2879e | 2017-09-27 11:02:09 +0900 | [diff] [blame] | 367 | args = parser.parse_args() |
Primiano Tucci | d775045 | 2017-09-29 14:38:51 +0100 | [diff] [blame] | 368 | deps = BUILD_DEPS_HOST |
| 369 | if not args.no_android: |
| 370 | deps += BUILD_DEPS_ANDROID + TEST_DEPS_ANDROID |
Primiano Tucci | 4bdc4c4 | 2018-05-10 15:52:33 +0100 | [diff] [blame] | 371 | if args.ui: |
| 372 | deps += UI_DEPS |
Eric Seckler | 6e828f3 | 2019-01-02 11:10:56 +0000 | [diff] [blame] | 373 | deps_updated = False |
Primiano Tucci | d775045 | 2017-09-29 14:38:51 +0100 | [diff] [blame] | 374 | for rel_path, url, expected_sha1, platform in deps: |
Lalit Maganti | 367fcd5 | 2018-02-05 16:06:13 +0000 | [diff] [blame] | 375 | if (platform != 'all' and platform != sys.platform): |
Primiano Tucci | ae2879e | 2017-09-27 11:02:09 +0900 | [diff] [blame] | 376 | continue |
Primiano Tucci | ae2879e | 2017-09-27 11:02:09 +0900 | [diff] [blame] | 377 | local_path = os.path.join(ROOT_DIR, rel_path) |
Primiano Tucci | 0825bc8 | 2017-09-28 18:50:23 +0100 | [diff] [blame] | 378 | if url.endswith('.git'): |
| 379 | CheckoutGitRepo(local_path, url, expected_sha1) |
| 380 | continue |
Primiano Tucci | b60d4b0 | 2017-11-10 11:03:00 +0000 | [diff] [blame] | 381 | is_zip = local_path.endswith('.zip') or local_path.endswith('.tgz') |
Primiano Tucci | ae2879e | 2017-09-27 11:02:09 +0900 | [diff] [blame] | 382 | zip_target_dir = local_path[:-4] if is_zip else None |
| 383 | zip_dir_stamp = os.path.join(zip_target_dir, '.stamp') if is_zip else None |
| 384 | |
| 385 | if ((not is_zip and HashLocalFile(local_path) == expected_sha1) or |
| 386 | (is_zip and ReadFile(zip_dir_stamp) == expected_sha1)): |
| 387 | continue |
Eric Seckler | 6e828f3 | 2019-01-02 11:10:56 +0000 | [diff] [blame] | 388 | deps_updated = True |
Primiano Tucci | ae2879e | 2017-09-27 11:02:09 +0900 | [diff] [blame] | 389 | MkdirRecursive(os.path.dirname(rel_path)) |
| 390 | if HashLocalFile(local_path) != expected_sha1: |
| 391 | download_path = local_path + '.tmp' |
| 392 | logging.info('Downloading %s from %s', local_path, url) |
| 393 | urllib.urlretrieve(url, download_path) |
| 394 | os.chmod(download_path, 0o755) |
Hector Dearman | d9628d3 | 2018-10-17 14:38:26 +0100 | [diff] [blame] | 395 | actual_sha1 = HashLocalFile(download_path) |
| 396 | if (actual_sha1 != expected_sha1): |
Primiano Tucci | ae2879e | 2017-09-27 11:02:09 +0900 | [diff] [blame] | 397 | os.remove(download_path) |
Hector Dearman | d9628d3 | 2018-10-17 14:38:26 +0100 | [diff] [blame] | 398 | logging.fatal('SHA1 mismatch for {} expected {} was {}'.format( |
| 399 | download_path, expected_sha1, actual_sha1)) |
Primiano Tucci | ae2879e | 2017-09-27 11:02:09 +0900 | [diff] [blame] | 400 | return 1 |
| 401 | os.rename(download_path, local_path) |
| 402 | assert(HashLocalFile(local_path) == expected_sha1) |
| 403 | |
| 404 | if is_zip: |
| 405 | logging.info('Extracting %s into %s' % (local_path, zip_target_dir)) |
| 406 | assert(os.path.commonprefix((ROOT_DIR, zip_target_dir)) == ROOT_DIR) |
| 407 | if os.path.exists(zip_target_dir): |
| 408 | logging.info('Deleting stale dir %s' % zip_target_dir) |
| 409 | shutil.rmtree(zip_target_dir) |
Primiano Tucci | ae2879e | 2017-09-27 11:02:09 +0900 | [diff] [blame] | 410 | |
Primiano Tucci | b60d4b0 | 2017-11-10 11:03:00 +0000 | [diff] [blame] | 411 | # Decompress the archive. |
| 412 | if local_path.endswith('.tgz'): |
| 413 | MkdirRecursive(zip_target_dir) |
Primiano Tucci | 4bdc4c4 | 2018-05-10 15:52:33 +0100 | [diff] [blame] | 414 | subprocess.check_call(['tar', '-xf', local_path], cwd=zip_target_dir) |
Primiano Tucci | b60d4b0 | 2017-11-10 11:03:00 +0000 | [diff] [blame] | 415 | elif local_path.endswith('.zip'): |
| 416 | with zipfile.ZipFile(local_path, 'r') as zf: |
| 417 | for info in zf.infolist(): |
| 418 | ExtractZipfilePreservePermissions(zf, info, zip_target_dir) |
| 419 | |
| 420 | # If the zip contains one root folder, rebase one level up moving all |
| 421 | # its sub files and folders inside |target_dir|. |
| 422 | subdir = os.listdir(zip_target_dir) |
| 423 | if len(subdir) == 1: |
| 424 | subdir = os.path.join(zip_target_dir, subdir[0]) |
| 425 | if os.path.isdir(subdir): |
| 426 | for subf in os.listdir(subdir): |
| 427 | shutil.move(os.path.join(subdir,subf), zip_target_dir) |
| 428 | os.rmdir(subdir) |
| 429 | |
| 430 | # Create stamp and remove the archive. |
| 431 | with open(zip_dir_stamp, 'w') as stamp_file: |
| 432 | stamp_file.write(expected_sha1) |
| 433 | os.remove(local_path) |
Primiano Tucci | ae2879e | 2017-09-27 11:02:09 +0900 | [diff] [blame] | 434 | |
Deepanjan Roy | acf8c07 | 2018-07-13 11:37:04 -0400 | [diff] [blame] | 435 | if args.ui: |
| 436 | # Needs to happen after nodejs is installed above. |
| 437 | InstallNodeModules() |
Primiano Tucci | ae2879e | 2017-09-27 11:02:09 +0900 | [diff] [blame] | 438 | |
Eric Seckler | 6e828f3 | 2019-01-02 11:10:56 +0000 | [diff] [blame] | 439 | if deps_updated: |
| 440 | # Stale binary files may be compiled against old sysroot headers that aren't |
| 441 | # tracked by gn. |
| 442 | logging.warn('Remember to run "gn clean <output_directory>" ' + |
| 443 | 'to avoid stale binary files.') |
| 444 | |
Primiano Tucci | ae2879e | 2017-09-27 11:02:09 +0900 | [diff] [blame] | 445 | if __name__ == '__main__': |
| 446 | logging.basicConfig(level=logging.INFO) |
| 447 | sys.exit(Main()) |