Primiano Tucci | ae2879e | 2017-09-27 11:02:09 +0900 | [diff] [blame] | 1 | #!/usr/bin/env python |
| 2 | # Copyright (C) 2017 The Android Open Source Project |
| 3 | # |
| 4 | # Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | # you may not use this file except in compliance with the License. |
| 6 | # You may obtain a copy of the License at |
| 7 | # |
| 8 | # http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | # |
| 10 | # Unless required by applicable law or agreed to in writing, software |
| 11 | # distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | # See the License for the specific language governing permissions and |
| 14 | # limitations under the License. |
| 15 | |
| 16 | import argparse |
| 17 | import hashlib |
| 18 | import logging |
| 19 | import os |
| 20 | import shutil |
Primiano Tucci | 0825bc8 | 2017-09-28 18:50:23 +0100 | [diff] [blame] | 21 | import subprocess |
Primiano Tucci | ae2879e | 2017-09-27 11:02:09 +0900 | [diff] [blame] | 22 | import sys |
| 23 | import urllib |
| 24 | import zipfile |
| 25 | |
Primiano Tucci | d775045 | 2017-09-29 14:38:51 +0100 | [diff] [blame] | 26 | from collections import namedtuple |
| 27 | |
| 28 | # Dependencies required to build code on the host or when targeting desktop OS. |
| 29 | BUILD_DEPS_HOST = [ |
Primiano Tucci | ae2879e | 2017-09-27 11:02:09 +0900 | [diff] [blame] | 30 | # GN |
| 31 | ('buildtools/mac/gn', |
| 32 | 'https://storage.googleapis.com/chromium-gn/c2c934d4dda1f470a6511b1015dda9a9fb1ce50b', |
| 33 | 'c2c934d4dda1f470a6511b1015dda9a9fb1ce50b', |
| 34 | 'darwin' |
| 35 | ), |
| 36 | ('buildtools/linux64/gn', |
| 37 | 'https://storage.googleapis.com/chromium-gn/b53fa13e950948c6f9a062189b76b34a9610281f', |
| 38 | 'b53fa13e950948c6f9a062189b76b34a9610281f', |
| 39 | 'linux2' |
| 40 | ), |
| 41 | |
Primiano Tucci | 104bd6d | 2017-10-12 00:10:24 +0200 | [diff] [blame] | 42 | # clang-format |
| 43 | ('buildtools/mac/clang-format', |
| 44 | 'https://storage.googleapis.com/chromium-clang-format/0679b295e2ce2fce7919d1e8d003e497475f24a3', |
| 45 | '0679b295e2ce2fce7919d1e8d003e497475f24a3', |
| 46 | 'darwin' |
| 47 | ), |
| 48 | ('buildtools/linux64/clang-format', |
| 49 | 'https://storage.googleapis.com/chromium-clang-format/5349d1954e17f6ccafb6e6663b0f13cdb2bb33c8', |
| 50 | '5349d1954e17f6ccafb6e6663b0f13cdb2bb33c8', |
| 51 | 'linux2' |
| 52 | ), |
| 53 | # Keep the SHA1 in sync with |clang_format_rev| in chromium //buildtools/DEPS. |
| 54 | ('buildtools/clang_format/script', |
| 55 | 'https://chromium.googlesource.com/chromium/llvm-project/cfe/tools/clang-format.git', |
| 56 | '0653eee0c81ea04715c635dd0885e8096ff6ba6d', |
| 57 | 'all' |
| 58 | ), |
| 59 | |
Primiano Tucci | ae2879e | 2017-09-27 11:02:09 +0900 | [diff] [blame] | 60 | # Ninja |
| 61 | ('buildtools/mac/ninja', |
| 62 | 'https://storage.googleapis.com/fuchsia-build/fuchsia/ninja/mac/a1db595e824c50cf565fbf0af2437fd91b7babf4', |
| 63 | 'a1db595e824c50cf565fbf0af2437fd91b7babf4', |
| 64 | 'darwin' |
| 65 | ), |
| 66 | ('buildtools/linux64/ninja', |
| 67 | 'https://storage.googleapis.com/fuchsia-build/fuchsia/ninja/linux64/d35b36c84a09f7e38b25947cafada10e8bf835bc', |
| 68 | 'd35b36c84a09f7e38b25947cafada10e8bf835bc', |
| 69 | 'linux2' |
| 70 | ), |
| 71 | |
Primiano Tucci | d775045 | 2017-09-29 14:38:51 +0100 | [diff] [blame] | 72 | # Keep in sync with Android's //external/googletest/README.version. |
| 73 | ('buildtools/googletest.zip', |
| 74 | 'https://github.com/google/googletest/archive/ff07a5de0e81580547f1685e101194ed1a4fcd56.zip', |
| 75 | 'c7edec7d7e6db1fc37a20710de9c4d89e3a3893b', |
| 76 | 'all' |
| 77 | ), |
| 78 | |
| 79 | # Keep in sync with Android's //external/protobuf/README.version. |
| 80 | ('buildtools/protobuf.zip', |
| 81 | 'https://github.com/google/protobuf/releases/download/v3.0.0-beta-3/protobuf-cpp-3.0.0-beta-3.zip', |
| 82 | '3caec60aa9d8eefc8c3c3201b6b8ca19935edb89', |
| 83 | 'all' |
| 84 | ), |
| 85 | |
| 86 | # libc++ and libc++abi, for clang msan that require rebuilding the C++ lib |
| 87 | # from sources. Keep the SHA1s in sync with Chrome's src/buildtools/DEPS. |
| 88 | ('buildtools/libcxx', |
| 89 | 'https://chromium.googlesource.com/chromium/llvm-project/libcxx.git', |
| 90 | '3a07dd740be63878167a0ea19fe81869954badd7', |
| 91 | 'all' |
| 92 | ), |
| 93 | ('buildtools/libcxxabi', |
| 94 | 'https://chromium.googlesource.com/chromium/llvm-project/libcxxabi.git', |
| 95 | '4072e8fd76febee37f60aeda76d6d9f5e3791daa', |
| 96 | 'all' |
| 97 | ), |
Hector Dearman | 88a1011 | 2017-10-12 11:07:10 +0100 | [diff] [blame^] | 98 | |
| 99 | # Benchmarking tool. |
| 100 | ('buildtools/benchmark.zip', |
| 101 | 'https://github.com/google/benchmark/archive/v1.2.0.zip', |
| 102 | '553b6bfa59c63f15c54cfd8c5122b0f7f84469af', |
| 103 | 'all' |
| 104 | ), |
Primiano Tucci | d775045 | 2017-09-29 14:38:51 +0100 | [diff] [blame] | 105 | ] |
| 106 | |
| 107 | # Dependencies required to build Android code. |
| 108 | # URLs and SHA1s taken from: |
| 109 | # - https://dl.google.com/android/repository/repository-11.xml |
| 110 | # - https://dl.google.com/android/repository/sys-img/android/sys-img.xml |
| 111 | BUILD_DEPS_ANDROID = [ |
Primiano Tucci | ae2879e | 2017-09-27 11:02:09 +0900 | [diff] [blame] | 112 | # Android NDK |
| 113 | ('buildtools/ndk.zip', |
| 114 | 'https://dl.google.com/android/repository/android-ndk-r15c-darwin-x86_64.zip', |
| 115 | 'ea4b5d76475db84745aa8828000d009625fc1f98', |
| 116 | 'darwin' |
| 117 | ), |
| 118 | ('buildtools/ndk.zip', |
| 119 | 'https://dl.google.com/android/repository/android-ndk-r15c-linux-x86_64.zip', |
| 120 | '0bf02d4e8b85fd770fd7b9b2cdec57f9441f27a2', |
| 121 | 'linux2' |
| 122 | ), |
Primiano Tucci | d775045 | 2017-09-29 14:38:51 +0100 | [diff] [blame] | 123 | ] |
Primiano Tucci | ae2879e | 2017-09-27 11:02:09 +0900 | [diff] [blame] | 124 | |
Primiano Tucci | d775045 | 2017-09-29 14:38:51 +0100 | [diff] [blame] | 125 | # Dependencies required to run Android tests. |
| 126 | TEST_DEPS_ANDROID = [ |
| 127 | # tools.zip contains the emulator binaries. |
| 128 | ('buildtools/android_sdk/tools.zip', |
| 129 | 'https://dl.google.com/android/repository/tools_r25.2.5-macosx.zip', |
| 130 | 'd2168d963ac5b616e3d3ddaf21511d084baf3659', |
| 131 | 'darwin' |
| 132 | ), |
| 133 | ('buildtools/android_sdk/tools.zip', |
| 134 | 'https://dl.google.com/android/repository/tools_r25.2.5-linux.zip', |
| 135 | '72df3aa1988c0a9003ccdfd7a13a7b8bd0f47fc1', |
| 136 | 'linux2' |
Primiano Tucci | ae2879e | 2017-09-27 11:02:09 +0900 | [diff] [blame] | 137 | ), |
| 138 | |
Primiano Tucci | d775045 | 2017-09-29 14:38:51 +0100 | [diff] [blame] | 139 | # platform-tools.zip contains adb binaries. |
| 140 | ('buildtools/android_sdk/platform-tools.zip', |
| 141 | 'https://dl.google.com/android/repository/platform-tools_r26.0.0-darwin.zip', |
| 142 | 'e75b6137dc444f777eb02f44a6d9819b3aabff82', |
| 143 | 'darwin' |
| 144 | ), |
| 145 | ('buildtools/android_sdk/platform-tools.zip', |
| 146 | 'https://dl.google.com/android/repository/platform-tools_r26.0.0-linux.zip', |
| 147 | '00de8a6631405b617c10f68cd11ff2e1cd528e23', |
| 148 | 'linux2' |
Primiano Tucci | ae2879e | 2017-09-27 11:02:09 +0900 | [diff] [blame] | 149 | ), |
Primiano Tucci | 0825bc8 | 2017-09-28 18:50:23 +0100 | [diff] [blame] | 150 | |
Primiano Tucci | d775045 | 2017-09-29 14:38:51 +0100 | [diff] [blame] | 151 | # Android emulator images. |
| 152 | ('buildtools/android_sdk/system-images/android-24/default/armeabi-v7a.zip', |
| 153 | 'https://dl.google.com/android/repository/sys-img/android/armeabi-v7a-24_r07.zip', |
| 154 | '3454546b4eed2d6c3dd06d47757d6da9f4176033', |
Primiano Tucci | 0825bc8 | 2017-09-28 18:50:23 +0100 | [diff] [blame] | 155 | 'all' |
| 156 | ), |
Primiano Tucci | d775045 | 2017-09-29 14:38:51 +0100 | [diff] [blame] | 157 | ('buildtools/android_sdk/system-images/android-24/default/arm64-v8a.zip', |
| 158 | 'https://dl.google.com/android/repository/sys-img/android/arm64-v8a-24_r07.zip', |
| 159 | 'e8ab2e49e4efe4b064232b33b5eeaded61437d7f', |
Primiano Tucci | 0825bc8 | 2017-09-28 18:50:23 +0100 | [diff] [blame] | 160 | 'all' |
| 161 | ), |
Primiano Tucci | d775045 | 2017-09-29 14:38:51 +0100 | [diff] [blame] | 162 | ] |
Primiano Tucci | ae2879e | 2017-09-27 11:02:09 +0900 | [diff] [blame] | 163 | |
| 164 | ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) |
| 165 | |
| 166 | |
| 167 | def ReadFile(path): |
| 168 | if not os.path.exists(path): |
| 169 | return None |
| 170 | with open(path) as f: |
| 171 | return f.read().strip() |
| 172 | |
| 173 | |
Primiano Tucci | 0825bc8 | 2017-09-28 18:50:23 +0100 | [diff] [blame] | 174 | def MkdirRecursive(path): |
| 175 | # Works with both relative and absolute paths |
| 176 | cwd = '/' if path.startswith('/') else ROOT_DIR |
| 177 | for part in path.split('/'): |
Primiano Tucci | ae2879e | 2017-09-27 11:02:09 +0900 | [diff] [blame] | 178 | cwd = os.path.join(cwd, part) |
| 179 | if not os.path.exists(cwd): |
| 180 | os.makedirs(cwd) |
| 181 | else: |
| 182 | assert(os.path.isdir(cwd)) |
| 183 | |
| 184 | |
| 185 | def HashLocalFile(path): |
| 186 | if not os.path.exists(path): |
| 187 | return None |
| 188 | with open(path, 'rb') as f: |
| 189 | return hashlib.sha1(f.read()).hexdigest() |
| 190 | |
| 191 | |
| 192 | def ExtractZipfilePreservePermissions(zf, info, path): |
| 193 | zf.extract(info.filename, path=path) |
| 194 | target_path = os.path.join(path, info.filename) |
| 195 | min_acls = 0o755 if info.filename.endswith('/') else 0o644 |
| 196 | os.chmod(target_path, (info.external_attr >> 16L) | min_acls) |
| 197 | |
| 198 | |
Primiano Tucci | 0825bc8 | 2017-09-28 18:50:23 +0100 | [diff] [blame] | 199 | def IsGitRepoCheckoutOutAtRevision(path, revision): |
| 200 | return ReadFile(os.path.join(path, '.git', 'HEAD')) == revision |
| 201 | |
| 202 | |
| 203 | def CheckoutGitRepo(path, git_url, revision): |
| 204 | if IsGitRepoCheckoutOutAtRevision(path, revision): |
| 205 | return |
| 206 | if os.path.exists(path): |
| 207 | shutil.rmtree(path) |
| 208 | MkdirRecursive(path) |
| 209 | logging.info('Fetching %s @ %s into %s', git_url, revision, path) |
| 210 | subprocess.check_call(['git', 'clone', git_url, path], cwd=path) |
| 211 | subprocess.check_call(['git', 'checkout', revision, '--quiet'], cwd=path) |
| 212 | assert(IsGitRepoCheckoutOutAtRevision(path, revision)) |
| 213 | |
| 214 | |
Primiano Tucci | ae2879e | 2017-09-27 11:02:09 +0900 | [diff] [blame] | 215 | def Main(): |
| 216 | parser = argparse.ArgumentParser() |
Primiano Tucci | d775045 | 2017-09-29 14:38:51 +0100 | [diff] [blame] | 217 | parser.add_argument('--no-android', action='store_true') |
Primiano Tucci | ae2879e | 2017-09-27 11:02:09 +0900 | [diff] [blame] | 218 | args = parser.parse_args() |
Primiano Tucci | d775045 | 2017-09-29 14:38:51 +0100 | [diff] [blame] | 219 | deps = BUILD_DEPS_HOST |
| 220 | if not args.no_android: |
| 221 | deps += BUILD_DEPS_ANDROID + TEST_DEPS_ANDROID |
| 222 | for rel_path, url, expected_sha1, platform in deps: |
Primiano Tucci | ae2879e | 2017-09-27 11:02:09 +0900 | [diff] [blame] | 223 | if platform != 'all' and platform != sys.platform: |
| 224 | continue |
Primiano Tucci | ae2879e | 2017-09-27 11:02:09 +0900 | [diff] [blame] | 225 | local_path = os.path.join(ROOT_DIR, rel_path) |
Primiano Tucci | 0825bc8 | 2017-09-28 18:50:23 +0100 | [diff] [blame] | 226 | if url.endswith('.git'): |
| 227 | CheckoutGitRepo(local_path, url, expected_sha1) |
| 228 | continue |
Primiano Tucci | ae2879e | 2017-09-27 11:02:09 +0900 | [diff] [blame] | 229 | is_zip = local_path.lower().endswith('.zip') |
| 230 | zip_target_dir = local_path[:-4] if is_zip else None |
| 231 | zip_dir_stamp = os.path.join(zip_target_dir, '.stamp') if is_zip else None |
| 232 | |
| 233 | if ((not is_zip and HashLocalFile(local_path) == expected_sha1) or |
| 234 | (is_zip and ReadFile(zip_dir_stamp) == expected_sha1)): |
| 235 | continue |
| 236 | MkdirRecursive(os.path.dirname(rel_path)) |
| 237 | if HashLocalFile(local_path) != expected_sha1: |
| 238 | download_path = local_path + '.tmp' |
| 239 | logging.info('Downloading %s from %s', local_path, url) |
| 240 | urllib.urlretrieve(url, download_path) |
| 241 | os.chmod(download_path, 0o755) |
| 242 | if (HashLocalFile(download_path) != expected_sha1): |
| 243 | os.remove(download_path) |
| 244 | logging.fatal('SHA1 mismatch for %s', download_path) |
| 245 | return 1 |
| 246 | os.rename(download_path, local_path) |
| 247 | assert(HashLocalFile(local_path) == expected_sha1) |
| 248 | |
| 249 | if is_zip: |
| 250 | logging.info('Extracting %s into %s' % (local_path, zip_target_dir)) |
| 251 | assert(os.path.commonprefix((ROOT_DIR, zip_target_dir)) == ROOT_DIR) |
| 252 | if os.path.exists(zip_target_dir): |
| 253 | logging.info('Deleting stale dir %s' % zip_target_dir) |
| 254 | shutil.rmtree(zip_target_dir) |
| 255 | with zipfile.ZipFile(local_path, 'r') as zf: |
| 256 | for info in zf.infolist(): |
| 257 | ExtractZipfilePreservePermissions(zf, info, zip_target_dir) |
| 258 | |
| 259 | # If the zip contains one root folder, rebase one level up moving all |
| 260 | # its sub files and folders inside |target_dir|. |
| 261 | subdir = os.listdir(zip_target_dir) |
| 262 | if len(subdir) == 1: |
| 263 | subdir = os.path.join(zip_target_dir, subdir[0]) |
| 264 | if os.path.isdir(subdir): |
| 265 | for subf in os.listdir(subdir): |
| 266 | shutil.move(os.path.join(subdir,subf), zip_target_dir) |
| 267 | os.rmdir(subdir) |
| 268 | with open(zip_dir_stamp, 'w') as stamp_file: |
| 269 | stamp_file.write(expected_sha1) |
| 270 | os.remove(local_path) |
| 271 | |
| 272 | |
| 273 | if __name__ == '__main__': |
| 274 | logging.basicConfig(level=logging.INFO) |
| 275 | sys.exit(Main()) |