Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 1 | #!/usr/bin/env python |
| 2 | # Copyright 2014 The Chromium Authors. All rights reserved. |
| 3 | # Use of this source code is governed by a BSD-style license that can be |
| 4 | # found in the LICENSE file. |
| 5 | |
| 6 | # Modified from go/bootstrap.py in Chromium infrastructure's repository to patch |
| 7 | # out everything but the core toolchain. |
| 8 | # |
| 9 | # https://chromium.googlesource.com/infra/infra/ |
| 10 | |
| 11 | """Prepares a local hermetic Go installation. |
| 12 | |
| 13 | - Downloads and unpacks the Go toolset in ../golang. |
| 14 | """ |
| 15 | |
| 16 | import contextlib |
| 17 | import logging |
| 18 | import os |
| 19 | import platform |
| 20 | import shutil |
| 21 | import stat |
| 22 | import subprocess |
| 23 | import sys |
| 24 | import tarfile |
| 25 | import tempfile |
| 26 | import urllib |
| 27 | import zipfile |
| 28 | |
| 29 | # TODO(vadimsh): Migrate to new golang.org/x/ paths once Golang moves to |
| 30 | # git completely. |
| 31 | |
| 32 | LOGGER = logging.getLogger(__name__) |
| 33 | |
| 34 | |
| 35 | # /path/to/util/bot |
| 36 | ROOT = os.path.dirname(os.path.abspath(__file__)) |
| 37 | |
| 38 | # Where to install Go toolset to. GOROOT would be <TOOLSET_ROOT>/go. |
| 39 | TOOLSET_ROOT = os.path.join(os.path.dirname(ROOT), 'golang') |
| 40 | |
| 41 | # Default workspace with infra go code. |
| 42 | WORKSPACE = os.path.join(ROOT, 'go') |
| 43 | |
| 44 | # Platform depended suffix for executable files. |
| 45 | EXE_SFX = '.exe' if sys.platform == 'win32' else '' |
| 46 | |
| 47 | # Pinned version of Go toolset to download. |
Pete Bentley | 0c61efe | 2019-08-13 09:32:23 +0100 | [diff] [blame^] | 48 | TOOLSET_VERSION = 'go1.12.6' |
Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 49 | |
| 50 | # Platform dependent portion of a download URL. See http://golang.org/dl/. |
| 51 | TOOLSET_VARIANTS = { |
Kenny Root | e99801b | 2015-11-06 15:31:15 -0800 | [diff] [blame] | 52 | ('darwin', 'x86-64'): 'darwin-amd64.tar.gz', |
Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 53 | ('linux2', 'x86-32'): 'linux-386.tar.gz', |
| 54 | ('linux2', 'x86-64'): 'linux-amd64.tar.gz', |
| 55 | ('win32', 'x86-32'): 'windows-386.zip', |
| 56 | ('win32', 'x86-64'): 'windows-amd64.zip', |
| 57 | } |
| 58 | |
| 59 | # Download URL root. |
| 60 | DOWNLOAD_URL_PREFIX = 'https://storage.googleapis.com/golang' |
| 61 | |
| 62 | |
| 63 | class Failure(Exception): |
| 64 | """Bootstrap failed.""" |
| 65 | |
| 66 | |
| 67 | def get_toolset_url(): |
| 68 | """URL of a platform specific Go toolset archive.""" |
| 69 | # TODO(vadimsh): Support toolset for cross-compilation. |
| 70 | arch = { |
| 71 | 'amd64': 'x86-64', |
| 72 | 'x86_64': 'x86-64', |
| 73 | 'i386': 'x86-32', |
| 74 | 'x86': 'x86-32', |
| 75 | }.get(platform.machine().lower()) |
| 76 | variant = TOOLSET_VARIANTS.get((sys.platform, arch)) |
| 77 | if not variant: |
| 78 | # TODO(vadimsh): Compile go lang from source. |
| 79 | raise Failure('Unrecognized platform') |
| 80 | return '%s/%s.%s' % (DOWNLOAD_URL_PREFIX, TOOLSET_VERSION, variant) |
| 81 | |
| 82 | |
| 83 | def read_file(path): |
| 84 | """Returns contents of a given file or None if not readable.""" |
| 85 | assert isinstance(path, (list, tuple)) |
| 86 | try: |
| 87 | with open(os.path.join(*path), 'r') as f: |
| 88 | return f.read() |
| 89 | except IOError: |
| 90 | return None |
| 91 | |
| 92 | |
| 93 | def write_file(path, data): |
| 94 | """Writes |data| to a file.""" |
| 95 | assert isinstance(path, (list, tuple)) |
| 96 | with open(os.path.join(*path), 'w') as f: |
| 97 | f.write(data) |
| 98 | |
| 99 | |
| 100 | def remove_directory(path): |
| 101 | """Recursively removes a directory.""" |
| 102 | assert isinstance(path, (list, tuple)) |
| 103 | p = os.path.join(*path) |
| 104 | if not os.path.exists(p): |
| 105 | return |
| 106 | LOGGER.info('Removing %s', p) |
| 107 | # Crutch to remove read-only file (.git/* in particular) on Windows. |
| 108 | def onerror(func, path, _exc_info): |
| 109 | if not os.access(path, os.W_OK): |
| 110 | os.chmod(path, stat.S_IWUSR) |
| 111 | func(path) |
| 112 | else: |
| 113 | raise |
| 114 | shutil.rmtree(p, onerror=onerror if sys.platform == 'win32' else None) |
| 115 | |
| 116 | |
| 117 | def install_toolset(toolset_root, url): |
| 118 | """Downloads and installs Go toolset. |
| 119 | |
| 120 | GOROOT would be <toolset_root>/go/. |
| 121 | """ |
| 122 | if not os.path.exists(toolset_root): |
| 123 | os.makedirs(toolset_root) |
| 124 | pkg_path = os.path.join(toolset_root, url[url.rfind('/')+1:]) |
| 125 | |
| 126 | LOGGER.info('Downloading %s...', url) |
| 127 | download_file(url, pkg_path) |
| 128 | |
| 129 | LOGGER.info('Extracting...') |
| 130 | if pkg_path.endswith('.zip'): |
| 131 | with zipfile.ZipFile(pkg_path, 'r') as f: |
| 132 | f.extractall(toolset_root) |
| 133 | elif pkg_path.endswith('.tar.gz'): |
| 134 | with tarfile.open(pkg_path, 'r:gz') as f: |
| 135 | f.extractall(toolset_root) |
| 136 | else: |
| 137 | raise Failure('Unrecognized archive format') |
| 138 | |
| 139 | LOGGER.info('Validating...') |
| 140 | if not check_hello_world(toolset_root): |
| 141 | raise Failure('Something is not right, test program doesn\'t work') |
| 142 | |
| 143 | |
| 144 | def download_file(url, path): |
| 145 | """Fetches |url| to |path|.""" |
| 146 | last_progress = [0] |
| 147 | def report(a, b, c): |
| 148 | progress = int(a * b * 100.0 / c) |
| 149 | if progress != last_progress[0]: |
| 150 | print >> sys.stderr, 'Downloading... %d%%' % progress |
| 151 | last_progress[0] = progress |
| 152 | # TODO(vadimsh): Use something less crippled, something that validates SSL. |
| 153 | urllib.urlretrieve(url, path, reporthook=report) |
| 154 | |
| 155 | |
| 156 | @contextlib.contextmanager |
| 157 | def temp_dir(path): |
| 158 | """Creates a temporary directory, then deletes it.""" |
| 159 | tmp = tempfile.mkdtemp(dir=path) |
| 160 | try: |
| 161 | yield tmp |
| 162 | finally: |
| 163 | remove_directory([tmp]) |
| 164 | |
| 165 | |
| 166 | def check_hello_world(toolset_root): |
| 167 | """Compiles and runs 'hello world' program to verify that toolset works.""" |
| 168 | with temp_dir(toolset_root) as tmp: |
| 169 | path = os.path.join(tmp, 'hello.go') |
| 170 | write_file([path], r""" |
| 171 | package main |
| 172 | func main() { println("hello, world\n") } |
| 173 | """) |
| 174 | out = subprocess.check_output( |
| 175 | [get_go_exe(toolset_root), 'run', path], |
| 176 | env=get_go_environ(toolset_root, tmp), |
| 177 | stderr=subprocess.STDOUT) |
| 178 | if out.strip() != 'hello, world': |
| 179 | LOGGER.error('Failed to run sample program:\n%s', out) |
| 180 | return False |
| 181 | return True |
| 182 | |
| 183 | |
| 184 | def ensure_toolset_installed(toolset_root): |
| 185 | """Installs or updates Go toolset if necessary. |
| 186 | |
| 187 | Returns True if new toolset was installed. |
| 188 | """ |
| 189 | installed = read_file([toolset_root, 'INSTALLED_TOOLSET']) |
| 190 | available = get_toolset_url() |
| 191 | if installed == available: |
| 192 | LOGGER.debug('Go toolset is up-to-date: %s', TOOLSET_VERSION) |
| 193 | return False |
| 194 | |
| 195 | LOGGER.info('Installing Go toolset.') |
| 196 | LOGGER.info(' Old toolset is %s', installed) |
| 197 | LOGGER.info(' New toolset is %s', available) |
| 198 | remove_directory([toolset_root]) |
| 199 | install_toolset(toolset_root, available) |
| 200 | LOGGER.info('Go toolset installed: %s', TOOLSET_VERSION) |
| 201 | write_file([toolset_root, 'INSTALLED_TOOLSET'], available) |
| 202 | return True |
| 203 | |
| 204 | |
| 205 | def get_go_environ( |
| 206 | toolset_root, |
| 207 | workspace=None): |
| 208 | """Returns a copy of os.environ with added GO* environment variables. |
| 209 | |
| 210 | Overrides GOROOT, GOPATH and GOBIN. Keeps everything else. Idempotent. |
| 211 | |
| 212 | Args: |
| 213 | toolset_root: GOROOT would be <toolset_root>/go. |
| 214 | workspace: main workspace directory or None if compiling in GOROOT. |
| 215 | """ |
| 216 | env = os.environ.copy() |
| 217 | env['GOROOT'] = os.path.join(toolset_root, 'go') |
| 218 | if workspace: |
| 219 | env['GOBIN'] = os.path.join(workspace, 'bin') |
| 220 | else: |
| 221 | env.pop('GOBIN', None) |
| 222 | |
| 223 | all_go_paths = [] |
| 224 | if workspace: |
| 225 | all_go_paths.append(workspace) |
| 226 | env['GOPATH'] = os.pathsep.join(all_go_paths) |
| 227 | |
| 228 | # New PATH entries. |
| 229 | paths_to_add = [ |
| 230 | os.path.join(env['GOROOT'], 'bin'), |
| 231 | env.get('GOBIN'), |
| 232 | ] |
| 233 | |
| 234 | # Make sure not to add duplicates entries to PATH over and over again when |
| 235 | # get_go_environ is invoked multiple times. |
| 236 | path = env['PATH'].split(os.pathsep) |
| 237 | paths_to_add = [p for p in paths_to_add if p and p not in path] |
| 238 | env['PATH'] = os.pathsep.join(paths_to_add + path) |
| 239 | |
| 240 | return env |
| 241 | |
| 242 | |
| 243 | def get_go_exe(toolset_root): |
| 244 | """Returns path to go executable.""" |
| 245 | return os.path.join(toolset_root, 'go', 'bin', 'go' + EXE_SFX) |
| 246 | |
| 247 | |
| 248 | def bootstrap(logging_level): |
| 249 | """Installs all dependencies in default locations. |
| 250 | |
| 251 | Supposed to be called at the beginning of some script (it modifies logger). |
| 252 | |
| 253 | Args: |
| 254 | logging_level: logging level of bootstrap process. |
| 255 | """ |
| 256 | logging.basicConfig() |
| 257 | LOGGER.setLevel(logging_level) |
| 258 | ensure_toolset_installed(TOOLSET_ROOT) |
| 259 | |
| 260 | |
| 261 | def prepare_go_environ(): |
| 262 | """Returns dict with environment variables to set to use Go toolset. |
| 263 | |
| 264 | Installs or updates the toolset if necessary. |
| 265 | """ |
| 266 | bootstrap(logging.INFO) |
| 267 | return get_go_environ(TOOLSET_ROOT, WORKSPACE) |
| 268 | |
| 269 | |
| 270 | def find_executable(name, workspaces): |
| 271 | """Returns full path to an executable in some bin/ (in GOROOT or GOBIN).""" |
| 272 | basename = name |
| 273 | if EXE_SFX and basename.endswith(EXE_SFX): |
| 274 | basename = basename[:-len(EXE_SFX)] |
| 275 | roots = [os.path.join(TOOLSET_ROOT, 'go', 'bin')] |
| 276 | for path in workspaces: |
| 277 | roots.extend([ |
| 278 | os.path.join(path, 'bin'), |
| 279 | ]) |
| 280 | for root in roots: |
| 281 | full_path = os.path.join(root, basename + EXE_SFX) |
| 282 | if os.path.exists(full_path): |
| 283 | return full_path |
| 284 | return name |
| 285 | |
| 286 | |
| 287 | def main(args): |
| 288 | if args: |
| 289 | print >> sys.stderr, sys.modules[__name__].__doc__, |
| 290 | return 2 |
| 291 | bootstrap(logging.DEBUG) |
| 292 | return 0 |
| 293 | |
| 294 | |
| 295 | if __name__ == '__main__': |
| 296 | sys.exit(main(sys.argv[1:])) |