Caroline Tice | 88272d4 | 2016-01-13 09:48:29 -0800 | [diff] [blame^] | 1 | #!/usr/bin/python2 |
bjanakiraman | 7f4a485 | 2013-02-15 04:35:28 +0000 | [diff] [blame] | 2 | # |
Rahul Chaudhry | c1635a8 | 2014-10-31 14:59:39 -0700 | [diff] [blame] | 3 | # Copyright 2010 The Chromium OS Authors. All rights reserved. |
| 4 | # Use of this source code is governed by a BSD-style license that can be |
| 5 | # found in the LICENSE file. |
bjanakiraman | 7f4a485 | 2013-02-15 04:35:28 +0000 | [diff] [blame] | 6 | """Script to checkout the ChromeOS source. |
| 7 | |
| 8 | This script sets up the ChromeOS source in the given directory, matching a |
| 9 | particular release of ChromeOS. |
| 10 | """ |
| 11 | |
Caroline Tice | 88272d4 | 2016-01-13 09:48:29 -0800 | [diff] [blame^] | 12 | from __future__ import print_function |
| 13 | |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 14 | __author__ = 'raymes@google.com (Raymes Khoury)' |
bjanakiraman | 7f4a485 | 2013-02-15 04:35:28 +0000 | [diff] [blame] | 15 | |
llozano | 9efafde | 2013-02-23 01:37:04 +0000 | [diff] [blame] | 16 | from datetime import datetime |
Luis Lozano | 8cf5308 | 2013-08-01 12:35:11 -0700 | [diff] [blame] | 17 | |
Caroline Tice | 88272d4 | 2016-01-13 09:48:29 -0800 | [diff] [blame^] | 18 | import argparse |
yunlian | 3802fbf | 2013-02-19 19:58:44 +0000 | [diff] [blame] | 19 | import os |
llozano | 9efafde | 2013-02-23 01:37:04 +0000 | [diff] [blame] | 20 | import pickle |
bjanakiraman | 7f4a485 | 2013-02-15 04:35:28 +0000 | [diff] [blame] | 21 | import sys |
yunlian | 3802fbf | 2013-02-19 19:58:44 +0000 | [diff] [blame] | 22 | import tempfile |
| 23 | import time |
Caroline Tice | 88272d4 | 2016-01-13 09:48:29 -0800 | [diff] [blame^] | 24 | from cros_utils import command_executer |
| 25 | from cros_utils import logger |
| 26 | from cros_utils import manifest_versions |
bjanakiraman | 7f4a485 | 2013-02-15 04:35:28 +0000 | [diff] [blame] | 27 | |
raymes | bfb5799 | 2013-02-15 04:35:45 +0000 | [diff] [blame] | 28 | GCLIENT_FILE = """solutions = [ |
| 29 | { "name" : "CHROME_DEPS", |
| 30 | "url" : |
| 31 | "svn://svn.chromium.org/chrome-internal/trunk/tools/buildspec/releases/%s", |
| 32 | "custom_deps" : { |
| 33 | "src/third_party/WebKit/LayoutTests": None, |
| 34 | "src-pdf": None, |
| 35 | "src/pdf": None, |
| 36 | }, |
| 37 | "safesync_url": "", |
| 38 | }, |
| 39 | ] |
| 40 | """ |
| 41 | |
llozano | 9efafde | 2013-02-23 01:37:04 +0000 | [diff] [blame] | 42 | # List of stable versions used for common team image |
| 43 | # Sheriff must update this list when a new common version becomes available |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 44 | COMMON_VERSIONS = '/home/mobiletc-prebuild/common_images/common_list.txt' |
| 45 | |
llozano | 9efafde | 2013-02-23 01:37:04 +0000 | [diff] [blame] | 46 | |
bjanakiraman | 7f4a485 | 2013-02-15 04:35:28 +0000 | [diff] [blame] | 47 | def Usage(parser): |
| 48 | parser.print_help() |
| 49 | sys.exit(0) |
| 50 | |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 51 | |
shenhan | f6617ae | 2013-03-01 17:17:01 +0000 | [diff] [blame] | 52 | # Get version spec file, either from "paladin" or "buildspec" directory. |
| 53 | def GetVersionSpecFile(version, versions_git): |
| 54 | temp = tempfile.mkdtemp() |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 55 | commands = ['cd {0}'.format(temp), \ |
| 56 | 'git clone {0} versions'.format(versions_git)] |
shenhan | f6617ae | 2013-03-01 17:17:01 +0000 | [diff] [blame] | 57 | cmd_executer = command_executer.GetCommandExecuter() |
| 58 | ret = cmd_executer.RunCommands(commands) |
| 59 | err_msg = None |
| 60 | if ret: |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 61 | err_msg = 'Failed to checkout versions_git - {0}'.format(versions_git) |
shenhan | f6617ae | 2013-03-01 17:17:01 +0000 | [diff] [blame] | 62 | ret = None |
| 63 | else: |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 64 | v, m = version.split('.', 1) |
| 65 | paladin_spec = 'paladin/buildspecs/{0}/{1}.xml'.format(v, m) |
| 66 | generic_spec = 'buildspecs/{0}/{1}.xml'.format(v, m) |
| 67 | paladin_path = '{0}/versions/{1}'.format(temp, paladin_spec) |
| 68 | generic_path = '{0}/versions/{1}'.format(temp, generic_spec) |
shenhan | f6617ae | 2013-03-01 17:17:01 +0000 | [diff] [blame] | 69 | if os.path.exists(paladin_path): |
| 70 | ret = paladin_spec |
| 71 | elif os.path.exists(generic_path): |
| 72 | ret = generic_spec |
| 73 | else: |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 74 | err_msg = 'No spec found for version {0}'.format(version) |
shenhan | f6617ae | 2013-03-01 17:17:01 +0000 | [diff] [blame] | 75 | ret = None |
| 76 | # Fall through to clean up. |
| 77 | |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 78 | commands = ['rm -rf {0}'.format(temp)] |
shenhan | f6617ae | 2013-03-01 17:17:01 +0000 | [diff] [blame] | 79 | cmd_executer.RunCommands(commands) |
| 80 | if err_msg: |
| 81 | logger.GetLogger().LogFatal(err_msg) |
| 82 | return ret |
yunlian | 3802fbf | 2013-02-19 19:58:44 +0000 | [diff] [blame] | 83 | |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 84 | |
llozano | 9efafde | 2013-02-23 01:37:04 +0000 | [diff] [blame] | 85 | def TimeToCommonVersion(timestamp): |
| 86 | """Convert timestamp to common image version.""" |
| 87 | tdt = datetime.fromtimestamp(float(timestamp)) |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 88 | with open(COMMON_VERSIONS, 'r') as f: |
llozano | 9efafde | 2013-02-23 01:37:04 +0000 | [diff] [blame] | 89 | common_list = pickle.load(f) |
| 90 | for sv in common_list: |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 91 | sdt = datetime.strptime(sv['date'], '%Y-%m-%d %H:%M:%S.%f') |
llozano | 9efafde | 2013-02-23 01:37:04 +0000 | [diff] [blame] | 92 | if tdt >= sdt: |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 93 | return '%s.%s' % (sv['chrome_major_version'], sv['chromeos_version']) |
llozano | 9efafde | 2013-02-23 01:37:04 +0000 | [diff] [blame] | 94 | # should never reach here |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 95 | logger.GetLogger().LogFatal('No common version for timestamp') |
llozano | 9efafde | 2013-02-23 01:37:04 +0000 | [diff] [blame] | 96 | return None |
| 97 | |
| 98 | |
asharif | 0d3535a | 2013-02-15 04:50:33 +0000 | [diff] [blame] | 99 | def Main(argv): |
bjanakiraman | 7f4a485 | 2013-02-15 04:35:28 +0000 | [diff] [blame] | 100 | """Checkout the ChromeOS source.""" |
Caroline Tice | 88272d4 | 2016-01-13 09:48:29 -0800 | [diff] [blame^] | 101 | parser = argparse.ArgumentParser() |
| 102 | parser.add_argument('--dir', |
| 103 | dest='directory', |
| 104 | help='Target directory for ChromeOS installation.') |
| 105 | parser.add_argument('--version', |
| 106 | dest='version', |
| 107 | default='latest_lkgm', |
| 108 | help="""ChromeOS version. Can be: |
llozano | 9efafde | 2013-02-23 01:37:04 +0000 | [diff] [blame] | 109 | (1) A release version in the format: 'X.X.X.X' |
| 110 | (2) 'top' for top of trunk |
| 111 | (3) 'latest_lkgm' for the latest lkgm version |
| 112 | (4) 'lkgm' for the lkgm release before timestamp |
| 113 | (5) 'latest_common' for the latest team common stable version |
| 114 | (6) 'common' for the team common stable version before timestamp |
| 115 | Default is 'latest_lkgm'.""") |
Caroline Tice | 88272d4 | 2016-01-13 09:48:29 -0800 | [diff] [blame^] | 116 | parser.add_argument('--timestamp', |
| 117 | dest='timestamp', |
| 118 | default=None, |
| 119 | help='Timestamps in epoch format. It will check out the' |
| 120 | 'latest LKGM or the latest COMMON version of ChromeOS' |
| 121 | ' before the timestamp. Use in combination with' |
| 122 | ' --version=latest or --version=common. Use ' |
| 123 | '"date -d <date string> +%s" to find epoch time') |
| 124 | parser.add_argument('--minilayout', |
| 125 | dest='minilayout', |
| 126 | default=False, |
| 127 | action='store_true', |
| 128 | help='Whether to checkout the minilayout (smaller ' |
| 129 | 'checkout).') |
| 130 | parser.add_argument('--jobs', |
| 131 | '-j', |
| 132 | dest='jobs', |
| 133 | help='Number of repo sync threads to use.') |
| 134 | parser.add_argument('--public', |
| 135 | '-p', |
| 136 | dest='public', |
| 137 | default=False, |
| 138 | action='store_true', |
| 139 | help='Use the public checkout instead of the private ' |
| 140 | 'one.') |
bjanakiraman | 7f4a485 | 2013-02-15 04:35:28 +0000 | [diff] [blame] | 141 | |
Caroline Tice | 88272d4 | 2016-01-13 09:48:29 -0800 | [diff] [blame^] | 142 | options = parser.parse_args(argv) |
bjanakiraman | 7f4a485 | 2013-02-15 04:35:28 +0000 | [diff] [blame] | 143 | |
asharif | bf6899d | 2013-02-15 21:42:35 +0000 | [diff] [blame] | 144 | if not options.version: |
| 145 | parser.print_help() |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 146 | logger.GetLogger().LogFatal('No version specified.') |
bjanakiraman | 7f4a485 | 2013-02-15 04:35:28 +0000 | [diff] [blame] | 147 | else: |
| 148 | version = options.version.strip() |
| 149 | |
yunlian | 3802fbf | 2013-02-19 19:58:44 +0000 | [diff] [blame] | 150 | if not options.timestamp: |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 151 | timestamp = '' |
yunlian | 3802fbf | 2013-02-19 19:58:44 +0000 | [diff] [blame] | 152 | else: |
| 153 | timestamp = options.timestamp.strip() |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 154 | if version not in ('lkgm', 'common'): |
llozano | 9efafde | 2013-02-23 01:37:04 +0000 | [diff] [blame] | 155 | parser.print_help() |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 156 | logger.GetLogger().LogFatal('timestamp option only applies for ' |
llozano | 9efafde | 2013-02-23 01:37:04 +0000 | [diff] [blame] | 157 | "versions \"lkgm\" or \"common\"") |
yunlian | 3802fbf | 2013-02-19 19:58:44 +0000 | [diff] [blame] | 158 | |
asharif | bf6899d | 2013-02-15 21:42:35 +0000 | [diff] [blame] | 159 | if not options.directory: |
| 160 | parser.print_help() |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 161 | logger.GetLogger().LogFatal('No directory specified.') |
bjanakiraman | 7f4a485 | 2013-02-15 04:35:28 +0000 | [diff] [blame] | 162 | |
| 163 | directory = options.directory.strip() |
| 164 | |
asharif | bf6899d | 2013-02-15 21:42:35 +0000 | [diff] [blame] | 165 | if options.public: |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 166 | manifest_repo = 'https://chromium.googlesource.com/chromiumos/manifest.git' |
| 167 | versions_repo = ('https://chromium.googlesource.com/' |
| 168 | 'chromiumos/manifest-versions.git') |
asharif | bf6899d | 2013-02-15 21:42:35 +0000 | [diff] [blame] | 169 | else: |
| 170 | manifest_repo = ( |
Caroline Tice | 88272d4 | 2016-01-13 09:48:29 -0800 | [diff] [blame^] | 171 | 'https://chrome-internal.googlesource.com/chromeos/' |
| 172 | 'manifest-internal.git' |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 173 | ) |
asharif | bf6899d | 2013-02-15 21:42:35 +0000 | [diff] [blame] | 174 | versions_repo = ( |
Caroline Tice | 88272d4 | 2016-01-13 09:48:29 -0800 | [diff] [blame^] | 175 | 'https://chrome-internal.googlesource.com/chromeos/' |
| 176 | 'manifest-versions.git' |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 177 | ) |
asharif | bf6899d | 2013-02-15 21:42:35 +0000 | [diff] [blame] | 178 | |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 179 | if version == 'top': |
| 180 | init = 'repo init -u %s' % manifest_repo |
| 181 | elif version == 'latest_lkgm': |
Luis Lozano | 8cf5308 | 2013-08-01 12:35:11 -0700 | [diff] [blame] | 182 | manifests = manifest_versions.ManifestVersions() |
| 183 | version = manifests.TimeToVersion(time.mktime(time.gmtime())) |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 184 | version, manifest = version.split('.', 1) |
| 185 | logger.GetLogger().LogOutput('found version %s.%s for latest LKGM' % |
| 186 | (version, manifest)) |
| 187 | init = ('repo init -u %s -m paladin/buildspecs/%s/%s.xml' % |
| 188 | (versions_repo, version, manifest)) |
Luis Lozano | 8cf5308 | 2013-08-01 12:35:11 -0700 | [diff] [blame] | 189 | del manifests |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 190 | elif version == 'lkgm': |
llozano | 9efafde | 2013-02-23 01:37:04 +0000 | [diff] [blame] | 191 | if not timestamp: |
| 192 | parser.print_help() |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 193 | logger.GetLogger().LogFatal('No timestamp specified for version=lkgm') |
Luis Lozano | 8cf5308 | 2013-08-01 12:35:11 -0700 | [diff] [blame] | 194 | manifests = manifest_versions.ManifestVersions() |
| 195 | version = manifests.TimeToVersion(timestamp) |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 196 | version, manifest = version.split('.', 1) |
| 197 | logger.GetLogger().LogOutput('found version %s.%s for LKGM at timestamp %s' |
llozano | 9efafde | 2013-02-23 01:37:04 +0000 | [diff] [blame] | 198 | % (version, manifest, timestamp)) |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 199 | init = ('repo init -u %s -m paladin/buildspecs/%s/%s.xml' % |
| 200 | (versions_repo, version, manifest)) |
Luis Lozano | 8cf5308 | 2013-08-01 12:35:11 -0700 | [diff] [blame] | 201 | del manifests |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 202 | elif version == 'latest_common': |
llozano | 9efafde | 2013-02-23 01:37:04 +0000 | [diff] [blame] | 203 | version = TimeToCommonVersion(time.mktime(time.gmtime())) |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 204 | version, manifest = version.split('.', 1) |
| 205 | logger.GetLogger().LogOutput('found version %s.%s for latest Common image' % |
llozano | 9efafde | 2013-02-23 01:37:04 +0000 | [diff] [blame] | 206 | (version, manifest)) |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 207 | init = ('repo init -u %s -m buildspecs/%s/%s.xml' % (versions_repo, version, |
| 208 | manifest)) |
| 209 | elif version == 'common': |
llozano | 9efafde | 2013-02-23 01:37:04 +0000 | [diff] [blame] | 210 | if not timestamp: |
| 211 | parser.print_help() |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 212 | logger.GetLogger().LogFatal('No timestamp specified for version=lkgm') |
llozano | 9efafde | 2013-02-23 01:37:04 +0000 | [diff] [blame] | 213 | version = TimeToCommonVersion(timestamp) |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 214 | version, manifest = version.split('.', 1) |
| 215 | logger.GetLogger().LogOutput('found version %s.%s for latest common image ' |
| 216 | 'at timestamp %s' % (version, manifest, |
| 217 | timestamp)) |
| 218 | init = ('repo init -u %s -m buildspecs/%s/%s.xml' % (versions_repo, version, |
| 219 | manifest)) |
raymes | 69c8d72 | 2013-02-15 17:55:36 +0000 | [diff] [blame] | 220 | else: |
llozano | 9efafde | 2013-02-23 01:37:04 +0000 | [diff] [blame] | 221 | # user specified a specific version number |
shenhan | f6617ae | 2013-03-01 17:17:01 +0000 | [diff] [blame] | 222 | version_spec_file = GetVersionSpecFile(version, versions_repo) |
| 223 | if not version_spec_file: |
| 224 | return 1 |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 225 | init = 'repo init -u %s -m %s' % (versions_repo, version_spec_file) |
llozano | 9efafde | 2013-02-23 01:37:04 +0000 | [diff] [blame] | 226 | |
yunlian | 3802fbf | 2013-02-19 19:58:44 +0000 | [diff] [blame] | 227 | if options.minilayout: |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 228 | init += ' -g minilayout' |
yunlian | 3802fbf | 2013-02-19 19:58:44 +0000 | [diff] [blame] | 229 | |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 230 | init += ' --repo-url=https://chromium.googlesource.com/external/repo.git' |
bjanakiraman | 7f4a485 | 2013-02-15 04:35:28 +0000 | [diff] [blame] | 231 | |
Rahul Chaudhry | c1635a8 | 2014-10-31 14:59:39 -0700 | [diff] [blame] | 232 | # crosbug#31837 - "Sources need to be world-readable to properly |
| 233 | # function inside the chroot" |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 234 | sync = 'umask 022 && repo sync' |
Rahul Chaudhry | c1635a8 | 2014-10-31 14:59:39 -0700 | [diff] [blame] | 235 | if options.jobs: |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 236 | sync += ' -j %s' % options.jobs |
Rahul Chaudhry | c1635a8 | 2014-10-31 14:59:39 -0700 | [diff] [blame] | 237 | |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 238 | commands = ['mkdir -p %s' % directory, 'cd %s' % directory, init, sync] |
asharif | bf6899d | 2013-02-15 21:42:35 +0000 | [diff] [blame] | 239 | cmd_executer = command_executer.GetCommandExecuter() |
| 240 | ret = cmd_executer.RunCommands(commands) |
| 241 | if ret: |
| 242 | return ret |
bjanakiraman | 7f4a485 | 2013-02-15 04:35:28 +0000 | [diff] [blame] | 243 | |
asharif | bf6899d | 2013-02-15 21:42:35 +0000 | [diff] [blame] | 244 | return cmd_executer.RunCommand( |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 245 | 'git ls-remote ' |
| 246 | 'https://chrome-internal.googlesource.com/chrome/src-internal.git ' |
| 247 | '> /dev/null') |
bjanakiraman | 7f4a485 | 2013-02-15 04:35:28 +0000 | [diff] [blame] | 248 | |
| 249 | |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 250 | if __name__ == '__main__': |
Caroline Tice | 88272d4 | 2016-01-13 09:48:29 -0800 | [diff] [blame^] | 251 | retval = Main(sys.argv[1:]) |
asharif | 2198c51 | 2013-02-15 09:21:35 +0000 | [diff] [blame] | 252 | sys.exit(retval) |