Mike Frysinger | d03e6b5 | 2019-08-03 12:49:01 -0400 | [diff] [blame] | 1 | #!/usr/bin/python2 |
Shuqian Zhao | ae2d078 | 2016-11-15 16:58:47 -0800 | [diff] [blame] | 2 | |
| 3 | # Copyright (c) 2016 The Chromium OS Authors. All rights reserved. |
| 4 | # Use of this source code is governed by a BSD-style license that can be |
| 5 | # found in the LICENSE file. |
| 6 | |
| 7 | """Module to automate the process of deploying to production. |
| 8 | |
| 9 | Example usage of this script: |
| 10 | 1. Update both autotest and chromite to the lastest commit that has passed |
| 11 | the test instance. |
| 12 | $ ./site_utils/automated_deploy.py |
| 13 | 2. Skip updating a repo, e.g. autotest |
| 14 | $ ./site_utils/automated_deploy.py --skip_autotest |
| 15 | 3. Update a given repo to a specific commit |
| 16 | $ ./site_utils/automated_deploy.py --autotest_hash='1234' |
| 17 | """ |
| 18 | |
| 19 | import argparse |
C Shapiro | f76bb96 | 2019-05-10 17:22:40 -0600 | [diff] [blame] | 20 | import filecmp |
Shuqian Zhao | ae2d078 | 2016-11-15 16:58:47 -0800 | [diff] [blame] | 21 | import os |
| 22 | import re |
| 23 | import sys |
| 24 | import subprocess |
| 25 | |
| 26 | import common |
| 27 | from autotest_lib.client.common_lib import revision_control |
| 28 | from autotest_lib.site_utils.lib import infra |
| 29 | |
C Shapiro | f76bb96 | 2019-05-10 17:22:40 -0600 | [diff] [blame] | 30 | INFRA_DATA = 'infradata' |
Shuqian Zhao | ae2d078 | 2016-11-15 16:58:47 -0800 | [diff] [blame] | 31 | AUTOTEST_DIR = common.autotest_dir |
| 32 | GIT_URL = {'autotest': |
| 33 | 'https://chromium.googlesource.com/chromiumos/third_party/autotest', |
| 34 | 'chromite': |
C Shapiro | f76bb96 | 2019-05-10 17:22:40 -0600 | [diff] [blame] | 35 | 'https://chromium.googlesource.com/chromiumos/chromite', |
| 36 | INFRA_DATA: |
| 37 | 'https://chrome-internal.googlesource.com/infradata/config'} |
Shuqian Zhao | ae2d078 | 2016-11-15 16:58:47 -0800 | [diff] [blame] | 38 | PROD_BRANCH = 'prod' |
| 39 | MASTER_AFE = 'cautotest' |
Shuqian Zhao | a482c4a | 2016-11-21 18:49:41 -0800 | [diff] [blame] | 40 | NOTIFY_GROUP = 'chromeos-infra-discuss@google.com' |
Shuqian Zhao | ae2d078 | 2016-11-15 16:58:47 -0800 | [diff] [blame] | 41 | |
Daniel Erat | 590a499 | 2018-01-29 15:57:58 -0800 | [diff] [blame] | 42 | # CIPD packages whose prod refs should be updated. |
| 43 | _CIPD_PACKAGES = ( |
| 44 | 'chromiumos/infra/lucifer', |
Prathmesh Prabhu | 2f27423 | 2019-03-06 13:57:54 -0800 | [diff] [blame] | 45 | 'chromiumos/infra/skylab/linux-amd64', |
Allen Li | 9faf4cb | 2018-07-10 10:45:56 -0700 | [diff] [blame] | 46 | 'chromiumos/infra/skylab-inventory', |
Allen Li | 146a361 | 2018-08-28 16:47:08 -0700 | [diff] [blame] | 47 | 'chromiumos/infra/skylab_swarming_worker/linux-amd64', |
Alex Zamorzaev | 37f8c5e | 2019-06-26 17:04:59 -0700 | [diff] [blame] | 48 | 'chromiumos/infra/autotest_status_parser/linux-amd64', |
Daniel Erat | 590a499 | 2018-01-29 15:57:58 -0800 | [diff] [blame] | 49 | ) |
| 50 | |
Shuqian Zhao | ae2d078 | 2016-11-15 16:58:47 -0800 | [diff] [blame] | 51 | |
| 52 | class AutoDeployException(Exception): |
| 53 | """Raised when any deploy step fails.""" |
| 54 | |
| 55 | |
| 56 | def parse_arguments(): |
| 57 | """Parse command line arguments. |
| 58 | |
| 59 | @returns An argparse.Namespace populated with argument values. |
| 60 | """ |
| 61 | parser = argparse.ArgumentParser( |
| 62 | description=('Command to update prod branch for autotest, chromite ' |
| 63 | 'repos. Then deploy new changes to all lab servers.')) |
| 64 | parser.add_argument('--skip_autotest', action='store_true', default=False, |
| 65 | help='Skip updating autotest prod branch. Default is False.') |
| 66 | parser.add_argument('--skip_chromite', action='store_true', default=False, |
| 67 | help='Skip updating chromite prod branch. Default is False.') |
C Shapiro | f76bb96 | 2019-05-10 17:22:40 -0600 | [diff] [blame] | 68 | parser.add_argument('--skip_prod_config_check', action='store_true', |
| 69 | default=False, |
| 70 | help='Skip checking prod config matches dev. ' |
| 71 | 'Default is False.') |
Shuqian Zhao | 67ee3b4 | 2017-12-07 11:12:18 -0800 | [diff] [blame] | 72 | parser.add_argument('--force_update', action='store_true', default=False, |
| 73 | help=('Force a deployment without updating both autotest and ' |
| 74 | 'chromite prod branch')) |
Shuqian Zhao | ae2d078 | 2016-11-15 16:58:47 -0800 | [diff] [blame] | 75 | parser.add_argument('--autotest_hash', type=str, default=None, |
| 76 | help='Update autotest prod branch to the given hash. If it is not' |
| 77 | ' specified, autotest prod branch will be rebased to ' |
| 78 | 'prod-next branch, which is the latest commit that has ' |
| 79 | 'passed our test instance.') |
| 80 | parser.add_argument('--chromite_hash', type=str, default=None, |
| 81 | help='Same as autotest_hash option.') |
| 82 | |
| 83 | results = parser.parse_args(sys.argv[1:]) |
| 84 | |
| 85 | # Verify the validity of the options. |
| 86 | if ((results.skip_autotest and results.autotest_hash) or |
| 87 | (results.skip_chromite and results.chromite_hash)): |
| 88 | parser.print_help() |
| 89 | print 'Cannot specify skip_* and *_hash options at the same time.' |
| 90 | sys.exit(1) |
Shuqian Zhao | 67ee3b4 | 2017-12-07 11:12:18 -0800 | [diff] [blame] | 91 | if results.force_update: |
| 92 | results.skip_autotest = True |
| 93 | results.skip_chromite = True |
Shuqian Zhao | ae2d078 | 2016-11-15 16:58:47 -0800 | [diff] [blame] | 94 | return results |
| 95 | |
| 96 | |
C Shapiro | f76bb96 | 2019-05-10 17:22:40 -0600 | [diff] [blame] | 97 | def verify_dev_infradata_config_matches_prod(): |
| 98 | """Checks that dev config matches prod config before push |
| 99 | |
| 100 | Based on crbug.com/949696, this checks to make sure that dev |
| 101 | config changes have been cloned to prod before a prod push. |
| 102 | |
| 103 | @raises subprocess.CalledProcessError on a command failure. |
| 104 | @raised revision_control.GitCloneError when git clone fails. |
| 105 | """ |
| 106 | |
| 107 | repo_dir = os.path.join('/tmp', INFRA_DATA) |
| 108 | git_url = GIT_URL[INFRA_DATA] |
| 109 | print 'Checking PROD matches DEV config for %s repo' % git_url |
| 110 | print 'Cloning %s master branch under %s' % (git_url, repo_dir) |
| 111 | if os.path.exists(repo_dir): |
| 112 | infra.local_runner('rm -rf %s' % repo_dir) |
| 113 | git_repo = revision_control.GitRepo(repo_dir, git_url) |
| 114 | git_repo.clone(shallow=True) |
| 115 | |
| 116 | dev_to_prod_files = { |
| 117 | 'configs/chromium-swarm-dev/scripts/skylab.py': |
| 118 | 'configs/chromeos-swarming/scripts/skylab.py', |
| 119 | } |
| 120 | |
| 121 | for dev_rel_path, prod_rel_path in dev_to_prod_files.items(): |
| 122 | if filecmp.cmp(os.path.join(repo_dir, dev_rel_path), |
| 123 | os.path.join(repo_dir, prod_rel_path), |
| 124 | shallow=False): |
| 125 | continue |
| 126 | |
| 127 | master_url = git_url + '/+/refs/heads/master/' |
| 128 | raise AutoDeployException( |
| 129 | '\n\n%s\nDOES NOT MATCH\n%s' |
| 130 | '\n\nCopy DEV config to PROD config before performing release' % |
| 131 | (master_url + prod_rel_path, master_url + dev_rel_path)) |
| 132 | |
| 133 | print 'Successfully verified PROD config matches DEV config' |
| 134 | |
| 135 | |
Shuqian Zhao | ae2d078 | 2016-11-15 16:58:47 -0800 | [diff] [blame] | 136 | def clone_prod_branch(repo): |
| 137 | """Method to clone the prod branch for a given repo under /tmp/ dir. |
| 138 | |
| 139 | @param repo: Name of the git repo to be cloned. |
| 140 | |
| 141 | @returns path to the cloned repo. |
| 142 | @raises subprocess.CalledProcessError on a command failure. |
| 143 | @raised revision_control.GitCloneError when git clone fails. |
| 144 | """ |
| 145 | repo_dir = '/tmp/%s' % repo |
| 146 | print 'Cloning %s prod branch under %s' % (repo, repo_dir) |
| 147 | if os.path.exists(repo_dir): |
| 148 | infra.local_runner('rm -rf %s' % repo_dir) |
| 149 | git_repo = revision_control.GitRepo(repo_dir, GIT_URL[repo]) |
| 150 | git_repo.clone(remote_branch=PROD_BRANCH) |
| 151 | print 'Successfully cloned %s prod branch' % repo |
| 152 | return repo_dir |
| 153 | |
| 154 | |
| 155 | def update_prod_branch(repo, repo_dir, hash_to_rebase): |
| 156 | """Method to update the prod branch of the given repo to the given hash. |
| 157 | |
| 158 | @param repo: Name of the git repo to be updated. |
| 159 | @param repo_dir: path to the cloned repo. |
| 160 | @param hash_to_rebase: Hash to rebase the prod branch to. If it is None, |
| 161 | prod branch will rebase to prod-next branch. |
| 162 | |
Shuqian Zhao | 673519b | 2017-05-05 15:13:25 -0700 | [diff] [blame] | 163 | @returns the range of the pushed commits as a string. E.g 123...345. If the |
| 164 | prod branch is already up-to-date, return None. |
Shuqian Zhao | ae2d078 | 2016-11-15 16:58:47 -0800 | [diff] [blame] | 165 | @raises subprocess.CalledProcessError on a command failure. |
| 166 | """ |
| 167 | with infra.chdir(repo_dir): |
Shuqian Zhao | a482c4a | 2016-11-21 18:49:41 -0800 | [diff] [blame] | 168 | print 'Updating %s prod branch.' % repo |
| 169 | rebase_to = hash_to_rebase if hash_to_rebase else 'origin/prod-next' |
Shuqian Zhao | 673519b | 2017-05-05 15:13:25 -0700 | [diff] [blame] | 170 | # Check whether prod branch is already up-to-date, which means there is |
| 171 | # no changes since last push. |
| 172 | print 'Detecting new changes since last push...' |
| 173 | diff = infra.local_runner('git log prod..%s --oneline' % rebase_to, |
| 174 | stream_output=True) |
| 175 | if diff: |
| 176 | print 'Find new changes, will update prod branch...' |
| 177 | infra.local_runner('git rebase %s prod' % rebase_to, |
| 178 | stream_output=True) |
| 179 | result = infra.local_runner('git push origin prod', |
| 180 | stream_output=True) |
| 181 | print 'Successfully pushed %s prod branch!\n' % repo |
Shuqian Zhao | ae2d078 | 2016-11-15 16:58:47 -0800 | [diff] [blame] | 182 | |
Shuqian Zhao | 673519b | 2017-05-05 15:13:25 -0700 | [diff] [blame] | 183 | # Get the pushed commit range, which is used to get pushed commits |
| 184 | # using git log E.g. 123..456, then run git log --oneline 123..456. |
| 185 | grep = re.search('(\w)*\.\.(\w)*', result) |
Shuqian Zhao | ae2d078 | 2016-11-15 16:58:47 -0800 | [diff] [blame] | 186 | |
Shuqian Zhao | 673519b | 2017-05-05 15:13:25 -0700 | [diff] [blame] | 187 | if not grep: |
| 188 | raise AutoDeployException( |
| 189 | 'Fail to get pushed commits for repo %s from git log: %s' % |
| 190 | (repo, result)) |
| 191 | return grep.group(0) |
| 192 | else: |
| 193 | print 'No new %s changes found since last push.' % repo |
| 194 | return None |
Shuqian Zhao | ae2d078 | 2016-11-15 16:58:47 -0800 | [diff] [blame] | 195 | |
| 196 | |
| 197 | def get_pushed_commits(repo, repo_dir, pushed_commits_range): |
| 198 | """Method to get the pushed commits. |
| 199 | |
| 200 | @param repo: Name of the updated git repo. |
| 201 | @param repo_dir: path to the cloned repo. |
| 202 | @param pushed_commits_range: The range of the pushed commits. E.g 123...345 |
| 203 | @return: the commits that are pushed to prod branch. The format likes this: |
| 204 | "git log --oneline A...B | grep autotest |
| 205 | A xxxx |
| 206 | B xxxx" |
| 207 | @raises subprocess.CalledProcessError on a command failure. |
| 208 | """ |
Shuqian Zhao | a482c4a | 2016-11-21 18:49:41 -0800 | [diff] [blame] | 209 | print 'Getting pushed CLs for %s repo.' % repo |
Shuqian Zhao | 673519b | 2017-05-05 15:13:25 -0700 | [diff] [blame] | 210 | if not pushed_commits_range: |
| 211 | return '\n%s:\nNo new changes since last push.' % repo |
| 212 | |
Shuqian Zhao | ae2d078 | 2016-11-15 16:58:47 -0800 | [diff] [blame] | 213 | with infra.chdir(repo_dir): |
| 214 | get_commits_cmd = 'git log --oneline %s' % pushed_commits_range |
xixuan | 6d782dc | 2017-06-21 18:08:48 -0700 | [diff] [blame] | 215 | |
| 216 | pushed_commits = infra.local_runner( |
| 217 | get_commits_cmd, stream_output=True) |
Shuqian Zhao | ae2d078 | 2016-11-15 16:58:47 -0800 | [diff] [blame] | 218 | if repo == 'autotest': |
xixuan | 6d782dc | 2017-06-21 18:08:48 -0700 | [diff] [blame] | 219 | autotest_commits = '' |
| 220 | for cl in pushed_commits.splitlines(): |
| 221 | if 'autotest' in cl: |
| 222 | autotest_commits += '%s\n' % cl |
| 223 | |
| 224 | pushed_commits = autotest_commits |
| 225 | |
Shuqian Zhao | a482c4a | 2016-11-21 18:49:41 -0800 | [diff] [blame] | 226 | print 'Successfully got pushed CLs for %s repo!\n' % repo |
Aviv Keshet | 6dd1c3d | 2017-09-26 17:46:49 -0700 | [diff] [blame] | 227 | displayed_cmd = get_commits_cmd |
| 228 | if repo == 'autotest': |
| 229 | displayed_cmd += ' | grep autotest' |
| 230 | return '\n%s:\n%s\n%s\n' % (repo, displayed_cmd, pushed_commits) |
Shuqian Zhao | ae2d078 | 2016-11-15 16:58:47 -0800 | [diff] [blame] | 231 | |
| 232 | |
| 233 | def kick_off_deploy(): |
| 234 | """Method to kick off deploy script to deploy changes to lab servers. |
| 235 | |
| 236 | @raises subprocess.CalledProcessError on a repo command failure. |
| 237 | """ |
| 238 | print 'Start deploying changes to all lab servers...' |
| 239 | with infra.chdir(AUTOTEST_DIR): |
Shuqian Zhao | ae2d078 | 2016-11-15 16:58:47 -0800 | [diff] [blame] | 240 | # Then kick off the deploy script. |
Aviv Keshet | fc59c14 | 2017-10-31 09:27:57 -0700 | [diff] [blame] | 241 | deploy_cmd = ('runlocalssh ./site_utils/deploy_server.py -x --afe=%s' % |
Shuqian Zhao | ae2d078 | 2016-11-15 16:58:47 -0800 | [diff] [blame] | 242 | MASTER_AFE) |
Shuqian Zhao | 6ad127a | 2017-05-22 22:57:19 +0000 | [diff] [blame] | 243 | infra.local_runner(deploy_cmd, stream_output=True) |
Aviv Keshet | fc59c14 | 2017-10-31 09:27:57 -0700 | [diff] [blame] | 244 | print 'Successfully deployed changes to all lab servers.' |
Shuqian Zhao | ae2d078 | 2016-11-15 16:58:47 -0800 | [diff] [blame] | 245 | |
| 246 | |
| 247 | def main(args): |
| 248 | """Main entry""" |
| 249 | options = parse_arguments() |
| 250 | repos = dict() |
| 251 | if not options.skip_autotest: |
| 252 | repos.update({'autotest': options.autotest_hash}) |
| 253 | if not options.skip_chromite: |
| 254 | repos.update({'chromite': options.chromite_hash}) |
| 255 | |
C Shapiro | f76bb96 | 2019-05-10 17:22:40 -0600 | [diff] [blame] | 256 | if not options.skip_prod_config_check: |
| 257 | verify_dev_infradata_config_matches_prod() |
| 258 | |
Daniel Erat | 590a499 | 2018-01-29 15:57:58 -0800 | [diff] [blame] | 259 | print 'Moving CIPD prod refs to prod-next' |
| 260 | for pkg in _CIPD_PACKAGES: |
| 261 | subprocess.check_call(['cipd', 'set-ref', pkg, '-version', 'prod-next', |
| 262 | '-ref', 'prod']) |
Shuqian Zhao | ae2d078 | 2016-11-15 16:58:47 -0800 | [diff] [blame] | 263 | try: |
| 264 | # update_log saves the git log of the updated repo. |
| 265 | update_log = '' |
| 266 | for repo, hash_to_rebase in repos.iteritems(): |
| 267 | repo_dir = clone_prod_branch(repo) |
| 268 | push_commits_range = update_prod_branch( |
| 269 | repo, repo_dir, hash_to_rebase) |
| 270 | update_log += get_pushed_commits(repo, repo_dir, push_commits_range) |
| 271 | |
| 272 | kick_off_deploy() |
| 273 | except revision_control.GitCloneError as e: |
| 274 | print 'Fail to clone prod branch. Error:\n%s\n' % e |
| 275 | raise |
| 276 | except subprocess.CalledProcessError as e: |
Shuqian Zhao | 1c3f246 | 2017-02-02 17:21:42 -0800 | [diff] [blame] | 277 | print ('Deploy fails when running a subprocess cmd :\n%s\n' |
| 278 | 'Below is the push log:\n%s\n' % (e.output, update_log)) |
Shuqian Zhao | ae2d078 | 2016-11-15 16:58:47 -0800 | [diff] [blame] | 279 | raise |
| 280 | except Exception as e: |
Shuqian Zhao | 1c3f246 | 2017-02-02 17:21:42 -0800 | [diff] [blame] | 281 | print 'Deploy fails with error:\n%s\nPush log:\n%s\n' % (e, update_log) |
Shuqian Zhao | ae2d078 | 2016-11-15 16:58:47 -0800 | [diff] [blame] | 282 | raise |
| 283 | |
| 284 | # When deploy succeeds, print the update_log. |
| 285 | print ('Deploy succeeds!!! Below is the push log of the updated repo:\n%s' |
Shuqian Zhao | a482c4a | 2016-11-21 18:49:41 -0800 | [diff] [blame] | 286 | 'Please email this to %s.'% (update_log, NOTIFY_GROUP)) |
Shuqian Zhao | ae2d078 | 2016-11-15 16:58:47 -0800 | [diff] [blame] | 287 | |
| 288 | |
| 289 | if __name__ == '__main__': |
| 290 | sys.exit(main(sys.argv)) |