Don Garrett | 8db752c | 2014-10-17 16:56:55 -0700 | [diff] [blame] | 1 | #!/usr/bin/python |
| 2 | # Copyright (c) 2014 The Chromium OS Authors. All rights reserved. |
| 3 | # Use of this source code is governed by a BSD-style license that can be |
| 4 | # found in the LICENSE file. |
| 5 | |
| 6 | """Runs on autotest servers from a cron job to self update them. |
| 7 | |
| 8 | This script is designed to run on all autotest servers to allow them to |
| 9 | automatically self-update based on the manifests used to create their (existing) |
| 10 | repos. |
| 11 | """ |
| 12 | |
| 13 | from __future__ import print_function |
| 14 | |
| 15 | import ConfigParser |
Don Garrett | 03432d6 | 2014-11-19 18:18:35 -0800 | [diff] [blame] | 16 | import argparse |
Don Garrett | 8db752c | 2014-10-17 16:56:55 -0700 | [diff] [blame] | 17 | import os |
Don Garrett | 699b4b3 | 2014-12-11 13:10:15 -0800 | [diff] [blame] | 18 | import re |
Don Garrett | 8db752c | 2014-10-17 16:56:55 -0700 | [diff] [blame] | 19 | import subprocess |
Shuqian Zhao | 79f248d | 2018-05-10 14:47:55 -0700 | [diff] [blame] | 20 | import socket |
Don Garrett | 8db752c | 2014-10-17 16:56:55 -0700 | [diff] [blame] | 21 | import sys |
| 22 | import time |
| 23 | |
| 24 | import common |
| 25 | |
| 26 | from autotest_lib.client.common_lib import global_config |
Dan Shi | ac6fdbf | 2016-04-09 17:36:28 -0700 | [diff] [blame] | 27 | from autotest_lib.server import utils as server_utils |
Dan Shi | cf27804 | 2016-04-06 21:16:34 -0700 | [diff] [blame] | 28 | from autotest_lib.server.cros.dynamic_suite import frontend_wrappers |
Don Garrett | 8db752c | 2014-10-17 16:56:55 -0700 | [diff] [blame] | 29 | |
Dan Shi | ac6fdbf | 2016-04-09 17:36:28 -0700 | [diff] [blame] | 30 | |
Don Garrett | d032172 | 2014-11-18 16:03:33 -0800 | [diff] [blame] | 31 | # How long after restarting a service do we watch it to see if it's stable. |
Prathmesh Prabhu | a62f92d | 2017-03-16 14:30:05 -0700 | [diff] [blame] | 32 | SERVICE_STABILITY_TIMER = 60 |
Don Garrett | d032172 | 2014-11-18 16:03:33 -0800 | [diff] [blame] | 33 | |
Shuqian Zhao | a3438a5 | 2016-09-20 15:11:02 -0700 | [diff] [blame] | 34 | # A dict to map update_commands defined in config file to repos or files that |
| 35 | # decide whether need to update these commands. E.g. if no changes under |
| 36 | # frontend repo, no need to update afe. |
Richard Barnette | 1b72e7f | 2018-02-21 15:28:48 -0800 | [diff] [blame] | 37 | COMMANDS_TO_REPOS_DICT = {'afe': 'frontend/client/', |
| 38 | 'tko': 'frontend/client/'} |
Shuqian Zhao | 697f7ed | 2016-10-21 14:58:28 -0700 | [diff] [blame] | 39 | BUILD_EXTERNALS_COMMAND = 'build_externals' |
Allen Li | a2749cd | 2017-10-31 18:03:19 -0700 | [diff] [blame] | 40 | |
| 41 | _RESTART_SERVICES_FILE = os.path.join(os.environ['HOME'], |
| 42 | 'push_restart_services') |
Dan Shi | cf27804 | 2016-04-06 21:16:34 -0700 | [diff] [blame] | 43 | |
Dan Shi | ac6fdbf | 2016-04-09 17:36:28 -0700 | [diff] [blame] | 44 | AFE = frontend_wrappers.RetryingAFE( |
| 45 | server=server_utils.get_global_afe_hostname(), timeout_min=5, |
| 46 | delay_sec=10) |
Shuqian Zhao | 79f248d | 2018-05-10 14:47:55 -0700 | [diff] [blame] | 47 | HOSTNAME = socket.gethostname() |
Don Garrett | 8db752c | 2014-10-17 16:56:55 -0700 | [diff] [blame] | 48 | |
| 49 | class DirtyTreeException(Exception): |
Don Garrett | d032172 | 2014-11-18 16:03:33 -0800 | [diff] [blame] | 50 | """Raised when the tree has been modified in an unexpected way.""" |
Don Garrett | 8db752c | 2014-10-17 16:56:55 -0700 | [diff] [blame] | 51 | |
| 52 | |
| 53 | class UnknownCommandException(Exception): |
Don Garrett | d032172 | 2014-11-18 16:03:33 -0800 | [diff] [blame] | 54 | """Raised when we try to run a command name with no associated command.""" |
Don Garrett | 8db752c | 2014-10-17 16:56:55 -0700 | [diff] [blame] | 55 | |
| 56 | |
| 57 | class UnstableServices(Exception): |
Don Garrett | d032172 | 2014-11-18 16:03:33 -0800 | [diff] [blame] | 58 | """Raised if a service appears unstable after restart.""" |
Don Garrett | 8db752c | 2014-10-17 16:56:55 -0700 | [diff] [blame] | 59 | |
| 60 | |
Don Garrett | 3571121 | 2014-12-18 14:33:41 -0800 | [diff] [blame] | 61 | def strip_terminal_codes(text): |
| 62 | """This function removes all terminal formatting codes from a string. |
| 63 | |
| 64 | @param text: String of text to cleanup. |
| 65 | @returns String with format codes removed. |
| 66 | """ |
| 67 | ESC = '\x1b' |
| 68 | return re.sub(ESC+r'\[[^m]*m', '', text) |
| 69 | |
| 70 | |
Richard Barnette | 488ac8b | 2017-09-01 09:28:55 -0700 | [diff] [blame] | 71 | def _clean_pyc_files(): |
| 72 | print('Removing .pyc files') |
| 73 | try: |
| 74 | subprocess.check_output([ |
| 75 | 'find', '.', |
| 76 | '(', |
| 77 | # These are ignored to reduce IO load (crbug.com/759780). |
| 78 | '-path', './site-packages', |
| 79 | '-o', '-path', './containers', |
| 80 | '-o', '-path', './logs', |
| 81 | '-o', '-path', './results', |
| 82 | ')', |
| 83 | '-prune', |
| 84 | '-o', '-name', '*.pyc', |
| 85 | '-exec', 'rm', '-f', '{}', '+']) |
| 86 | except Exception as e: |
| 87 | print('Warning: fail to remove .pyc! %s' % e) |
| 88 | |
| 89 | |
Don Garrett | 8db752c | 2014-10-17 16:56:55 -0700 | [diff] [blame] | 90 | def verify_repo_clean(): |
Shuqian Zhao | 8754a1a | 2016-08-24 12:54:11 -0700 | [diff] [blame] | 91 | """This function cleans the current repo then verifies that it is valid. |
Don Garrett | 8db752c | 2014-10-17 16:56:55 -0700 | [diff] [blame] | 92 | |
Shuqian Zhao | 8754a1a | 2016-08-24 12:54:11 -0700 | [diff] [blame] | 93 | @raises DirtyTreeException if the repo is still not clean. |
Don Garrett | 8db752c | 2014-10-17 16:56:55 -0700 | [diff] [blame] | 94 | @raises subprocess.CalledProcessError on a repo command failure. |
| 95 | """ |
Shuqian Zhao | 0647a21 | 2018-03-05 17:53:47 -0800 | [diff] [blame] | 96 | subprocess.check_output(['git', 'stash', '-u']) |
| 97 | subprocess.check_output(['git', 'stash', 'clear']) |
Don Garrett | 8db752c | 2014-10-17 16:56:55 -0700 | [diff] [blame] | 98 | out = subprocess.check_output(['repo', 'status'], stderr=subprocess.STDOUT) |
Don Garrett | 3571121 | 2014-12-18 14:33:41 -0800 | [diff] [blame] | 99 | out = strip_terminal_codes(out).strip() |
Don Garrett | 699b4b3 | 2014-12-11 13:10:15 -0800 | [diff] [blame] | 100 | |
Shuqian Zhao | da985b6 | 2018-03-19 11:28:50 -0700 | [diff] [blame] | 101 | if not 'working directory clean' in out and not 'working tree clean' in out: |
Shuqian Zhao | 79f248d | 2018-05-10 14:47:55 -0700 | [diff] [blame] | 102 | raise DirtyTreeException('%s repo not clean: %s' % (HOSTNAME, out)) |
Don Garrett | 8db752c | 2014-10-17 16:56:55 -0700 | [diff] [blame] | 103 | |
Don Garrett | 8db752c | 2014-10-17 16:56:55 -0700 | [diff] [blame] | 104 | |
Aviv Keshet | ccbf643 | 2017-10-19 12:03:34 -0700 | [diff] [blame] | 105 | def _clean_externals(): |
| 106 | """Clean untracked files within ExternalSource and site-packages/ |
| 107 | |
| 108 | @raises subprocess.CalledProcessError on a git command failure. |
| 109 | """ |
| 110 | dirs_to_clean = ['site-packages/', 'ExternalSource/'] |
| 111 | cmd = ['git', 'clean', '-fxd'] + dirs_to_clean |
| 112 | subprocess.check_output(cmd) |
| 113 | |
| 114 | |
Don Garrett | 8db752c | 2014-10-17 16:56:55 -0700 | [diff] [blame] | 115 | def repo_versions(): |
| 116 | """This function collects the versions of all git repos in the general repo. |
| 117 | |
Don Garrett | fa2c1c4 | 2014-12-11 12:11:49 -0800 | [diff] [blame] | 118 | @returns A dictionary mapping project names to git hashes for HEAD. |
Don Garrett | 8db752c | 2014-10-17 16:56:55 -0700 | [diff] [blame] | 119 | @raises subprocess.CalledProcessError on a repo command failure. |
| 120 | """ |
Don Garrett | fa2c1c4 | 2014-12-11 12:11:49 -0800 | [diff] [blame] | 121 | cmd = ['repo', 'forall', '-p', '-c', 'pwd && git log -1 --format=%h'] |
Don Garrett | 3571121 | 2014-12-18 14:33:41 -0800 | [diff] [blame] | 122 | output = strip_terminal_codes(subprocess.check_output(cmd)) |
Don Garrett | fa2c1c4 | 2014-12-11 12:11:49 -0800 | [diff] [blame] | 123 | |
| 124 | # The expected output format is: |
| 125 | |
| 126 | # project chrome_build/ |
| 127 | # /dir/holding/chrome_build |
| 128 | # 73dee9d |
| 129 | # |
| 130 | # project chrome_release/ |
| 131 | # /dir/holding/chrome_release |
| 132 | # 9f3a5d8 |
| 133 | |
| 134 | lines = output.splitlines() |
| 135 | |
| 136 | PROJECT_PREFIX = 'project ' |
| 137 | |
| 138 | project_heads = {} |
| 139 | for n in range(0, len(lines), 4): |
| 140 | project_line = lines[n] |
| 141 | project_dir = lines[n+1] |
| 142 | project_hash = lines[n+2] |
| 143 | # lines[n+3] is a blank line, but doesn't exist for the final block. |
| 144 | |
| 145 | # Convert 'project chrome_build/' -> 'chrome_build' |
| 146 | assert project_line.startswith(PROJECT_PREFIX) |
| 147 | name = project_line[len(PROJECT_PREFIX):].rstrip('/') |
| 148 | |
| 149 | project_heads[name] = (project_dir, project_hash) |
| 150 | |
| 151 | return project_heads |
Don Garrett | 8db752c | 2014-10-17 16:56:55 -0700 | [diff] [blame] | 152 | |
| 153 | |
Shuqian Zhao | a3438a5 | 2016-09-20 15:11:02 -0700 | [diff] [blame] | 154 | def repo_versions_to_decide_whether_run_cmd_update(): |
| 155 | """Collect versions of repos/files defined in COMMANDS_TO_REPOS_DICT. |
| 156 | |
| 157 | For the update_commands defined in config files, no need to run the command |
| 158 | every time. Only run it when the repos/files related to the commands have |
| 159 | been changed. |
| 160 | |
| 161 | @returns A set of tuples: {(cmd, repo_version), ()...} |
| 162 | """ |
| 163 | results = set() |
| 164 | for cmd, repo in COMMANDS_TO_REPOS_DICT.iteritems(): |
| 165 | version = subprocess.check_output( |
| 166 | ['git', 'log', '-1', '--pretty=tformat:%h', |
| 167 | '%s/%s' % (common.autotest_dir, repo)]) |
| 168 | results.add((cmd, version.strip())) |
| 169 | return results |
| 170 | |
| 171 | |
Shuqian Zhao | 8754a1a | 2016-08-24 12:54:11 -0700 | [diff] [blame] | 172 | def repo_sync(update_push_servers=False): |
Don Garrett | 8db752c | 2014-10-17 16:56:55 -0700 | [diff] [blame] | 173 | """Perform a repo sync. |
| 174 | |
Shuqian Zhao | 8754a1a | 2016-08-24 12:54:11 -0700 | [diff] [blame] | 175 | @param update_push_servers: If True, then update test_push servers to ToT. |
| 176 | Otherwise, update server to prod branch. |
Don Garrett | 8db752c | 2014-10-17 16:56:55 -0700 | [diff] [blame] | 177 | @raises subprocess.CalledProcessError on a repo command failure. |
| 178 | """ |
Prathmesh Prabhu | d77bf0d | 2018-06-12 09:57:04 -0700 | [diff] [blame] | 179 | subprocess.check_output(['repo', 'sync', '--force-sync']) |
Shuqian Zhao | 8754a1a | 2016-08-24 12:54:11 -0700 | [diff] [blame] | 180 | if update_push_servers: |
| 181 | print('Updating push servers, checkout cros/master') |
Shuqian Zhao | 9febd45 | 2017-01-31 15:36:40 -0800 | [diff] [blame] | 182 | subprocess.check_output(['git', 'checkout', 'cros/master'], |
| 183 | stderr=subprocess.STDOUT) |
Shuqian Zhao | 8754a1a | 2016-08-24 12:54:11 -0700 | [diff] [blame] | 184 | else: |
| 185 | print('Updating server to prod branch') |
Shuqian Zhao | 9febd45 | 2017-01-31 15:36:40 -0800 | [diff] [blame] | 186 | subprocess.check_output(['git', 'checkout', 'cros/prod'], |
| 187 | stderr=subprocess.STDOUT) |
Richard Barnette | 488ac8b | 2017-09-01 09:28:55 -0700 | [diff] [blame] | 188 | _clean_pyc_files() |
| 189 | |
Don Garrett | 8db752c | 2014-10-17 16:56:55 -0700 | [diff] [blame] | 190 | |
Don Garrett | d032172 | 2014-11-18 16:03:33 -0800 | [diff] [blame] | 191 | def discover_update_commands(): |
| 192 | """Lookup the commands to run on this server. |
Don Garrett | 8db752c | 2014-10-17 16:56:55 -0700 | [diff] [blame] | 193 | |
Don Garrett | d032172 | 2014-11-18 16:03:33 -0800 | [diff] [blame] | 194 | These commonly come from shadow_config.ini, since they vary by server type. |
Don Garrett | 8db752c | 2014-10-17 16:56:55 -0700 | [diff] [blame] | 195 | |
Don Garrett | d032172 | 2014-11-18 16:03:33 -0800 | [diff] [blame] | 196 | @returns List of command names in string format. |
Don Garrett | 8db752c | 2014-10-17 16:56:55 -0700 | [diff] [blame] | 197 | """ |
Don Garrett | 8db752c | 2014-10-17 16:56:55 -0700 | [diff] [blame] | 198 | try: |
Don Garrett | d032172 | 2014-11-18 16:03:33 -0800 | [diff] [blame] | 199 | return global_config.global_config.get_config_value( |
Don Garrett | 8db752c | 2014-10-17 16:56:55 -0700 | [diff] [blame] | 200 | 'UPDATE', 'commands', type=list) |
| 201 | |
| 202 | except (ConfigParser.NoSectionError, global_config.ConfigError): |
Don Garrett | d032172 | 2014-11-18 16:03:33 -0800 | [diff] [blame] | 203 | return [] |
Don Garrett | 8db752c | 2014-10-17 16:56:55 -0700 | [diff] [blame] | 204 | |
Don Garrett | d032172 | 2014-11-18 16:03:33 -0800 | [diff] [blame] | 205 | |
Allen Li | a2749cd | 2017-10-31 18:03:19 -0700 | [diff] [blame] | 206 | def get_restart_services(): |
Don Garrett | d032172 | 2014-11-18 16:03:33 -0800 | [diff] [blame] | 207 | """Find the services that need restarting on the current server. |
| 208 | |
| 209 | These commonly come from shadow_config.ini, since they vary by server type. |
| 210 | |
Allen Li | a2749cd | 2017-10-31 18:03:19 -0700 | [diff] [blame] | 211 | @returns Iterable of service names in string format. |
Don Garrett | d032172 | 2014-11-18 16:03:33 -0800 | [diff] [blame] | 212 | """ |
Allen Li | a2749cd | 2017-10-31 18:03:19 -0700 | [diff] [blame] | 213 | with open(_RESTART_SERVICES_FILE) as f: |
| 214 | for line in f: |
| 215 | yield line.rstrip() |
Don Garrett | 8db752c | 2014-10-17 16:56:55 -0700 | [diff] [blame] | 216 | |
Don Garrett | 8db752c | 2014-10-17 16:56:55 -0700 | [diff] [blame] | 217 | |
Shuqian Zhao | 697f7ed | 2016-10-21 14:58:28 -0700 | [diff] [blame] | 218 | def update_command(cmd_tag, dryrun=False, use_chromite_master=False): |
Don Garrett | d032172 | 2014-11-18 16:03:33 -0800 | [diff] [blame] | 219 | """Restart a command. |
Don Garrett | 8db752c | 2014-10-17 16:56:55 -0700 | [diff] [blame] | 220 | |
Don Garrett | d032172 | 2014-11-18 16:03:33 -0800 | [diff] [blame] | 221 | The command name is looked up in global_config.ini to find the full command |
| 222 | to run, then it's executed. |
Don Garrett | 8db752c | 2014-10-17 16:56:55 -0700 | [diff] [blame] | 223 | |
Don Garrett | d032172 | 2014-11-18 16:03:33 -0800 | [diff] [blame] | 224 | @param cmd_tag: Which command to restart. |
Don Garrett | 03432d6 | 2014-11-19 18:18:35 -0800 | [diff] [blame] | 225 | @param dryrun: If true print the command that would have been run. |
Shuqian Zhao | 697f7ed | 2016-10-21 14:58:28 -0700 | [diff] [blame] | 226 | @param use_chromite_master: True if updating chromite to master, rather |
| 227 | than prod. |
Don Garrett | 8db752c | 2014-10-17 16:56:55 -0700 | [diff] [blame] | 228 | |
Don Garrett | d032172 | 2014-11-18 16:03:33 -0800 | [diff] [blame] | 229 | @raises UnknownCommandException If cmd_tag can't be looked up. |
| 230 | @raises subprocess.CalledProcessError on a command failure. |
| 231 | """ |
| 232 | # Lookup the list of commands to consider. They are intended to be |
| 233 | # in global_config.ini so that they can be shared everywhere. |
| 234 | cmds = dict(global_config.global_config.config.items( |
| 235 | 'UPDATE_COMMANDS')) |
Don Garrett | 8db752c | 2014-10-17 16:56:55 -0700 | [diff] [blame] | 236 | |
Don Garrett | d032172 | 2014-11-18 16:03:33 -0800 | [diff] [blame] | 237 | if cmd_tag not in cmds: |
| 238 | raise UnknownCommandException(cmd_tag, cmds) |
Don Garrett | 8db752c | 2014-10-17 16:56:55 -0700 | [diff] [blame] | 239 | |
Richard Barnette | 2e0a423 | 2018-03-07 17:13:16 -0800 | [diff] [blame] | 240 | command = cmds[cmd_tag] |
Shuqian Zhao | 697f7ed | 2016-10-21 14:58:28 -0700 | [diff] [blame] | 241 | # When updating push servers, pass an arg to build_externals to update |
| 242 | # chromite to master branch for testing |
| 243 | if use_chromite_master and cmd_tag == BUILD_EXTERNALS_COMMAND: |
Richard Barnette | 2e0a423 | 2018-03-07 17:13:16 -0800 | [diff] [blame] | 244 | command += ' --use_chromite_master' |
Don Garrett | 8db752c | 2014-10-17 16:56:55 -0700 | [diff] [blame] | 245 | |
Richard Barnette | 2e0a423 | 2018-03-07 17:13:16 -0800 | [diff] [blame] | 246 | print('Running: %s: %s' % (cmd_tag, command)) |
Don Garrett | 03432d6 | 2014-11-19 18:18:35 -0800 | [diff] [blame] | 247 | if dryrun: |
Richard Barnette | 2e0a423 | 2018-03-07 17:13:16 -0800 | [diff] [blame] | 248 | print('Skip: %s' % command) |
Don Garrett | 03432d6 | 2014-11-19 18:18:35 -0800 | [diff] [blame] | 249 | else: |
Don Garrett | 4769c90 | 2015-01-05 15:58:56 -0800 | [diff] [blame] | 250 | try: |
Richard Barnette | 2e0a423 | 2018-03-07 17:13:16 -0800 | [diff] [blame] | 251 | subprocess.check_output(command, shell=True, |
| 252 | cwd=common.autotest_dir, |
Don Garrett | 4769c90 | 2015-01-05 15:58:56 -0800 | [diff] [blame] | 253 | stderr=subprocess.STDOUT) |
| 254 | except subprocess.CalledProcessError as e: |
Shuqian Zhao | 79f248d | 2018-05-10 14:47:55 -0700 | [diff] [blame] | 255 | print('FAILED %s :' % HOSTNAME) |
Don Garrett | 4769c90 | 2015-01-05 15:58:56 -0800 | [diff] [blame] | 256 | print(e.output) |
| 257 | raise |
Don Garrett | 8db752c | 2014-10-17 16:56:55 -0700 | [diff] [blame] | 258 | |
Don Garrett | 8db752c | 2014-10-17 16:56:55 -0700 | [diff] [blame] | 259 | |
Don Garrett | 03432d6 | 2014-11-19 18:18:35 -0800 | [diff] [blame] | 260 | def restart_service(service_name, dryrun=False): |
Don Garrett | d032172 | 2014-11-18 16:03:33 -0800 | [diff] [blame] | 261 | """Restart a service. |
| 262 | |
| 263 | Restarts the standard service with "service <name> restart". |
| 264 | |
| 265 | @param service_name: The name of the service to restart. |
Don Garrett | 03432d6 | 2014-11-19 18:18:35 -0800 | [diff] [blame] | 266 | @param dryrun: Don't really run anything, just print out the command. |
Don Garrett | d032172 | 2014-11-18 16:03:33 -0800 | [diff] [blame] | 267 | |
| 268 | @raises subprocess.CalledProcessError on a command failure. |
| 269 | """ |
Don Garrett | 03432d6 | 2014-11-19 18:18:35 -0800 | [diff] [blame] | 270 | cmd = ['sudo', 'service', service_name, 'restart'] |
Don Garrett | 699b4b3 | 2014-12-11 13:10:15 -0800 | [diff] [blame] | 271 | print('Restarting: %s' % service_name) |
Don Garrett | 03432d6 | 2014-11-19 18:18:35 -0800 | [diff] [blame] | 272 | if dryrun: |
Don Garrett | 699b4b3 | 2014-12-11 13:10:15 -0800 | [diff] [blame] | 273 | print('Skip: %s' % ' '.join(cmd)) |
Don Garrett | 03432d6 | 2014-11-19 18:18:35 -0800 | [diff] [blame] | 274 | else: |
Shuqian Zhao | 9febd45 | 2017-01-31 15:36:40 -0800 | [diff] [blame] | 275 | subprocess.check_call(cmd, stderr=subprocess.STDOUT) |
Don Garrett | d032172 | 2014-11-18 16:03:33 -0800 | [diff] [blame] | 276 | |
| 277 | |
| 278 | def service_status(service_name): |
| 279 | """Return the results "status <name>" for a given service. |
| 280 | |
| 281 | This string is expected to contain the pid, and so to change is the service |
| 282 | is shutdown or restarted for any reason. |
| 283 | |
| 284 | @param service_name: The name of the service to check on. |
Don Garrett | 03432d6 | 2014-11-19 18:18:35 -0800 | [diff] [blame] | 285 | |
Don Garrett | d032172 | 2014-11-18 16:03:33 -0800 | [diff] [blame] | 286 | @returns The output of the external command. |
| 287 | Ex: autofs start/running, process 1931 |
| 288 | |
| 289 | @raises subprocess.CalledProcessError on a command failure. |
| 290 | """ |
Shuqian Zhao | dcaa616 | 2017-11-09 10:58:45 -0800 | [diff] [blame] | 291 | return subprocess.check_output(['sudo', 'service', service_name, 'status']) |
Don Garrett | d032172 | 2014-11-18 16:03:33 -0800 | [diff] [blame] | 292 | |
| 293 | |
Dan Shi | 57d4c73 | 2015-01-22 18:38:50 -0800 | [diff] [blame] | 294 | def restart_services(service_names, dryrun=False, skip_service_status=False): |
Don Garrett | d032172 | 2014-11-18 16:03:33 -0800 | [diff] [blame] | 295 | """Restart services as needed for the current server type. |
| 296 | |
| 297 | Restart the listed set of services, and watch to see if they are stable for |
| 298 | at least SERVICE_STABILITY_TIMER. It restarts all services quickly, |
| 299 | waits for that delay, then verifies the status of all of them. |
| 300 | |
| 301 | @param service_names: The list of service to restart and monitor. |
Don Garrett | 03432d6 | 2014-11-19 18:18:35 -0800 | [diff] [blame] | 302 | @param dryrun: Don't really restart the service, just print out the command. |
Dan Shi | 57d4c73 | 2015-01-22 18:38:50 -0800 | [diff] [blame] | 303 | @param skip_service_status: Set to True to skip service status check. |
| 304 | Default is False. |
Don Garrett | d032172 | 2014-11-18 16:03:33 -0800 | [diff] [blame] | 305 | |
| 306 | @raises subprocess.CalledProcessError on a command failure. |
Don Garrett | 03432d6 | 2014-11-19 18:18:35 -0800 | [diff] [blame] | 307 | @raises UnstableServices if any services are unstable after restart. |
Don Garrett | d032172 | 2014-11-18 16:03:33 -0800 | [diff] [blame] | 308 | """ |
| 309 | service_statuses = {} |
| 310 | |
Don Garrett | 03432d6 | 2014-11-19 18:18:35 -0800 | [diff] [blame] | 311 | if dryrun: |
| 312 | for name in service_names: |
| 313 | restart_service(name, dryrun=True) |
| 314 | return |
| 315 | |
Don Garrett | d032172 | 2014-11-18 16:03:33 -0800 | [diff] [blame] | 316 | # Restart each, and record the status (including pid). |
| 317 | for name in service_names: |
| 318 | restart_service(name) |
Don Garrett | d032172 | 2014-11-18 16:03:33 -0800 | [diff] [blame] | 319 | |
Dan Shi | 57d4c73 | 2015-01-22 18:38:50 -0800 | [diff] [blame] | 320 | # Skip service status check if --skip-service-status is specified. Used for |
| 321 | # servers in backup status. |
| 322 | if skip_service_status: |
| 323 | print('--skip-service-status is specified, skip checking services.') |
| 324 | return |
| 325 | |
Don Garrett | d032172 | 2014-11-18 16:03:33 -0800 | [diff] [blame] | 326 | # Wait for a while to let the services settle. |
| 327 | time.sleep(SERVICE_STABILITY_TIMER) |
Prathmesh Prabhu | a62f92d | 2017-03-16 14:30:05 -0700 | [diff] [blame] | 328 | service_statuses = {name: service_status(name) for name in service_names} |
| 329 | time.sleep(SERVICE_STABILITY_TIMER) |
Don Garrett | d032172 | 2014-11-18 16:03:33 -0800 | [diff] [blame] | 330 | # Look for any services that changed status. |
| 331 | unstable_services = [n for n in service_names |
| 332 | if service_status(n) != service_statuses[n]] |
| 333 | |
| 334 | # Report any services having issues. |
| 335 | if unstable_services: |
Shuqian Zhao | 79f248d | 2018-05-10 14:47:55 -0700 | [diff] [blame] | 336 | raise UnstableServices('%s service restart failed: %s' % |
| 337 | (HOSTNAME, unstable_services)) |
Don Garrett | 8db752c | 2014-10-17 16:56:55 -0700 | [diff] [blame] | 338 | |
| 339 | |
Shuqian Zhao | a3438a5 | 2016-09-20 15:11:02 -0700 | [diff] [blame] | 340 | def run_deploy_actions(cmds_skip=set(), dryrun=False, |
Shuqian Zhao | 697f7ed | 2016-10-21 14:58:28 -0700 | [diff] [blame] | 341 | skip_service_status=False, use_chromite_master=False): |
Don Garrett | fa2c1c4 | 2014-12-11 12:11:49 -0800 | [diff] [blame] | 342 | """Run arbitrary update commands specified in global.ini. |
Don Garrett | 8db752c | 2014-10-17 16:56:55 -0700 | [diff] [blame] | 343 | |
Shuqian Zhao | a3438a5 | 2016-09-20 15:11:02 -0700 | [diff] [blame] | 344 | @param cmds_skip: cmds no need to run since the corresponding repo/file |
| 345 | does not change. |
Don Garrett | 03432d6 | 2014-11-19 18:18:35 -0800 | [diff] [blame] | 346 | @param dryrun: Don't really restart the service, just print out the command. |
Dan Shi | 57d4c73 | 2015-01-22 18:38:50 -0800 | [diff] [blame] | 347 | @param skip_service_status: Set to True to skip service status check. |
| 348 | Default is False. |
Shuqian Zhao | 697f7ed | 2016-10-21 14:58:28 -0700 | [diff] [blame] | 349 | @param use_chromite_master: True if updating chromite to master, rather |
| 350 | than prod. |
Don Garrett | 8db752c | 2014-10-17 16:56:55 -0700 | [diff] [blame] | 351 | |
Don Garrett | 03432d6 | 2014-11-19 18:18:35 -0800 | [diff] [blame] | 352 | @raises subprocess.CalledProcessError on a command failure. |
| 353 | @raises UnstableServices if any services are unstable after restart. |
| 354 | """ |
Shuqian Zhao | a3438a5 | 2016-09-20 15:11:02 -0700 | [diff] [blame] | 355 | defined_cmds = set(discover_update_commands()) |
| 356 | cmds = defined_cmds - cmds_skip |
Don Garrett | d032172 | 2014-11-18 16:03:33 -0800 | [diff] [blame] | 357 | if cmds: |
| 358 | print('Running update commands:', ', '.join(cmds)) |
| 359 | for cmd in cmds: |
Shuqian Zhao | 697f7ed | 2016-10-21 14:58:28 -0700 | [diff] [blame] | 360 | update_command(cmd, dryrun=dryrun, |
| 361 | use_chromite_master=use_chromite_master) |
Don Garrett | d032172 | 2014-11-18 16:03:33 -0800 | [diff] [blame] | 362 | |
Allen Li | a2749cd | 2017-10-31 18:03:19 -0700 | [diff] [blame] | 363 | services = list(get_restart_services()) |
Don Garrett | d032172 | 2014-11-18 16:03:33 -0800 | [diff] [blame] | 364 | if services: |
Don Garrett | 03432d6 | 2014-11-19 18:18:35 -0800 | [diff] [blame] | 365 | print('Restarting Services:', ', '.join(services)) |
Dan Shi | 57d4c73 | 2015-01-22 18:38:50 -0800 | [diff] [blame] | 366 | restart_services(services, dryrun=dryrun, |
| 367 | skip_service_status=skip_service_status) |
Don Garrett | 03432d6 | 2014-11-19 18:18:35 -0800 | [diff] [blame] | 368 | |
| 369 | |
Don Garrett | fa2c1c4 | 2014-12-11 12:11:49 -0800 | [diff] [blame] | 370 | def report_changes(versions_before, versions_after): |
| 371 | """Produce a report describing what changed in all repos. |
| 372 | |
| 373 | @param versions_before: Results of repo_versions() from before the update. |
| 374 | @param versions_after: Results of repo_versions() from after the update. |
| 375 | |
| 376 | @returns string containing a human friendly changes report. |
| 377 | """ |
| 378 | result = [] |
| 379 | |
Don Garrett | 3571121 | 2014-12-18 14:33:41 -0800 | [diff] [blame] | 380 | if versions_after: |
| 381 | for project in sorted(set(versions_before.keys() + versions_after.keys())): |
| 382 | result.append('%s:' % project) |
Don Garrett | fa2c1c4 | 2014-12-11 12:11:49 -0800 | [diff] [blame] | 383 | |
Don Garrett | 3571121 | 2014-12-18 14:33:41 -0800 | [diff] [blame] | 384 | _, before_hash = versions_before.get(project, (None, None)) |
| 385 | after_dir, after_hash = versions_after.get(project, (None, None)) |
Don Garrett | fa2c1c4 | 2014-12-11 12:11:49 -0800 | [diff] [blame] | 386 | |
Don Garrett | 3571121 | 2014-12-18 14:33:41 -0800 | [diff] [blame] | 387 | if project not in versions_before: |
| 388 | result.append('Added.') |
Don Garrett | fa2c1c4 | 2014-12-11 12:11:49 -0800 | [diff] [blame] | 389 | |
Don Garrett | 3571121 | 2014-12-18 14:33:41 -0800 | [diff] [blame] | 390 | elif project not in versions_after: |
| 391 | result.append('Removed.') |
Don Garrett | fa2c1c4 | 2014-12-11 12:11:49 -0800 | [diff] [blame] | 392 | |
Don Garrett | 3571121 | 2014-12-18 14:33:41 -0800 | [diff] [blame] | 393 | elif before_hash == after_hash: |
| 394 | result.append('No Change.') |
Don Garrett | fa2c1c4 | 2014-12-11 12:11:49 -0800 | [diff] [blame] | 395 | |
Don Garrett | 3571121 | 2014-12-18 14:33:41 -0800 | [diff] [blame] | 396 | else: |
| 397 | hashes = '%s..%s' % (before_hash, after_hash) |
| 398 | cmd = ['git', 'log', hashes, '--oneline'] |
| 399 | out = subprocess.check_output(cmd, cwd=after_dir, |
| 400 | stderr=subprocess.STDOUT) |
| 401 | result.append(out.strip()) |
Don Garrett | fa2c1c4 | 2014-12-11 12:11:49 -0800 | [diff] [blame] | 402 | |
Don Garrett | 3571121 | 2014-12-18 14:33:41 -0800 | [diff] [blame] | 403 | result.append('') |
| 404 | else: |
| 405 | for project in sorted(versions_before.keys()): |
| 406 | _, before_hash = versions_before[project] |
| 407 | result.append('%s: %s' % (project, before_hash)) |
Don Garrett | fa2c1c4 | 2014-12-11 12:11:49 -0800 | [diff] [blame] | 408 | result.append('') |
| 409 | |
| 410 | return '\n'.join(result) |
| 411 | |
| 412 | |
Don Garrett | 03432d6 | 2014-11-19 18:18:35 -0800 | [diff] [blame] | 413 | def parse_arguments(args): |
| 414 | """Parse command line arguments. |
| 415 | |
| 416 | @param args: The command line arguments to parse. (ususally sys.argsv[1:]) |
| 417 | |
Don Garrett | 4003636 | 2014-12-08 15:52:44 -0800 | [diff] [blame] | 418 | @returns An argparse.Namespace populated with argument values. |
Don Garrett | 03432d6 | 2014-11-19 18:18:35 -0800 | [diff] [blame] | 419 | """ |
| 420 | parser = argparse.ArgumentParser( |
| 421 | description='Command to update an autotest server.') |
| 422 | parser.add_argument('--skip-verify', action='store_false', |
| 423 | dest='verify', default=True, |
| 424 | help='Disable verification of a clean repository.') |
| 425 | parser.add_argument('--skip-update', action='store_false', |
| 426 | dest='update', default=True, |
| 427 | help='Skip the repository source code update.') |
| 428 | parser.add_argument('--skip-actions', action='store_false', |
| 429 | dest='actions', default=True, |
| 430 | help='Skip the post update actions.') |
| 431 | parser.add_argument('--skip-report', action='store_false', |
| 432 | dest='report', default=True, |
| 433 | help='Skip the git version report.') |
Don Garrett | e371891 | 2014-12-05 13:11:44 -0800 | [diff] [blame] | 434 | parser.add_argument('--actions-only', action='store_true', |
| 435 | help='Run the post update actions (restart services).') |
Don Garrett | 03432d6 | 2014-11-19 18:18:35 -0800 | [diff] [blame] | 436 | parser.add_argument('--dryrun', action='store_true', |
| 437 | help='Don\'t actually run any commands, just log.') |
Dan Shi | 57d4c73 | 2015-01-22 18:38:50 -0800 | [diff] [blame] | 438 | parser.add_argument('--skip-service-status', action='store_true', |
| 439 | help='Skip checking the service status.') |
Shuqian Zhao | 8754a1a | 2016-08-24 12:54:11 -0700 | [diff] [blame] | 440 | parser.add_argument('--update_push_servers', action='store_true', |
| 441 | help='Indicate to update test_push server. If not ' |
| 442 | 'specify, then update server to production.') |
Aviv Keshet | ccbf643 | 2017-10-19 12:03:34 -0700 | [diff] [blame] | 443 | parser.add_argument('--force-clean-externals', action='store_true', |
| 444 | default=False, |
| 445 | help='Force a cleanup of all untracked files within ' |
| 446 | 'site-packages/ and ExternalSource/, so that ' |
| 447 | 'build_externals will build from scratch.') |
Shuqian Zhao | a3438a5 | 2016-09-20 15:11:02 -0700 | [diff] [blame] | 448 | parser.add_argument('--force_update', action='store_true', |
| 449 | help='Force to run the update commands for afe, tko ' |
| 450 | 'and build_externals') |
Don Garrett | 03432d6 | 2014-11-19 18:18:35 -0800 | [diff] [blame] | 451 | |
| 452 | results = parser.parse_args(args) |
| 453 | |
Don Garrett | e371891 | 2014-12-05 13:11:44 -0800 | [diff] [blame] | 454 | if results.actions_only: |
| 455 | results.verify = False |
| 456 | results.update = False |
| 457 | results.report = False |
| 458 | |
Don Garrett | 03432d6 | 2014-11-19 18:18:35 -0800 | [diff] [blame] | 459 | # TODO(dgarrett): Make these behaviors support dryrun. |
| 460 | if results.dryrun: |
| 461 | results.verify = False |
| 462 | results.update = False |
Aviv Keshet | ccbf643 | 2017-10-19 12:03:34 -0700 | [diff] [blame] | 463 | results.force_clean_externals = False |
Don Garrett | 03432d6 | 2014-11-19 18:18:35 -0800 | [diff] [blame] | 464 | |
Shuqian Zhao | 87a4d6a | 2017-12-04 17:36:28 -0800 | [diff] [blame] | 465 | if not results.update_push_servers: |
| 466 | print('Will skip service check for pushing servers in prod.') |
| 467 | results.skip_service_status = True |
Don Garrett | 03432d6 | 2014-11-19 18:18:35 -0800 | [diff] [blame] | 468 | return results |
| 469 | |
| 470 | |
Allen Li | e8c4ea4 | 2016-10-05 18:08:29 -0700 | [diff] [blame] | 471 | class ChangeDir(object): |
| 472 | |
| 473 | """Context manager for changing to a directory temporarily.""" |
| 474 | |
| 475 | def __init__(self, dir): |
| 476 | self.new_dir = dir |
| 477 | self.old_dir = None |
| 478 | |
| 479 | def __enter__(self): |
| 480 | self.old_dir = os.getcwd() |
| 481 | os.chdir(self.new_dir) |
| 482 | |
| 483 | def __exit__(self, exc_type, exc_val, exc_tb): |
| 484 | os.chdir(self.old_dir) |
| 485 | |
| 486 | |
Allen Li | 8802a5e | 2017-01-17 15:04:15 -0800 | [diff] [blame] | 487 | def _sync_chromiumos_repo(): |
| 488 | """Update ~chromeos-test/chromiumos repo.""" |
| 489 | print('Updating ~chromeos-test/chromiumos') |
Allen Li | 19c48eb | 2017-01-25 15:03:10 -0800 | [diff] [blame] | 490 | with ChangeDir(os.path.expanduser('~chromeos-test/chromiumos')): |
Prathmesh Prabhu | d77bf0d | 2018-06-12 09:57:04 -0700 | [diff] [blame] | 491 | ret = subprocess.call(['repo', 'sync', '--force-sync'], |
| 492 | stderr=subprocess.STDOUT) |
Richard Barnette | 488ac8b | 2017-09-01 09:28:55 -0700 | [diff] [blame] | 493 | _clean_pyc_files() |
Allen Li | e8c4ea4 | 2016-10-05 18:08:29 -0700 | [diff] [blame] | 494 | if ret != 0: |
| 495 | print('Update failed, exited with status: %d' % ret) |
| 496 | |
| 497 | |
Don Garrett | 03432d6 | 2014-11-19 18:18:35 -0800 | [diff] [blame] | 498 | def main(args): |
| 499 | """Main method.""" |
Richard Barnette | 2e0a423 | 2018-03-07 17:13:16 -0800 | [diff] [blame] | 500 | # Be careful before you change this call to `os.chdir()`: |
| 501 | # We make several calls to `subprocess.check_output()` and |
| 502 | # friends that depend on this directory, most notably calls to |
| 503 | # the 'repo' command from `verify_repo_clean()`. |
Don Garrett | 03432d6 | 2014-11-19 18:18:35 -0800 | [diff] [blame] | 504 | os.chdir(common.autotest_dir) |
| 505 | global_config.global_config.parse_config_file() |
| 506 | |
| 507 | behaviors = parse_arguments(args) |
Shuqian Zhao | 79f248d | 2018-05-10 14:47:55 -0700 | [diff] [blame] | 508 | print('Updating server: %s' % HOSTNAME) |
Don Garrett | 03432d6 | 2014-11-19 18:18:35 -0800 | [diff] [blame] | 509 | if behaviors.verify: |
Prathmesh Prabhu | 32f6c22 | 2017-03-16 14:53:12 -0700 | [diff] [blame] | 510 | print('Checking tree status:') |
| 511 | verify_repo_clean() |
| 512 | print('Tree status: clean') |
Don Garrett | 8db752c | 2014-10-17 16:56:55 -0700 | [diff] [blame] | 513 | |
Aviv Keshet | ccbf643 | 2017-10-19 12:03:34 -0700 | [diff] [blame] | 514 | if behaviors.force_clean_externals: |
| 515 | print('Cleaning all external packages and their cache...') |
| 516 | _clean_externals() |
| 517 | print('...done.') |
| 518 | |
Don Garrett | 3571121 | 2014-12-18 14:33:41 -0800 | [diff] [blame] | 519 | versions_before = repo_versions() |
Shuqian Zhao | 697f7ed | 2016-10-21 14:58:28 -0700 | [diff] [blame] | 520 | versions_after = set() |
Shuqian Zhao | a3438a5 | 2016-09-20 15:11:02 -0700 | [diff] [blame] | 521 | cmd_versions_before = repo_versions_to_decide_whether_run_cmd_update() |
Shuqian Zhao | 697f7ed | 2016-10-21 14:58:28 -0700 | [diff] [blame] | 522 | cmd_versions_after = set() |
Don Garrett | fa2c1c4 | 2014-12-11 12:11:49 -0800 | [diff] [blame] | 523 | |
Don Garrett | 03432d6 | 2014-11-19 18:18:35 -0800 | [diff] [blame] | 524 | if behaviors.update: |
Don Garrett | 03432d6 | 2014-11-19 18:18:35 -0800 | [diff] [blame] | 525 | print('Updating Repo.') |
Shuqian Zhao | 8754a1a | 2016-08-24 12:54:11 -0700 | [diff] [blame] | 526 | repo_sync(behaviors.update_push_servers) |
Don Garrett | fa2c1c4 | 2014-12-11 12:11:49 -0800 | [diff] [blame] | 527 | versions_after = repo_versions() |
Shuqian Zhao | a3438a5 | 2016-09-20 15:11:02 -0700 | [diff] [blame] | 528 | cmd_versions_after = repo_versions_to_decide_whether_run_cmd_update() |
Allen Li | 8802a5e | 2017-01-17 15:04:15 -0800 | [diff] [blame] | 529 | _sync_chromiumos_repo() |
Allen Li | e8c4ea4 | 2016-10-05 18:08:29 -0700 | [diff] [blame] | 530 | |
Don Garrett | 03432d6 | 2014-11-19 18:18:35 -0800 | [diff] [blame] | 531 | if behaviors.actions: |
Shuqian Zhao | a3438a5 | 2016-09-20 15:11:02 -0700 | [diff] [blame] | 532 | # If the corresponding repo/file not change, no need to run the cmd. |
| 533 | cmds_skip = (set() if behaviors.force_update else |
| 534 | {t[0] for t in cmd_versions_before & cmd_versions_after}) |
Prathmesh Prabhu | 32f6c22 | 2017-03-16 14:53:12 -0700 | [diff] [blame] | 535 | run_deploy_actions( |
| 536 | cmds_skip, behaviors.dryrun, behaviors.skip_service_status, |
| 537 | use_chromite_master=behaviors.update_push_servers) |
Don Garrett | 03432d6 | 2014-11-19 18:18:35 -0800 | [diff] [blame] | 538 | |
Don Garrett | 3571121 | 2014-12-18 14:33:41 -0800 | [diff] [blame] | 539 | if behaviors.report: |
Don Garrett | fa2c1c4 | 2014-12-11 12:11:49 -0800 | [diff] [blame] | 540 | print('Changes:') |
| 541 | print(report_changes(versions_before, versions_after)) |
Don Garrett | 8db752c | 2014-10-17 16:56:55 -0700 | [diff] [blame] | 542 | |
| 543 | |
| 544 | if __name__ == '__main__': |
Don Garrett | 03432d6 | 2014-11-19 18:18:35 -0800 | [diff] [blame] | 545 | sys.exit(main(sys.argv[1:])) |