| #!/usr/bin/python |
| # Copyright (c) 2014 The Chromium OS Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| """Runs on autotest servers from a cron job to self update them. |
| |
| This script is designed to run on all autotest servers to allow them to |
| automatically self-update based on the manifests used to create their (existing) |
| repos. |
| """ |
| |
| from __future__ import print_function |
| |
| import ConfigParser |
| import argparse |
| import os |
| import re |
| import subprocess |
| import sys |
| import time |
| |
| import common |
| |
| from autotest_lib.client.common_lib import global_config |
| |
| # How long after restarting a service do we watch it to see if it's stable. |
| SERVICE_STABILITY_TIMER = 60 |
| |
| |
| class DirtyTreeException(Exception): |
| """Raised when the tree has been modified in an unexpected way.""" |
| |
| |
| class UnknownCommandException(Exception): |
| """Raised when we try to run a command name with no associated command.""" |
| |
| |
| class UnstableServices(Exception): |
| """Raised if a service appears unstable after restart.""" |
| |
| |
| def strip_terminal_codes(text): |
| """This function removes all terminal formatting codes from a string. |
| |
| @param text: String of text to cleanup. |
| @returns String with format codes removed. |
| """ |
| ESC = '\x1b' |
| return re.sub(ESC+r'\[[^m]*m', '', text) |
| |
| |
| def verify_repo_clean(): |
| """This function verifies that the current repo is valid, and clean. |
| |
| @raises DirtyTreeException if the repo is not clean. |
| @raises subprocess.CalledProcessError on a repo command failure. |
| """ |
| out = subprocess.check_output(['repo', 'status'], stderr=subprocess.STDOUT) |
| out = strip_terminal_codes(out).strip() |
| |
| CLEAN_STATUS_OUTPUT = 'nothing to commit (working directory clean)' |
| if out != CLEAN_STATUS_OUTPUT: |
| raise DirtyTreeException(out) |
| |
| |
| def repo_versions(): |
| """This function collects the versions of all git repos in the general repo. |
| |
| @returns A dictionary mapping project names to git hashes for HEAD. |
| @raises subprocess.CalledProcessError on a repo command failure. |
| """ |
| cmd = ['repo', 'forall', '-p', '-c', 'pwd && git log -1 --format=%h'] |
| output = strip_terminal_codes(subprocess.check_output(cmd)) |
| |
| # The expected output format is: |
| |
| # project chrome_build/ |
| # /dir/holding/chrome_build |
| # 73dee9d |
| # |
| # project chrome_release/ |
| # /dir/holding/chrome_release |
| # 9f3a5d8 |
| |
| lines = output.splitlines() |
| |
| PROJECT_PREFIX = 'project ' |
| |
| project_heads = {} |
| for n in range(0, len(lines), 4): |
| project_line = lines[n] |
| project_dir = lines[n+1] |
| project_hash = lines[n+2] |
| # lines[n+3] is a blank line, but doesn't exist for the final block. |
| |
| # Convert 'project chrome_build/' -> 'chrome_build' |
| assert project_line.startswith(PROJECT_PREFIX) |
| name = project_line[len(PROJECT_PREFIX):].rstrip('/') |
| |
| project_heads[name] = (project_dir, project_hash) |
| |
| return project_heads |
| |
| |
| def repo_sync(): |
| """Perform a repo sync. |
| |
| @raises subprocess.CalledProcessError on a repo command failure. |
| """ |
| subprocess.check_output(['repo', 'sync']) |
| |
| |
| def discover_update_commands(): |
| """Lookup the commands to run on this server. |
| |
| These commonly come from shadow_config.ini, since they vary by server type. |
| |
| @returns List of command names in string format. |
| """ |
| try: |
| return global_config.global_config.get_config_value( |
| 'UPDATE', 'commands', type=list) |
| |
| except (ConfigParser.NoSectionError, global_config.ConfigError): |
| return [] |
| |
| |
| def discover_restart_services(): |
| """Find the services that need restarting on the current server. |
| |
| These commonly come from shadow_config.ini, since they vary by server type. |
| |
| @returns List of service names in string format. |
| """ |
| try: |
| # From shadow_config.ini, lookup which services to restart. |
| return global_config.global_config.get_config_value( |
| 'UPDATE', 'services', type=list) |
| |
| except (ConfigParser.NoSectionError, global_config.ConfigError): |
| return [] |
| |
| |
| def update_command(cmd_tag, dryrun=False): |
| """Restart a command. |
| |
| The command name is looked up in global_config.ini to find the full command |
| to run, then it's executed. |
| |
| @param cmd_tag: Which command to restart. |
| @param dryrun: If true print the command that would have been run. |
| |
| @raises UnknownCommandException If cmd_tag can't be looked up. |
| @raises subprocess.CalledProcessError on a command failure. |
| """ |
| # Lookup the list of commands to consider. They are intended to be |
| # in global_config.ini so that they can be shared everywhere. |
| cmds = dict(global_config.global_config.config.items( |
| 'UPDATE_COMMANDS')) |
| |
| if cmd_tag not in cmds: |
| raise UnknownCommandException(cmd_tag, cmds) |
| |
| expanded_command = cmds[cmd_tag].replace('AUTOTEST_REPO', |
| common.autotest_dir) |
| |
| print('Running: %s: %s' % (cmd_tag, expanded_command)) |
| if dryrun: |
| print('Skip: %s' % expanded_command) |
| else: |
| try: |
| subprocess.check_output(expanded_command, shell=True, |
| stderr=subprocess.STDOUT) |
| except subprocess.CalledProcessError as e: |
| print('FAILED:') |
| print(e.output) |
| raise |
| |
| |
| def restart_service(service_name, dryrun=False): |
| """Restart a service. |
| |
| Restarts the standard service with "service <name> restart". |
| |
| @param service_name: The name of the service to restart. |
| @param dryrun: Don't really run anything, just print out the command. |
| |
| @raises subprocess.CalledProcessError on a command failure. |
| """ |
| cmd = ['sudo', 'service', service_name, 'restart'] |
| print('Restarting: %s' % service_name) |
| if dryrun: |
| print('Skip: %s' % ' '.join(cmd)) |
| else: |
| subprocess.check_call(cmd) |
| |
| |
| def service_status(service_name): |
| """Return the results "status <name>" for a given service. |
| |
| This string is expected to contain the pid, and so to change is the service |
| is shutdown or restarted for any reason. |
| |
| @param service_name: The name of the service to check on. |
| |
| @returns The output of the external command. |
| Ex: autofs start/running, process 1931 |
| |
| @raises subprocess.CalledProcessError on a command failure. |
| """ |
| return subprocess.check_output(['sudo', 'status', service_name]) |
| |
| |
| def restart_services(service_names, dryrun=False, skip_service_status=False): |
| """Restart services as needed for the current server type. |
| |
| Restart the listed set of services, and watch to see if they are stable for |
| at least SERVICE_STABILITY_TIMER. It restarts all services quickly, |
| waits for that delay, then verifies the status of all of them. |
| |
| @param service_names: The list of service to restart and monitor. |
| @param dryrun: Don't really restart the service, just print out the command. |
| @param skip_service_status: Set to True to skip service status check. |
| Default is False. |
| |
| @raises subprocess.CalledProcessError on a command failure. |
| @raises UnstableServices if any services are unstable after restart. |
| """ |
| service_statuses = {} |
| |
| if dryrun: |
| for name in service_names: |
| restart_service(name, dryrun=True) |
| return |
| |
| # Restart each, and record the status (including pid). |
| for name in service_names: |
| restart_service(name) |
| service_statuses[name] = service_status(name) |
| |
| # Skip service status check if --skip-service-status is specified. Used for |
| # servers in backup status. |
| if skip_service_status: |
| print('--skip-service-status is specified, skip checking services.') |
| return |
| |
| # Wait for a while to let the services settle. |
| time.sleep(SERVICE_STABILITY_TIMER) |
| |
| # Look for any services that changed status. |
| unstable_services = [n for n in service_names |
| if service_status(n) != service_statuses[n]] |
| |
| # Report any services having issues. |
| if unstable_services: |
| raise UnstableServices(unstable_services) |
| |
| |
| def run_deploy_actions(dryrun=False, skip_service_status=False): |
| """Run arbitrary update commands specified in global.ini. |
| |
| @param dryrun: Don't really restart the service, just print out the command. |
| @param skip_service_status: Set to True to skip service status check. |
| Default is False. |
| |
| @raises subprocess.CalledProcessError on a command failure. |
| @raises UnstableServices if any services are unstable after restart. |
| """ |
| cmds = discover_update_commands() |
| if cmds: |
| print('Running update commands:', ', '.join(cmds)) |
| for cmd in cmds: |
| update_command(cmd, dryrun=dryrun) |
| |
| services = discover_restart_services() |
| if services: |
| print('Restarting Services:', ', '.join(services)) |
| restart_services(services, dryrun=dryrun, |
| skip_service_status=skip_service_status) |
| |
| |
| def report_changes(versions_before, versions_after): |
| """Produce a report describing what changed in all repos. |
| |
| @param versions_before: Results of repo_versions() from before the update. |
| @param versions_after: Results of repo_versions() from after the update. |
| |
| @returns string containing a human friendly changes report. |
| """ |
| result = [] |
| |
| if versions_after: |
| for project in sorted(set(versions_before.keys() + versions_after.keys())): |
| result.append('%s:' % project) |
| |
| _, before_hash = versions_before.get(project, (None, None)) |
| after_dir, after_hash = versions_after.get(project, (None, None)) |
| |
| if project not in versions_before: |
| result.append('Added.') |
| |
| elif project not in versions_after: |
| result.append('Removed.') |
| |
| elif before_hash == after_hash: |
| result.append('No Change.') |
| |
| else: |
| hashes = '%s..%s' % (before_hash, after_hash) |
| cmd = ['git', 'log', hashes, '--oneline'] |
| out = subprocess.check_output(cmd, cwd=after_dir, |
| stderr=subprocess.STDOUT) |
| result.append(out.strip()) |
| |
| result.append('') |
| else: |
| for project in sorted(versions_before.keys()): |
| _, before_hash = versions_before[project] |
| result.append('%s: %s' % (project, before_hash)) |
| result.append('') |
| |
| return '\n'.join(result) |
| |
| |
| def parse_arguments(args): |
| """Parse command line arguments. |
| |
| @param args: The command line arguments to parse. (ususally sys.argsv[1:]) |
| |
| @returns An argparse.Namespace populated with argument values. |
| """ |
| parser = argparse.ArgumentParser( |
| description='Command to update an autotest server.') |
| parser.add_argument('--skip-verify', action='store_false', |
| dest='verify', default=True, |
| help='Disable verification of a clean repository.') |
| parser.add_argument('--skip-update', action='store_false', |
| dest='update', default=True, |
| help='Skip the repository source code update.') |
| parser.add_argument('--skip-actions', action='store_false', |
| dest='actions', default=True, |
| help='Skip the post update actions.') |
| parser.add_argument('--skip-report', action='store_false', |
| dest='report', default=True, |
| help='Skip the git version report.') |
| parser.add_argument('--actions-only', action='store_true', |
| help='Run the post update actions (restart services).') |
| parser.add_argument('--dryrun', action='store_true', |
| help='Don\'t actually run any commands, just log.') |
| parser.add_argument('--skip-service-status', action='store_true', |
| help='Skip checking the service status.') |
| |
| results = parser.parse_args(args) |
| |
| if results.actions_only: |
| results.verify = False |
| results.update = False |
| results.report = False |
| |
| # TODO(dgarrett): Make these behaviors support dryrun. |
| if results.dryrun: |
| results.verify = False |
| results.update = False |
| |
| return results |
| |
| |
| def main(args): |
| """Main method.""" |
| os.chdir(common.autotest_dir) |
| global_config.global_config.parse_config_file() |
| |
| behaviors = parse_arguments(args) |
| |
| if behaviors.verify: |
| try: |
| print('Checking tree status:') |
| verify_repo_clean() |
| print('Clean.') |
| except DirtyTreeException as e: |
| print('Local tree is dirty, can\'t perform update safely.') |
| print() |
| print('repo status:') |
| print(e.args[0]) |
| return 1 |
| |
| versions_before = repo_versions() |
| versions_after = {} |
| |
| if behaviors.update: |
| print('Updating Repo.') |
| repo_sync() |
| |
| versions_after = repo_versions() |
| if versions_before == versions_after: |
| print('No change found.') |
| return |
| |
| if behaviors.actions: |
| try: |
| run_deploy_actions( |
| dryrun=behaviors.dryrun, |
| skip_service_status=behaviors.skip_service_status) |
| except UnstableServices as e: |
| print('The following services were not stable after ' |
| 'the update:') |
| print(e.args[0]) |
| return 1 |
| |
| if behaviors.report: |
| print('Changes:') |
| print(report_changes(versions_before, versions_after)) |
| |
| |
| if __name__ == '__main__': |
| sys.exit(main(sys.argv[1:])) |