Dan Shi | 56f1ba7 | 2014-12-03 19:16:53 -0800 | [diff] [blame] | 1 | # Copyright 2014 The Chromium OS Authors. All rights reserved. |
| 2 | # Use of this source code is governed by a BSD-style license that can be |
| 3 | # found in the LICENSE file. |
| 4 | |
| 5 | """This module provides utility functions to help managing servers in server |
| 6 | database (defined in global config section AUTOTEST_SERVER_DB). |
| 7 | |
| 8 | After a role is added or removed from a server, certain services may need to |
| 9 | be restarted. For example, scheduler needs to be restarted after a drone is |
| 10 | added to a primary server. This module includes functions to check if actions |
| 11 | are required to be executed and what actions to executed on which servers. |
| 12 | """ |
| 13 | |
| 14 | import subprocess |
| 15 | import sys |
| 16 | |
| 17 | import common |
| 18 | |
| 19 | from autotest_lib.frontend.server import models as server_models |
| 20 | from autotest_lib.site_utils import server_manager_utils |
| 21 | from autotest_lib.site_utils.lib import infra |
| 22 | |
| 23 | |
| 24 | # Actions that must be executed for server management action to be effective. |
| 25 | # Each action is a tuple: |
| 26 | # (the role of which the command should be executed, the command) |
| 27 | RESTART_SCHEDULER = (server_models.ServerRole.ROLE.SCHEDULER, |
| 28 | 'sudo service scheduler restart') |
| 29 | RESTART_HOST_SCHEDULER = (server_models.ServerRole.ROLE.HOST_SCHEDULER, |
| 30 | 'sudo service host-scheduler restart') |
Dan Shi | 56f1ba7 | 2014-12-03 19:16:53 -0800 | [diff] [blame] | 31 | RELOAD_APACHE = (server_models.ServerRole.ROLE.SCHEDULER, |
| 32 | 'sudo service apache reload') |
| 33 | |
| 34 | STOP_SCHEDULER = (server_models.ServerRole.ROLE.SCHEDULER, |
| 35 | 'sudo service scheduler stop') |
| 36 | STOP_HOST_SCHEDULER = (server_models.ServerRole.ROLE.HOST_SCHEDULER, |
| 37 | 'sudo service host-scheduler stop') |
Dan Shi | 56f1ba7 | 2014-12-03 19:16:53 -0800 | [diff] [blame] | 38 | |
| 39 | # Dictionary of actions needed for a role to be enabled. Key is the role, and |
| 40 | # value is a list of action. All these actions should be applied after the role |
| 41 | # is added to the server, or the server's status is changed to primary. |
| 42 | ACTIONS_AFTER_ROLE_APPLIED = { |
| 43 | server_models.ServerRole.ROLE.SCHEDULER: [RESTART_SCHEDULER], |
| 44 | server_models.ServerRole.ROLE.HOST_SCHEDULER: [RESTART_HOST_SCHEDULER], |
Dan Shi | 56f1ba7 | 2014-12-03 19:16:53 -0800 | [diff] [blame] | 45 | server_models.ServerRole.ROLE.DRONE: [RESTART_SCHEDULER], |
| 46 | server_models.ServerRole.ROLE.DATABASE: |
| 47 | [RESTART_SCHEDULER, RESTART_HOST_SCHEDULER, RELOAD_APACHE], |
| 48 | server_models.ServerRole.ROLE.DEVSERVER: [RESTART_SCHEDULER], |
| 49 | } |
| 50 | |
| 51 | # Dictionary of actions needed for a role to be disabled. Key is the role, and |
| 52 | # value is a list of action. |
| 53 | # Action should be taken before role is deleted from a server, or the server's |
| 54 | # status is changed to primary. |
| 55 | ACTIONS_BEFORE_ROLE_REMOVED = { |
| 56 | server_models.ServerRole.ROLE.SCHEDULER: [STOP_SCHEDULER], |
| 57 | server_models.ServerRole.ROLE.HOST_SCHEDULER: [STOP_HOST_SCHEDULER], |
Dan Shi | 56f1ba7 | 2014-12-03 19:16:53 -0800 | [diff] [blame] | 58 | server_models.ServerRole.ROLE.DATABASE: |
| 59 | [STOP_SCHEDULER, STOP_HOST_SCHEDULER], |
| 60 | } |
| 61 | # Action should be taken after role is deleted from a server, or the server's |
| 62 | # status is changed to primary. |
| 63 | ACTIONS_AFTER_ROLE_REMOVED = { |
| 64 | server_models.ServerRole.ROLE.DRONE: [RESTART_SCHEDULER], |
| 65 | server_models.ServerRole.ROLE.DEVSERVER: [RESTART_SCHEDULER], |
| 66 | } |
| 67 | |
| 68 | |
| 69 | def apply(action): |
| 70 | """Apply an given action. |
| 71 | |
| 72 | It usually involves ssh to the server with specific role and run the |
| 73 | command, e.g., ssh to scheduler server and restart scheduler. |
| 74 | |
| 75 | @param action: A tuple of (the role of which the command should be executed, |
| 76 | the command) |
| 77 | @raise ServerActionError: If the action can't be applied due to database |
| 78 | issue. |
| 79 | @param subprocess.CalledProcessError: If command is failed to be |
| 80 | executed. |
| 81 | """ |
| 82 | role = action[0] |
| 83 | command = action[1] |
| 84 | # Find the servers with role |
| 85 | servers = server_manager_utils.get_servers( |
| 86 | role=role, status=server_models.Server.STATUS.PRIMARY) |
| 87 | if not servers: |
| 88 | print >> sys.stderr, ('WARNING! Action %s failed to be applied. No ' |
| 89 | 'server with given role %s was found.' % |
| 90 | (action, role)) |
| 91 | return |
| 92 | |
| 93 | for server in servers: |
| 94 | print 'Run command `%s` on server %s' % (command, server.hostname) |
| 95 | try: |
| 96 | infra.execute_command(server.hostname, command) |
| 97 | except subprocess.CalledProcessError as e: |
| 98 | print >> sys.stderr, ('Failed to check server %s, error: %s' % |
| 99 | (server.hostname, e)) |
| 100 | |
| 101 | |
| 102 | def try_execute(server, roles, enable, post_change, |
Xixuan Wu | 91f58e1 | 2017-11-29 11:38:13 -0800 | [diff] [blame] | 103 | prev_status=server_models.Server.STATUS.REPAIR_REQUIRED, |
Dan Shi | 56f1ba7 | 2014-12-03 19:16:53 -0800 | [diff] [blame] | 104 | do_action=False): |
| 105 | """Try to execute actions for given role changes of the server. |
| 106 | |
| 107 | @param server: Server that has the role changes. |
| 108 | @param roles: A list of roles changed. |
| 109 | @param enable: Set to True if the roles are enabled, i.e., added to server. |
| 110 | If it's False, the roles are removed from the server. |
| 111 | @param post_change: Set to True if to apply actions should be applied after |
| 112 | the role changes, otherwise, set to False. |
| 113 | @param prev_status: The previous status after the status change if any. This |
| 114 | is to help to decide if actions should be executed, |
| 115 | since actions should be applied if the server's status |
| 116 | is changed from primary to other status. Default to |
Xixuan Wu | 91f58e1 | 2017-11-29 11:38:13 -0800 | [diff] [blame] | 117 | repair_required. |
Dan Shi | 56f1ba7 | 2014-12-03 19:16:53 -0800 | [diff] [blame] | 118 | @param do_action: Set to True to execute actions, otherwise, post a warning. |
| 119 | """ |
| 120 | if not server_manager_utils.use_server_db(): |
| 121 | return |
| 122 | # This check is to prevent actions to be applied to server not in primary |
| 123 | # role or server database is not enabled. Note that no action is needed |
| 124 | # before a server is changed to primary status. If that assumption is |
| 125 | # no longer valid, this method needs to be updated accordingly. |
| 126 | if (server.status != server_models.Server.STATUS.PRIMARY and |
| 127 | prev_status != server_models.Server.STATUS.PRIMARY): |
| 128 | return |
| 129 | |
Xixuan Wu | 91f58e1 | 2017-11-29 11:38:13 -0800 | [diff] [blame] | 130 | possible_actions = {} |
Dan Shi | 56f1ba7 | 2014-12-03 19:16:53 -0800 | [diff] [blame] | 131 | if enable: |
| 132 | if post_change: |
| 133 | possible_actions = ACTIONS_AFTER_ROLE_APPLIED |
| 134 | else: |
| 135 | if post_change: |
| 136 | possible_actions = ACTIONS_AFTER_ROLE_REMOVED |
| 137 | else: |
| 138 | possible_actions = ACTIONS_BEFORE_ROLE_REMOVED |
| 139 | |
| 140 | all_actions = [] |
| 141 | for role in roles: |
| 142 | all_actions.extend(possible_actions.get(role, [])) |
| 143 | for action in set(all_actions): |
| 144 | if do_action: |
| 145 | apply(action) |
| 146 | else: |
| 147 | message = ('WARNING! Action %s is skipped. Please manually ' |
| 148 | 'execute the action to make your change effective.' % |
| 149 | str(action)) |
| 150 | print >> sys.stderr, message |