mbligh | 7c8ea99 | 2009-06-22 19:03:08 +0000 | [diff] [blame^] | 1 | #!/usr/bin/python |
mbligh | 54d13e6 | 2008-06-04 20:37:27 +0000 | [diff] [blame] | 2 | |
| 3 | _author_ = 'Scott Zawalski (scottz@google.com)' |
| 4 | |
| 5 | """Console check script to be used with conmux. |
| 6 | |
| 7 | Checks if machines are not only connected to conmux but also |
| 8 | responding in an expected way |
| 9 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 10 | Supports options to show all, good, bad, unknown and add them |
| 11 | to autotest as well. |
mbligh | 54d13e6 | 2008-06-04 20:37:27 +0000 | [diff] [blame] | 12 | |
| 13 | *In order for the power update option to work you have to have |
| 14 | access to the etc directory of the conmux server |
| 15 | """ |
| 16 | |
| 17 | import sys, pexpect, commands, os |
| 18 | from optparse import OptionParser |
| 19 | |
| 20 | |
| 21 | def main(argv): |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 22 | consoles = {} |
| 23 | consoles['good'] = [] |
| 24 | consoles['bad'] = [] |
| 25 | consoles['unknown'] = [] |
| 26 | # 0, 1, 2 status |
| 27 | STATUS = [ 'good', 'bad', 'unknown'] |
| 28 | parser = OptionParser() |
| 29 | parser.add_option('--conmux-server', dest="conmux_server", |
| 30 | default='localhost', |
| 31 | help="Conmux server to connect to") |
| 32 | parser.add_option('--conmux-dir', dest="conmux_dir", |
| 33 | default='/usr/local/conmux', |
| 34 | help="Conmux server to connect to") |
| 35 | parser.add_option('--console-binary', dest="console_binary", |
| 36 | default='/usr/local/conmux/bin/console', |
| 37 | help="Conmux console binary location") |
| 38 | parser.add_option('--autotest-cli-dir', dest="autotest_cli_dir", |
| 39 | default='/usr/local/autotest/cli', |
| 40 | help="Autotest CLI dir") |
| 41 | parser.add_option('--add-hosts', |
| 42 | action="store_true", dest="add_hosts", |
| 43 | default=False, |
| 44 | help="If host not on autotest server try to add it") |
| 45 | parser.add_option('--power-label', dest="power_label", |
| 46 | default='remote-power', |
| 47 | help="Label to add to hosts that support hard reset") |
| 48 | parser.add_option('--console-label', dest="console_label", |
| 49 | default='console', |
| 50 | help="Label to add to hosts that support console") |
| 51 | parser.add_option('--update-console-label', |
| 52 | action="store_true", dest="update_console_label", |
| 53 | default=False, |
| 54 | help="Update console label on autotest server") |
| 55 | parser.add_option('--update-power-label', |
| 56 | action="store_true", dest="update_power_label", |
| 57 | default=False, |
| 58 | help="Update power label on autotest server" +\ |
| 59 | "*Note this runs then exists no consoles are checked") |
| 60 | parser.add_option('--verbose', |
| 61 | action="store_true", dest="verbose", |
| 62 | default=False, |
| 63 | help="Verbose output") |
| 64 | parser.add_option('--show-bad', |
| 65 | action="store_true", dest="show_bad", |
| 66 | default=False, |
| 67 | help="Show consoles that are no longer functioning") |
| 68 | parser.add_option('--show-good', |
| 69 | action="store_true", dest="show_good", |
| 70 | default=False, |
| 71 | help="Show consoles that are functioning properly") |
| 72 | parser.add_option('--show-unknown', |
| 73 | action="store_true", dest="show_unknown", |
| 74 | default=False, |
| 75 | help="Show consoles that are in an unknown state") |
| 76 | parser.add_option('--show-all', |
| 77 | action="store_true", dest="show_all", |
| 78 | default=False, |
| 79 | help="Show status of all consoles") |
| 80 | options, args = parser.parse_args() |
| 81 | if len(argv) == 2 and options.verbose: |
| 82 | parser.print_help() |
| 83 | return 1 |
| 84 | elif len(argv) < 2: |
| 85 | parser.print_help() |
| 86 | return 1 |
mbligh | 54d13e6 | 2008-06-04 20:37:27 +0000 | [diff] [blame] | 87 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 88 | if options.update_power_label: |
| 89 | remove_create_label(options.power_label, |
| 90 | options.autotest_cli_dir) |
| 91 | update_power_label(options.power_label, options.conmux_dir, |
| 92 | options.autotest_cli_dir, options.add_hosts) |
| 93 | return |
| 94 | print options.console_binary |
| 95 | if not os.path.exists(options.console_binary): |
| 96 | print "Error %s does not exist, please specify another path" %\ |
| 97 | options.console_binary |
| 98 | return 1 |
| 99 | hosts = get_console_hosts(options.console_binary, options.conmux_server) |
| 100 | for host in hosts: |
| 101 | rc = check_host(host, options.console_binary) |
| 102 | if options.verbose is True: |
| 103 | print "%s status: %s" % (host, STATUS[rc]) |
| 104 | consoles[STATUS[rc]].append(host) |
mbligh | 54d13e6 | 2008-06-04 20:37:27 +0000 | [diff] [blame] | 105 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 106 | if options.show_all: |
| 107 | for status in consoles: |
| 108 | print "--- %s ---" % status |
| 109 | for host in consoles[status]: |
| 110 | print host |
| 111 | if options.show_good: |
| 112 | print "--- good ---" |
| 113 | for host in consoles['good']: |
| 114 | print host |
| 115 | if options.show_bad: |
| 116 | print "--- bad ---" |
| 117 | for host in consoles['bad']: |
| 118 | print host |
| 119 | if options.show_unknown: |
| 120 | print "--- unknown ---" |
| 121 | for host in consoles['unknown']: |
| 122 | print host |
mbligh | 54d13e6 | 2008-06-04 20:37:27 +0000 | [diff] [blame] | 123 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 124 | if options.update_console_label: |
| 125 | remove_create_label(options.console_label, |
| 126 | options.autotest_cli_dir) |
| 127 | update_console_label(options.console_label, consoles['good'], |
| 128 | options.autotest_cli_dir, options.add_hosts) |
| 129 | |
mbligh | 54d13e6 | 2008-06-04 20:37:27 +0000 | [diff] [blame] | 130 | |
| 131 | def update_console_label(console_label, consoles, cli_dir, add_hosts=False): |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 132 | """Update CONSOLE_LABEL on your autotest server. |
| 133 | This removes the label and recreates it, then populating the label |
| 134 | with all the machines your conmux server knows about. |
mbligh | 54d13e6 | 2008-06-04 20:37:27 +0000 | [diff] [blame] | 135 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 136 | *Note If the hosts do not exist they are created. |
| 137 | Args: |
| 138 | console_label: |
| 139 | string, describes the autotest label to add to machines. |
| 140 | consoles: |
| 141 | list, all the consoles that have confirmed console support. |
| 142 | """ |
| 143 | # TODO: Update to new CLI and change logic until then |
| 144 | # this is the best way to ensure a machine is added i.e. one at a time |
| 145 | |
| 146 | for host in consoles: |
| 147 | if not host_label_add(host, console_label, cli_dir): |
| 148 | # Try to create host |
| 149 | if add_hosts: |
| 150 | if host_create(host, cli_dir): |
| 151 | host_label_add(host, power_label, |
| 152 | cli_dir) |
| 153 | else: |
| 154 | print "Unable to add host " + host |
mbligh | 54d13e6 | 2008-06-04 20:37:27 +0000 | [diff] [blame] | 155 | |
| 156 | |
| 157 | def update_power_label(power_label, conmux_dir, cli_dir, add_hosts=False): |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 158 | """Look in CONSOLE_DIR/etc and grab known power commands |
| 159 | Then remove POWER_LABEL and add machines to that label |
| 160 | """ |
| 161 | # remove label and add it |
| 162 | for host in hard_reset_hosts(conmux_dir): |
| 163 | rc = label_add_host(host, power_label, cli_dir) |
| 164 | if not rc: |
| 165 | # Try to create the host |
| 166 | if add_hosts: |
| 167 | if host_create(host, cli_dir): |
| 168 | rc = label_add_host(host, power_label, |
| 169 | cli_dir) |
| 170 | else: |
| 171 | print "Unable to add host " + host |
mbligh | 54d13e6 | 2008-06-04 20:37:27 +0000 | [diff] [blame] | 172 | |
| 173 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 174 | def hard_reset_hosts(conmux_dir): |
| 175 | """Go through conmux dir and find hosts that have reset commands""" |
| 176 | config_dir = os.path.join(conmux_dir, "etc") |
| 177 | hosts = [] |
| 178 | for file in os.listdir(config_dir): |
| 179 | if not file.endswith(".cf"): |
| 180 | continue |
| 181 | file_path = os.path.join(config_dir, file) |
| 182 | try: |
| 183 | try: |
| 184 | f = open(file_path) |
| 185 | for line in f: |
| 186 | if "reset" in line: |
| 187 | hosts.append(file.rstrip(".cf")) |
| 188 | except IOError: |
| 189 | pass |
| 190 | finally: |
| 191 | f.close() |
| 192 | return hosts |
mbligh | 54d13e6 | 2008-06-04 20:37:27 +0000 | [diff] [blame] | 193 | |
| 194 | |
| 195 | def host_create(host, cli_dir): |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 196 | """Create a host |
| 197 | Return: |
| 198 | True, if successfuly false if failed |
| 199 | """ |
| 200 | cmd = "%s/host-create %s" % (cli_dir, host) |
| 201 | status, output = commands.getstatusoutput(cmd) |
| 202 | return status == 0 |
mbligh | 54d13e6 | 2008-06-04 20:37:27 +0000 | [diff] [blame] | 203 | |
| 204 | |
| 205 | def label_add_host(host, label, cli_dir): |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 206 | """Add a host to a label""" |
| 207 | host_cmd = "%s/label-add-hosts %s %s" % (cli_dir, label, host) |
| 208 | (status, output) = commands.getstatusoutput(host_cmd) |
| 209 | if status != 0: |
| 210 | return False |
mbligh | 54d13e6 | 2008-06-04 20:37:27 +0000 | [diff] [blame] | 211 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 212 | return True |
mbligh | 54d13e6 | 2008-06-04 20:37:27 +0000 | [diff] [blame] | 213 | |
| 214 | |
| 215 | def remove_create_label(label, cli_dir): |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 216 | """Remove and recreate a given label""" |
| 217 | cmd = "%s/label-rm %s" % (cli_dir, label) |
| 218 | status, output = commands.getstatusoutput(cmd) |
| 219 | if status != 0: |
| 220 | raise Exception("Error deleting label: " + label) |
mbligh | 54d13e6 | 2008-06-04 20:37:27 +0000 | [diff] [blame] | 221 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 222 | cmd = "%s/label-create %s" % (cli_dir, label) |
| 223 | status, output = commands.getstatusoutput(cmd) |
| 224 | if status != 0: |
| 225 | raise Exception("Error creating label: " + label + output) |
mbligh | 54d13e6 | 2008-06-04 20:37:27 +0000 | [diff] [blame] | 226 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 227 | return True |
mbligh | 54d13e6 | 2008-06-04 20:37:27 +0000 | [diff] [blame] | 228 | |
| 229 | |
| 230 | def get_console_hosts(console_binary, conmux_server): |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 231 | """Use console to collect console hosts and return a list. |
mbligh | 54d13e6 | 2008-06-04 20:37:27 +0000 | [diff] [blame] | 232 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 233 | Args: |
| 234 | console_binary: |
| 235 | string, location of the conmux console binary |
| 236 | conmux_server: |
| 237 | string, hostname of the conmux server |
mbligh | 54d13e6 | 2008-06-04 20:37:27 +0000 | [diff] [blame] | 238 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 239 | Returns: |
| 240 | A List of console conmux is currently running on. |
| 241 | """ |
| 242 | |
| 243 | hosts_list = [] |
| 244 | cmd = "%s --list %s" % (console_binary, conmux_server) |
| 245 | for line in commands.getoutput(cmd).split('\n'): |
| 246 | host = (line.split(' '))[0] |
| 247 | hosts_list.append(host) |
| 248 | |
| 249 | return hosts_list |
mbligh | 54d13e6 | 2008-06-04 20:37:27 +0000 | [diff] [blame] | 250 | |
| 251 | |
| 252 | def check_host(host, console_binary): |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 253 | """Check hosts for common errors and return the status. |
mbligh | 54d13e6 | 2008-06-04 20:37:27 +0000 | [diff] [blame] | 254 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 255 | Args: |
| 256 | host: |
| 257 | string, the console host identifier |
mbligh | 54d13e6 | 2008-06-04 20:37:27 +0000 | [diff] [blame] | 258 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 259 | console_binary: |
| 260 | string, location of the conmux console binary |
| 261 | Returns: |
| 262 | int, 0: Machine state is good |
| 263 | int, 1: Machine state is bad |
| 264 | int, 2: Machine state is unknown |
| 265 | """ |
| 266 | RESPONSES = [ host + ' login:', |
| 267 | 'ENOENT entry not found', |
| 268 | 'login:', |
| 269 | 'Connection refused', |
| 270 | '<<<NOT CONNECTED>>>', |
| 271 | 'Authentication failure', |
| 272 | 'Give root password for maintenance', ] |
mbligh | 54d13e6 | 2008-06-04 20:37:27 +0000 | [diff] [blame] | 273 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 274 | cmd = '%s %s' % (console_binary, host) |
| 275 | shell = pexpect.spawn(cmd) |
mbligh | 54d13e6 | 2008-06-04 20:37:27 +0000 | [diff] [blame] | 276 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 277 | shell.send('\r\n') |
| 278 | shell.send('\r\n') |
| 279 | shell.send('\r\n') |
| 280 | try: |
| 281 | # May need to increase the timeout but good so far |
| 282 | response = shell.expect(RESPONSES, 1) |
| 283 | except pexpect.TIMEOUT: |
| 284 | shell.sendline('~$') |
| 285 | shell.expect('>') |
| 286 | shell.sendline('quit') |
| 287 | return 1 |
| 288 | except pexpect.EOF: |
| 289 | # unknown error |
| 290 | shell.sendline('~$') |
| 291 | shell.expect('>') |
| 292 | shell.sendline('quit') |
| 293 | return 2 |
| 294 | # TODO: Change actions based on what server returned |
| 295 | if response == 0: |
| 296 | # OK response |
| 297 | return 0 |
| 298 | else: |
| 299 | return 1 |
mbligh | 54d13e6 | 2008-06-04 20:37:27 +0000 | [diff] [blame] | 300 | |
| 301 | |
| 302 | if __name__ == '__main__': |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 303 | main(sys.argv) |