mbligh | dcd57a8 | 2007-07-11 23:06:47 +0000 | [diff] [blame] | 1 | #!/usr/bin/python |
| 2 | # |
| 3 | # Copyright 2007 Google Inc. Released under the GPL v2 |
| 4 | |
mbligh | 7d2bde8 | 2007-08-02 16:26:10 +0000 | [diff] [blame] | 5 | """ |
| 6 | This module defines the SSHHost class. |
mbligh | dcd57a8 | 2007-07-11 23:06:47 +0000 | [diff] [blame] | 7 | |
| 8 | Implementation details: |
| 9 | You should import the "hosts" package instead of importing each type of host. |
| 10 | |
| 11 | SSHHost: a remote machine with a ssh access |
| 12 | """ |
| 13 | |
mbligh | 7d2bde8 | 2007-08-02 16:26:10 +0000 | [diff] [blame] | 14 | __author__ = """ |
| 15 | mbligh@google.com (Martin J. Bligh), |
mbligh | dcd57a8 | 2007-07-11 23:06:47 +0000 | [diff] [blame] | 16 | poirier@google.com (Benjamin Poirier), |
mbligh | 7d2bde8 | 2007-08-02 16:26:10 +0000 | [diff] [blame] | 17 | stutsman@google.com (Ryan Stutsman) |
| 18 | """ |
mbligh | dcd57a8 | 2007-07-11 23:06:47 +0000 | [diff] [blame] | 19 | |
| 20 | |
mbligh | f5427bb | 2008-04-09 15:55:57 +0000 | [diff] [blame] | 21 | import types, os, sys, signal, subprocess, time, re, socket, pdb |
| 22 | |
| 23 | from autotest_lib.client.common_lib import error |
| 24 | from autotest_lib.server import utils |
| 25 | import remote, bootloader |
mbligh | dcd57a8 | 2007-07-11 23:06:47 +0000 | [diff] [blame] | 26 | |
| 27 | |
mbligh | f5427bb | 2008-04-09 15:55:57 +0000 | [diff] [blame] | 28 | class SSHHost(remote.RemoteHost): |
mbligh | 7d2bde8 | 2007-08-02 16:26:10 +0000 | [diff] [blame] | 29 | """ |
| 30 | This class represents a remote machine controlled through an ssh |
mbligh | dcd57a8 | 2007-07-11 23:06:47 +0000 | [diff] [blame] | 31 | session on which you can run programs. |
mbligh | 7d2bde8 | 2007-08-02 16:26:10 +0000 | [diff] [blame] | 32 | |
mbligh | dcd57a8 | 2007-07-11 23:06:47 +0000 | [diff] [blame] | 33 | It is not the machine autoserv is running on. The machine must be |
| 34 | configured for password-less login, for example through public key |
| 35 | authentication. |
mbligh | 7d2bde8 | 2007-08-02 16:26:10 +0000 | [diff] [blame] | 36 | |
mbligh | 3409ee7 | 2007-10-16 23:58:33 +0000 | [diff] [blame] | 37 | It includes support for controlling the machine through a serial |
| 38 | console on which you can run programs. If such a serial console is |
| 39 | set up on the machine then capabilities such as hard reset and |
| 40 | boot strap monitoring are available. If the machine does not have a |
| 41 | serial console available then ordinary SSH-based commands will |
| 42 | still be available, but attempts to use extensions such as |
| 43 | console logging or hard reset will fail silently. |
| 44 | |
mbligh | dcd57a8 | 2007-07-11 23:06:47 +0000 | [diff] [blame] | 45 | Implementation details: |
| 46 | This is a leaf class in an abstract class hierarchy, it must |
| 47 | implement the unimplemented methods in parent classes. |
| 48 | """ |
mbligh | 7d2bde8 | 2007-08-02 16:26:10 +0000 | [diff] [blame] | 49 | |
mbligh | 31a49de | 2007-11-05 18:41:19 +0000 | [diff] [blame] | 50 | DEFAULT_REBOOT_TIMEOUT = 1800 |
| 51 | job = None |
mbligh | 0faf91f | 2007-10-18 03:10:48 +0000 | [diff] [blame] | 52 | |
mbligh | de38437 | 2007-10-17 04:25:37 +0000 | [diff] [blame] | 53 | def __init__(self, hostname, user="root", port=22, initialize=True, |
mbligh | f4e0415 | 2008-02-21 16:05:53 +0000 | [diff] [blame] | 54 | conmux_log="console.log", |
mbligh | e6c995f | 2007-10-26 19:43:01 +0000 | [diff] [blame] | 55 | conmux_server=None, conmux_attach=None, |
mbligh | da13d54 | 2008-01-03 16:28:34 +0000 | [diff] [blame] | 56 | netconsole_log=None, netconsole_port=6666, autodir=None): |
mbligh | 7d2bde8 | 2007-08-02 16:26:10 +0000 | [diff] [blame] | 57 | """ |
| 58 | Construct a SSHHost object |
mbligh | dcd57a8 | 2007-07-11 23:06:47 +0000 | [diff] [blame] | 59 | |
| 60 | Args: |
| 61 | hostname: network hostname or address of remote machine |
| 62 | user: user to log in as on the remote machine |
| 63 | port: port the ssh daemon is listening on on the remote |
| 64 | machine |
mbligh | 9708f73 | 2007-10-18 03:18:54 +0000 | [diff] [blame] | 65 | """ |
mbligh | dcd57a8 | 2007-07-11 23:06:47 +0000 | [diff] [blame] | 66 | self.hostname= hostname |
| 67 | self.user= user |
| 68 | self.port= port |
| 69 | self.tmp_dirs= [] |
mbligh | 137a05c | 2007-10-04 15:56:51 +0000 | [diff] [blame] | 70 | self.initialize = initialize |
mbligh | da13d54 | 2008-01-03 16:28:34 +0000 | [diff] [blame] | 71 | self.autodir = autodir |
mbligh | 9133490 | 2007-09-28 01:47:59 +0000 | [diff] [blame] | 72 | |
mbligh | 9708f73 | 2007-10-18 03:18:54 +0000 | [diff] [blame] | 73 | super(SSHHost, self).__init__() |
| 74 | |
mbligh | 3409ee7 | 2007-10-16 23:58:33 +0000 | [diff] [blame] | 75 | self.conmux_server = conmux_server |
mbligh | 70cf0ec | 2008-01-18 17:57:14 +0000 | [diff] [blame] | 76 | if conmux_attach: |
| 77 | self.conmux_attach = conmux_attach |
| 78 | else: |
| 79 | self.conmux_attach = os.path.abspath(os.path.join( |
| 80 | self.serverdir, '..', |
| 81 | 'conmux', 'conmux-attach')) |
mbligh | fbb0354 | 2008-02-11 16:27:29 +0000 | [diff] [blame] | 82 | self.logger_popen = None |
mbligh | f4e0415 | 2008-02-21 16:05:53 +0000 | [diff] [blame] | 83 | self.warning_stream = None |
mbligh | de38437 | 2007-10-17 04:25:37 +0000 | [diff] [blame] | 84 | self.__start_console_log(conmux_log) |
mbligh | 3409ee7 | 2007-10-16 23:58:33 +0000 | [diff] [blame] | 85 | |
mbligh | a0452c8 | 2007-08-08 20:24:57 +0000 | [diff] [blame] | 86 | self.bootloader = bootloader.Bootloader(self) |
mbligh | 7d2bde8 | 2007-08-02 16:26:10 +0000 | [diff] [blame] | 87 | |
mbligh | c0e9239 | 2007-11-05 19:10:10 +0000 | [diff] [blame] | 88 | self.__netconsole_param = "" |
mbligh | fbb0354 | 2008-02-11 16:27:29 +0000 | [diff] [blame] | 89 | self.netlogger_popen = None |
mbligh | c0e9239 | 2007-11-05 19:10:10 +0000 | [diff] [blame] | 90 | if netconsole_log: |
| 91 | self.__init_netconsole_params(netconsole_port) |
| 92 | self.__start_netconsole_log(netconsole_log, netconsole_port) |
| 93 | self.__load_netconsole_module() |
mbligh | de38437 | 2007-10-17 04:25:37 +0000 | [diff] [blame] | 94 | |
mbligh | 7d2bde8 | 2007-08-02 16:26:10 +0000 | [diff] [blame] | 95 | |
mbligh | fbb0354 | 2008-02-11 16:27:29 +0000 | [diff] [blame] | 96 | @staticmethod |
mbligh | f4e0415 | 2008-02-21 16:05:53 +0000 | [diff] [blame] | 97 | def __kill(popen): |
mbligh | fbb0354 | 2008-02-11 16:27:29 +0000 | [diff] [blame] | 98 | return_code = popen.poll() |
mbligh | f4e0415 | 2008-02-21 16:05:53 +0000 | [diff] [blame] | 99 | if return_code is None: |
mbligh | fbb0354 | 2008-02-11 16:27:29 +0000 | [diff] [blame] | 100 | try: |
mbligh | f4e0415 | 2008-02-21 16:05:53 +0000 | [diff] [blame] | 101 | os.kill(popen.pid, signal.SIGTERM) |
mbligh | fbb0354 | 2008-02-11 16:27:29 +0000 | [diff] [blame] | 102 | except OSError: |
| 103 | pass |
| 104 | |
| 105 | |
mbligh | dcd57a8 | 2007-07-11 23:06:47 +0000 | [diff] [blame] | 106 | def __del__(self): |
mbligh | 7d2bde8 | 2007-08-02 16:26:10 +0000 | [diff] [blame] | 107 | """ |
| 108 | Destroy a SSHHost object |
mbligh | dcd57a8 | 2007-07-11 23:06:47 +0000 | [diff] [blame] | 109 | """ |
| 110 | for dir in self.tmp_dirs: |
| 111 | try: |
| 112 | self.run('rm -rf "%s"' % (utils.sh_escape(dir))) |
mbligh | f5427bb | 2008-04-09 15:55:57 +0000 | [diff] [blame] | 113 | except error.AutoservRunError: |
mbligh | dcd57a8 | 2007-07-11 23:06:47 +0000 | [diff] [blame] | 114 | pass |
mbligh | de38437 | 2007-10-17 04:25:37 +0000 | [diff] [blame] | 115 | # kill the console logger |
mbligh | fbb0354 | 2008-02-11 16:27:29 +0000 | [diff] [blame] | 116 | if getattr(self, 'logger_popen', None): |
mbligh | f4e0415 | 2008-02-21 16:05:53 +0000 | [diff] [blame] | 117 | self.__kill(self.logger_popen) |
mbligh | 6607d19 | 2008-04-17 15:23:15 +0000 | [diff] [blame] | 118 | if self.job: |
| 119 | self.job.warning_loggers.discard( |
| 120 | self.warning_stream) |
mbligh | f4e0415 | 2008-02-21 16:05:53 +0000 | [diff] [blame] | 121 | self.warning_stream.close() |
mbligh | de38437 | 2007-10-17 04:25:37 +0000 | [diff] [blame] | 122 | # kill the netconsole logger |
mbligh | fbb0354 | 2008-02-11 16:27:29 +0000 | [diff] [blame] | 123 | if getattr(self, 'netlogger_popen', None): |
mbligh | e6c995f | 2007-10-26 19:43:01 +0000 | [diff] [blame] | 124 | self.__unload_netconsole_module() |
mbligh | f4e0415 | 2008-02-21 16:05:53 +0000 | [diff] [blame] | 125 | self.__kill(self.netlogger_popen) |
mbligh | de38437 | 2007-10-17 04:25:37 +0000 | [diff] [blame] | 126 | |
| 127 | |
| 128 | def __init_netconsole_params(self, port): |
| 129 | """ |
| 130 | Connect to the remote machine and determine the values to use for the |
| 131 | required netconsole parameters. |
| 132 | """ |
mbligh | de38437 | 2007-10-17 04:25:37 +0000 | [diff] [blame] | 133 | # PROBLEM: on machines with multiple IPs this may not make any sense |
| 134 | # It also doesn't work with IPv6 |
| 135 | remote_ip = socket.gethostbyname(self.hostname) |
| 136 | local_ip = socket.gethostbyname(socket.gethostname()) |
| 137 | # Get the gateway of the remote machine |
| 138 | try: |
| 139 | traceroute = self.run('traceroute -n %s' % local_ip) |
mbligh | f5427bb | 2008-04-09 15:55:57 +0000 | [diff] [blame] | 140 | except error.AutoservRunError: |
mbligh | de38437 | 2007-10-17 04:25:37 +0000 | [diff] [blame] | 141 | return |
| 142 | first_node = traceroute.stdout.split("\n")[0] |
| 143 | match = re.search(r'\s+((\d+\.){3}\d+)\s+', first_node) |
| 144 | if match: |
| 145 | router_ip = match.group(1) |
| 146 | else: |
| 147 | return |
| 148 | # Look up the MAC address of the gateway |
| 149 | try: |
| 150 | self.run('ping -c 1 %s' % router_ip) |
| 151 | arp = self.run('arp -n -a %s' % router_ip) |
mbligh | f5427bb | 2008-04-09 15:55:57 +0000 | [diff] [blame] | 152 | except error.AutoservRunError: |
mbligh | de38437 | 2007-10-17 04:25:37 +0000 | [diff] [blame] | 153 | return |
| 154 | match = re.search(r'\s+(([0-9A-F]{2}:){5}[0-9A-F]{2})\s+', arp.stdout) |
| 155 | if match: |
| 156 | gateway_mac = match.group(1) |
| 157 | else: |
| 158 | return |
| 159 | self.__netconsole_param = 'netconsole=@%s/,%s@%s/%s' % (remote_ip, |
| 160 | port, |
| 161 | local_ip, |
| 162 | gateway_mac) |
| 163 | |
| 164 | |
| 165 | def __start_netconsole_log(self, logfilename, port): |
| 166 | """ |
| 167 | Log the output of netconsole to a specified file |
| 168 | """ |
| 169 | if logfilename == None: |
| 170 | return |
| 171 | cmd = ['nc', '-u', '-l', '-p', str(port)] |
mbligh | fbb0354 | 2008-02-11 16:27:29 +0000 | [diff] [blame] | 172 | logfile = open(logfilename, 'a', 0) |
| 173 | self.netlogger_popen = subprocess.Popen(cmd, stdout=logfile) |
mbligh | de38437 | 2007-10-17 04:25:37 +0000 | [diff] [blame] | 174 | |
| 175 | |
| 176 | def __load_netconsole_module(self): |
| 177 | """ |
| 178 | Make a best effort to load the netconsole module. |
| 179 | |
| 180 | Note that loading the module can fail even when the remote machine is |
| 181 | working correctly if netconsole is already compiled into the kernel |
| 182 | and started. |
| 183 | """ |
mbligh | c0e9239 | 2007-11-05 19:10:10 +0000 | [diff] [blame] | 184 | if not self.__netconsole_param: |
| 185 | return |
mbligh | de38437 | 2007-10-17 04:25:37 +0000 | [diff] [blame] | 186 | try: |
| 187 | self.run('modprobe netconsole %s' % self.__netconsole_param) |
mbligh | f5427bb | 2008-04-09 15:55:57 +0000 | [diff] [blame] | 188 | except error.AutoservRunError: |
mbligh | de38437 | 2007-10-17 04:25:37 +0000 | [diff] [blame] | 189 | # if it fails there isn't much we can do, just keep going |
| 190 | pass |
| 191 | |
| 192 | |
| 193 | def __unload_netconsole_module(self): |
| 194 | try: |
| 195 | self.run('modprobe -r netconsole') |
mbligh | f5427bb | 2008-04-09 15:55:57 +0000 | [diff] [blame] | 196 | except error.AutoservRunError: |
mbligh | de38437 | 2007-10-17 04:25:37 +0000 | [diff] [blame] | 197 | pass |
mbligh | 3409ee7 | 2007-10-16 23:58:33 +0000 | [diff] [blame] | 198 | |
| 199 | |
mbligh | 5deff3d | 2008-01-04 21:21:28 +0000 | [diff] [blame] | 200 | def wait_for_restart(self, timeout=DEFAULT_REBOOT_TIMEOUT): |
mbligh | d567f72 | 2007-10-30 15:37:33 +0000 | [diff] [blame] | 201 | if not self.wait_down(300): # Make sure he's dead, Jim |
mbligh | f3b7893 | 2007-11-07 16:52:47 +0000 | [diff] [blame] | 202 | self.__record("ABORT", None, "reboot.verify", "shutdown failed") |
mbligh | f5427bb | 2008-04-09 15:55:57 +0000 | [diff] [blame] | 203 | raise error.AutoservRebootError( |
| 204 | "Host did not shut down") |
mbligh | 3409ee7 | 2007-10-16 23:58:33 +0000 | [diff] [blame] | 205 | self.wait_up(timeout) |
| 206 | time.sleep(2) # this is needed for complete reliability |
mbligh | cf3d83a | 2007-11-05 19:21:39 +0000 | [diff] [blame] | 207 | if self.wait_up(timeout): |
mbligh | 3027030 | 2007-11-05 20:33:52 +0000 | [diff] [blame] | 208 | self.__record("GOOD", None, "reboot.verify") |
mbligh | cf3d83a | 2007-11-05 19:21:39 +0000 | [diff] [blame] | 209 | else: |
mbligh | 71d2422 | 2008-03-11 21:31:56 +0000 | [diff] [blame] | 210 | self.__record("ABORT", None, "reboot.verify", "Host did not return from reboot") |
mbligh | f5427bb | 2008-04-09 15:55:57 +0000 | [diff] [blame] | 211 | raise error.AutoservRebootError( |
| 212 | "Host did not return from reboot") |
mbligh | 3409ee7 | 2007-10-16 23:58:33 +0000 | [diff] [blame] | 213 | print "Reboot complete" |
| 214 | |
| 215 | |
mbligh | 80d2077 | 2007-10-29 17:10:10 +0000 | [diff] [blame] | 216 | def hardreset(self, timeout=DEFAULT_REBOOT_TIMEOUT, wait=True): |
mbligh | 3409ee7 | 2007-10-16 23:58:33 +0000 | [diff] [blame] | 217 | """ |
| 218 | Reach out and slap the box in the power switch |
| 219 | """ |
mbligh | f3b7893 | 2007-11-07 16:52:47 +0000 | [diff] [blame] | 220 | if not self.__console_run(r"'~$hardreset'"): |
| 221 | self.__record("ABORT", None, "reboot.start", "hard reset unavailable") |
mbligh | f5427bb | 2008-04-09 15:55:57 +0000 | [diff] [blame] | 222 | raise error.AutoservUnsupportedError( |
| 223 | 'Hard reset unavailable') |
mbligh | 37d53c3 | 2008-01-14 16:16:00 +0000 | [diff] [blame] | 224 | |
| 225 | if wait: |
| 226 | self.wait_for_restart(timeout) |
mbligh | a4d4f37 | 2008-01-22 15:49:50 +0000 | [diff] [blame] | 227 | self.__record("GOOD", None, "reboot.start", "hard reset") |
mbligh | 3409ee7 | 2007-10-16 23:58:33 +0000 | [diff] [blame] | 228 | |
| 229 | |
mbligh | e6c995f | 2007-10-26 19:43:01 +0000 | [diff] [blame] | 230 | def __conmux_hostname(self): |
| 231 | if self.conmux_server: |
| 232 | return '%s/%s' % (self.conmux_server, self.hostname) |
| 233 | else: |
| 234 | return self.hostname |
| 235 | |
| 236 | |
mbligh | 3409ee7 | 2007-10-16 23:58:33 +0000 | [diff] [blame] | 237 | def __start_console_log(self, logfilename): |
| 238 | """ |
| 239 | Log the output of the console session to a specified file |
| 240 | """ |
| 241 | if logfilename == None: |
| 242 | return |
| 243 | if not self.conmux_attach or not os.path.exists(self.conmux_attach): |
| 244 | return |
mbligh | f4e0415 | 2008-02-21 16:05:53 +0000 | [diff] [blame] | 245 | |
| 246 | r, w = os.pipe() |
| 247 | script_path = os.path.join(self.serverdir, |
| 248 | 'warning_monitor.py') |
mbligh | fbb0354 | 2008-02-11 16:27:29 +0000 | [diff] [blame] | 249 | cmd = [self.conmux_attach, self.__conmux_hostname(), |
mbligh | f4e0415 | 2008-02-21 16:05:53 +0000 | [diff] [blame] | 250 | '%s %s %s %d' % (sys.executable, script_path, |
| 251 | logfilename, w)] |
mbligh | 0c5ce31 | 2008-02-21 16:24:11 +0000 | [diff] [blame] | 252 | dev_null = open(os.devnull, 'w') |
mbligh | 3409ee7 | 2007-10-16 23:58:33 +0000 | [diff] [blame] | 253 | |
mbligh | f4e0415 | 2008-02-21 16:05:53 +0000 | [diff] [blame] | 254 | self.warning_stream = os.fdopen(r, 'r', 0) |
mbligh | 6607d19 | 2008-04-17 15:23:15 +0000 | [diff] [blame] | 255 | if self.job: |
| 256 | self.job.warning_loggers.add(self.warning_stream) |
mbligh | f4e0415 | 2008-02-21 16:05:53 +0000 | [diff] [blame] | 257 | self.logger_popen = subprocess.Popen(cmd, stderr=dev_null) |
| 258 | os.close(w) |
mbligh | e6c995f | 2007-10-26 19:43:01 +0000 | [diff] [blame] | 259 | |
| 260 | |
mbligh | 3409ee7 | 2007-10-16 23:58:33 +0000 | [diff] [blame] | 261 | def __console_run(self, cmd): |
| 262 | """ |
| 263 | Send a command to the conmux session |
| 264 | """ |
| 265 | if not self.conmux_attach or not os.path.exists(self.conmux_attach): |
| 266 | return False |
mbligh | 3409ee7 | 2007-10-16 23:58:33 +0000 | [diff] [blame] | 267 | cmd = '%s %s echo %s 2> /dev/null' % (self.conmux_attach, |
mbligh | e6c995f | 2007-10-26 19:43:01 +0000 | [diff] [blame] | 268 | self.__conmux_hostname(), |
mbligh | 3409ee7 | 2007-10-16 23:58:33 +0000 | [diff] [blame] | 269 | cmd) |
mbligh | 0f5ad64 | 2008-01-22 16:37:40 +0000 | [diff] [blame] | 270 | result = utils.system(cmd, ignore_status=True) |
mbligh | 3409ee7 | 2007-10-16 23:58:33 +0000 | [diff] [blame] | 271 | return result == 0 |
mbligh | 7d2bde8 | 2007-08-02 16:26:10 +0000 | [diff] [blame] | 272 | |
| 273 | |
jadmanski | 6e8bf75 | 2008-05-14 00:17:48 +0000 | [diff] [blame^] | 274 | def __run_group(self, function, *args, **dargs): |
| 275 | if self.job: |
| 276 | self.job.run_group(function, *args, **dargs) |
| 277 | else: |
| 278 | function(*args, **dargs) |
| 279 | |
| 280 | |
mbligh | 31a49de | 2007-11-05 18:41:19 +0000 | [diff] [blame] | 281 | def __record(self, status_code, subdir, operation, status = ''): |
| 282 | if self.job: |
| 283 | self.job.record(status_code, subdir, operation, status) |
| 284 | else: |
| 285 | if not subdir: |
| 286 | subdir = "----" |
| 287 | msg = "%s\t%s\t%s\t%s" % (status_code, subdir, operation, status) |
| 288 | sys.stderr.write(msg + "\n") |
| 289 | |
| 290 | |
mbligh | fa97160 | 2008-01-03 01:57:20 +0000 | [diff] [blame] | 291 | def ssh_base_command(self, connect_timeout=30): |
| 292 | SSH_BASE_COMMAND = '/usr/bin/ssh -a -x -o ' + \ |
mbligh | 0ad21ba | 2008-03-14 15:06:21 +0000 | [diff] [blame] | 293 | 'BatchMode=yes -o ConnectTimeout=%d ' + \ |
| 294 | '-o ServerAliveInterval=300' |
mbligh | fa97160 | 2008-01-03 01:57:20 +0000 | [diff] [blame] | 295 | assert isinstance(connect_timeout, (int, long)) |
| 296 | assert connect_timeout > 0 # can't disable the timeout |
| 297 | return SSH_BASE_COMMAND % connect_timeout |
| 298 | |
| 299 | |
| 300 | def ssh_command(self, connect_timeout=30): |
mbligh | e6647d1 | 2007-10-17 00:00:01 +0000 | [diff] [blame] | 301 | """Construct an ssh command with proper args for this host.""" |
mbligh | fa97160 | 2008-01-03 01:57:20 +0000 | [diff] [blame] | 302 | ssh = self.ssh_base_command(connect_timeout) |
| 303 | return r'%s -l %s -p %d %s' % (ssh, |
mbligh | 0faf91f | 2007-10-18 03:10:48 +0000 | [diff] [blame] | 304 | self.user, |
| 305 | self.port, |
| 306 | self.hostname) |
mbligh | e6647d1 | 2007-10-17 00:00:01 +0000 | [diff] [blame] | 307 | |
| 308 | |
mbligh | 07a923f | 2008-01-16 17:49:04 +0000 | [diff] [blame] | 309 | def run(self, command, timeout=3600, ignore_status=False, |
mbligh | fa97160 | 2008-01-03 01:57:20 +0000 | [diff] [blame] | 310 | stdout_tee=None, stderr_tee=None, connect_timeout=30): |
mbligh | 7d2bde8 | 2007-08-02 16:26:10 +0000 | [diff] [blame] | 311 | """ |
| 312 | Run a command on the remote host. |
mbligh | dcd57a8 | 2007-07-11 23:06:47 +0000 | [diff] [blame] | 313 | |
| 314 | Args: |
| 315 | command: the command line string |
| 316 | timeout: time limit in seconds before attempting to |
| 317 | kill the running process. The run() function |
| 318 | will take a few seconds longer than 'timeout' |
| 319 | to complete if it has to kill the process. |
mbligh | 8b85dfb | 2007-08-28 09:50:31 +0000 | [diff] [blame] | 320 | ignore_status: do not raise an exception, no matter |
| 321 | what the exit code of the command is. |
mbligh | dcd57a8 | 2007-07-11 23:06:47 +0000 | [diff] [blame] | 322 | |
| 323 | Returns: |
| 324 | a hosts.base_classes.CmdResult object |
| 325 | |
| 326 | Raises: |
| 327 | AutoservRunError: the exit code of the command |
| 328 | execution was not 0 |
mbligh | caa62c2 | 2008-04-07 21:51:17 +0000 | [diff] [blame] | 329 | AutoservSSHTimeout: ssh connection has timed out |
mbligh | dcd57a8 | 2007-07-11 23:06:47 +0000 | [diff] [blame] | 330 | """ |
mbligh | adf2aab | 2007-11-29 18:16:43 +0000 | [diff] [blame] | 331 | stdout = stdout_tee or sys.stdout |
mbligh | 8d4baaa | 2008-03-12 14:48:24 +0000 | [diff] [blame] | 332 | stderr = stderr_tee or sys.stdout |
mbligh | 7995cc6 | 2007-11-30 15:53:23 +0000 | [diff] [blame] | 333 | print "ssh: %s" % (command,) |
mbligh | adf2aab | 2007-11-29 18:16:43 +0000 | [diff] [blame] | 334 | env = " ".join("=".join(pair) for pair in self.env.iteritems()) |
mbligh | 34faa28 | 2008-01-16 17:44:49 +0000 | [diff] [blame] | 335 | full_cmd = '%s "%s %s"' % (self.ssh_command(connect_timeout), |
| 336 | env, utils.sh_escape(command)) |
mbligh | 8ea61e2 | 2008-05-09 18:09:37 +0000 | [diff] [blame] | 337 | try: |
| 338 | result = utils.run(full_cmd, timeout, True, stdout, stderr) |
| 339 | # we get a CmdError here only if there is timeout of that command. |
| 340 | # Catch that and stuff it into AutoservRunError and raise it. |
| 341 | except error.CmdError, cmderr: |
| 342 | raise error.AutoservRunError(cmderr.args[0], cmderr.args[1]) |
| 343 | |
mbligh | 34faa28 | 2008-01-16 17:44:49 +0000 | [diff] [blame] | 344 | if result.exit_status == 255: # ssh's exit status for timeout |
| 345 | if re.match(r'^ssh: connect to host .* port .*: ' + |
| 346 | r'Connection timed out\r$', result.stderr): |
mbligh | f5427bb | 2008-04-09 15:55:57 +0000 | [diff] [blame] | 347 | raise error.AutoservSSHTimeout("ssh timed out", |
| 348 | result) |
mbligh | 34faa28 | 2008-01-16 17:44:49 +0000 | [diff] [blame] | 349 | if not ignore_status and result.exit_status > 0: |
mbligh | f5427bb | 2008-04-09 15:55:57 +0000 | [diff] [blame] | 350 | raise error.AutoservRunError("command execution error", |
| 351 | result) |
mbligh | dcd57a8 | 2007-07-11 23:06:47 +0000 | [diff] [blame] | 352 | return result |
mbligh | 7d2bde8 | 2007-08-02 16:26:10 +0000 | [diff] [blame] | 353 | |
| 354 | |
mbligh | bda9c9c | 2008-04-08 17:45:00 +0000 | [diff] [blame] | 355 | def run_short(self, command, **kwargs): |
| 356 | """ |
| 357 | Calls the run() command with a short default timeout. |
| 358 | |
| 359 | Args: |
| 360 | Takes the same arguments as does run(), |
| 361 | with the exception of the timeout argument which |
| 362 | here is fixed at 60 seconds. |
| 363 | It returns the result of run. |
| 364 | """ |
| 365 | return self.run(command, timeout=60, **kwargs) |
| 366 | |
| 367 | |
mbligh | 78669ff | 2008-01-10 16:33:07 +0000 | [diff] [blame] | 368 | def run_grep(self, command, timeout=30, ignore_status=False, |
| 369 | stdout_ok_regexp=None, stdout_err_regexp=None, |
| 370 | stderr_ok_regexp=None, stderr_err_regexp=None, |
| 371 | connect_timeout=30): |
| 372 | """ |
| 373 | Run a command on the remote host and look for regexp |
| 374 | in stdout or stderr to determine if the command was |
| 375 | successul or not. |
mbligh | 6a2a2df | 2008-01-16 17:41:55 +0000 | [diff] [blame] | 376 | |
mbligh | 78669ff | 2008-01-10 16:33:07 +0000 | [diff] [blame] | 377 | Args: |
| 378 | command: the command line string |
mbligh | 6a2a2df | 2008-01-16 17:41:55 +0000 | [diff] [blame] | 379 | timeout: time limit in seconds before attempting to |
mbligh | 78669ff | 2008-01-10 16:33:07 +0000 | [diff] [blame] | 380 | kill the running process. The run() function |
| 381 | will take a few seconds longer than 'timeout' |
| 382 | to complete if it has to kill the process. |
mbligh | 6a2a2df | 2008-01-16 17:41:55 +0000 | [diff] [blame] | 383 | ignore_status: do not raise an exception, no matter |
mbligh | 78669ff | 2008-01-10 16:33:07 +0000 | [diff] [blame] | 384 | what the exit code of the command is. |
| 385 | stdout_ok_regexp: regexp that should be in stdout |
| 386 | if the command was successul. |
| 387 | stdout_err_regexp: regexp that should be in stdout |
| 388 | if the command failed. |
| 389 | stderr_ok_regexp: regexp that should be in stderr |
| 390 | if the command was successul. |
| 391 | stderr_err_regexp: regexp that should be in stderr |
| 392 | if the command failed. |
mbligh | 6a2a2df | 2008-01-16 17:41:55 +0000 | [diff] [blame] | 393 | |
mbligh | 78669ff | 2008-01-10 16:33:07 +0000 | [diff] [blame] | 394 | Returns: |
| 395 | if the command was successul, raises an exception |
| 396 | otherwise. |
mbligh | 6a2a2df | 2008-01-16 17:41:55 +0000 | [diff] [blame] | 397 | |
mbligh | 78669ff | 2008-01-10 16:33:07 +0000 | [diff] [blame] | 398 | Raises: |
| 399 | AutoservRunError: |
| 400 | - the exit code of the command execution was not 0. |
mbligh | 6a2a2df | 2008-01-16 17:41:55 +0000 | [diff] [blame] | 401 | - If stderr_err_regexp is found in stderr, |
| 402 | - If stdout_err_regexp is found in stdout, |
mbligh | 78669ff | 2008-01-10 16:33:07 +0000 | [diff] [blame] | 403 | - If stderr_ok_regexp is not found in stderr. |
| 404 | - If stdout_ok_regexp is not found in stdout, |
| 405 | """ |
| 406 | |
| 407 | # We ignore the status, because we will handle it at the end. |
| 408 | result = self.run(command, timeout, ignore_status=True, |
mbligh | 6a2a2df | 2008-01-16 17:41:55 +0000 | [diff] [blame] | 409 | connect_timeout=connect_timeout) |
mbligh | 78669ff | 2008-01-10 16:33:07 +0000 | [diff] [blame] | 410 | |
| 411 | # Look for the patterns, in order |
| 412 | for (regexp, stream) in ((stderr_err_regexp, result.stderr), |
mbligh | 6a2a2df | 2008-01-16 17:41:55 +0000 | [diff] [blame] | 413 | (stdout_err_regexp, result.stdout)): |
mbligh | 78669ff | 2008-01-10 16:33:07 +0000 | [diff] [blame] | 414 | if regexp and stream: |
| 415 | err_re = re.compile (regexp) |
| 416 | if err_re.search(stream): |
mbligh | f5427bb | 2008-04-09 15:55:57 +0000 | [diff] [blame] | 417 | raise error.AutoservRunError( |
mbligh | 6a2a2df | 2008-01-16 17:41:55 +0000 | [diff] [blame] | 418 | '%s failed, found error pattern: ' |
| 419 | '"%s"' % (command, regexp), result) |
mbligh | 78669ff | 2008-01-10 16:33:07 +0000 | [diff] [blame] | 420 | |
| 421 | for (regexp, stream) in ((stderr_ok_regexp, result.stderr), |
mbligh | 6a2a2df | 2008-01-16 17:41:55 +0000 | [diff] [blame] | 422 | (stdout_ok_regexp, result.stdout)): |
mbligh | 78669ff | 2008-01-10 16:33:07 +0000 | [diff] [blame] | 423 | if regexp and stream: |
| 424 | ok_re = re.compile (regexp) |
| 425 | if ok_re.search(stream): |
| 426 | if ok_re.search(stream): |
| 427 | return |
| 428 | |
| 429 | if not ignore_status and result.exit_status > 0: |
mbligh | f5427bb | 2008-04-09 15:55:57 +0000 | [diff] [blame] | 430 | raise error.AutoservRunError("command execution error", |
| 431 | result) |
mbligh | 78669ff | 2008-01-10 16:33:07 +0000 | [diff] [blame] | 432 | |
| 433 | |
mbligh | 80d2077 | 2007-10-29 17:10:10 +0000 | [diff] [blame] | 434 | def reboot(self, timeout=DEFAULT_REBOOT_TIMEOUT, label=None, |
| 435 | kernel_args=None, wait=True): |
mbligh | 7d2bde8 | 2007-08-02 16:26:10 +0000 | [diff] [blame] | 436 | """ |
| 437 | Reboot the remote host. |
mbligh | 8b85dfb | 2007-08-28 09:50:31 +0000 | [diff] [blame] | 438 | |
mbligh | a0452c8 | 2007-08-08 20:24:57 +0000 | [diff] [blame] | 439 | Args: |
| 440 | timeout |
mbligh | 8b85dfb | 2007-08-28 09:50:31 +0000 | [diff] [blame] | 441 | """ |
mbligh | 33ae090 | 2007-11-24 19:27:08 +0000 | [diff] [blame] | 442 | self.reboot_setup() |
| 443 | |
mbligh | de38437 | 2007-10-17 04:25:37 +0000 | [diff] [blame] | 444 | # forcibly include the "netconsole" kernel arg |
| 445 | if self.__netconsole_param: |
| 446 | if kernel_args is None: |
| 447 | kernel_args = self.__netconsole_param |
| 448 | else: |
| 449 | kernel_args += " " + self.__netconsole_param |
| 450 | # unload the (possibly loaded) module to avoid shutdown issues |
| 451 | self.__unload_netconsole_module() |
mbligh | a0452c8 | 2007-08-08 20:24:57 +0000 | [diff] [blame] | 452 | if label or kernel_args: |
| 453 | self.bootloader.install_boottool() |
| 454 | if label: |
| 455 | self.bootloader.set_default(label) |
| 456 | if kernel_args: |
| 457 | if not label: |
| 458 | default = int(self.bootloader.get_default()) |
| 459 | label = self.bootloader.get_titles()[default] |
| 460 | self.bootloader.add_args(label, kernel_args) |
jadmanski | 6e8bf75 | 2008-05-14 00:17:48 +0000 | [diff] [blame^] | 461 | |
| 462 | # define a function for the reboot and run it in a group |
mbligh | d742a22 | 2007-09-30 01:27:06 +0000 | [diff] [blame] | 463 | print "Reboot: initiating reboot" |
jadmanski | 6e8bf75 | 2008-05-14 00:17:48 +0000 | [diff] [blame^] | 464 | def reboot(): |
| 465 | self.__record("GOOD", None, "reboot.start") |
| 466 | try: |
| 467 | self.run('(sleep 5; reboot) ' |
| 468 | '</dev/null >/dev/null 2>&1 &') |
| 469 | except error.AutoservRunError: |
| 470 | self.__record("ABORT", None, "reboot.start", |
| 471 | "reboot command failed") |
| 472 | raise |
| 473 | if wait: |
| 474 | self.wait_for_restart(timeout) |
| 475 | self.reboot_followup() |
| 476 | self.__run_group(reboot) |
mbligh | 7f2befb | 2008-04-21 20:47:10 +0000 | [diff] [blame] | 477 | |
| 478 | |
| 479 | def reboot_followup(self): |
| 480 | super(SSHHost, self).reboot_followup() |
| 481 | self.__load_netconsole_module() # if the builtin fails |
mbligh | a0452c8 | 2007-08-08 20:24:57 +0000 | [diff] [blame] | 482 | |
mbligh | 7d2bde8 | 2007-08-02 16:26:10 +0000 | [diff] [blame] | 483 | |
mbligh | cfc7ab3 | 2008-01-25 16:35:28 +0000 | [diff] [blame] | 484 | def __copy_files(self, sources, dest): |
| 485 | """ |
| 486 | Copy files from one machine to another. |
| 487 | |
| 488 | This is for internal use by other methods that intend to move |
| 489 | files between machines. It expects a list of source files and |
| 490 | a destination (a filename if the source is a single file, a |
| 491 | destination otherwise). The names must already be |
| 492 | pre-processed into the appropriate rsync/scp friendly |
| 493 | format (%s@%s:%s). |
| 494 | """ |
| 495 | # wait until there are only a small number of copies running |
| 496 | # before starting this one |
| 497 | MAXIMUM_SIMULTANEOUS_COPIES = 4 |
| 498 | while True: |
| 499 | copy_count = 0 |
| 500 | procs = utils.system_output('ps -ef') |
| 501 | for line in procs: |
| 502 | if 'rsync ' in line or 'scp ' in line: |
| 503 | copy_count += 1 |
| 504 | if copy_count < MAXIMUM_SIMULTANEOUS_COPIES: |
| 505 | break |
| 506 | time.sleep(60) |
| 507 | |
mbligh | 22fdf17 | 2008-04-07 18:34:56 +0000 | [diff] [blame] | 508 | print '__copy_files: copying %s to %s' % (sources, dest) |
mbligh | cfc7ab3 | 2008-01-25 16:35:28 +0000 | [diff] [blame] | 509 | try: |
| 510 | utils.run('rsync --rsh="%s" -az %s %s' % ( |
| 511 | self.ssh_base_command(), ' '.join(sources), dest)) |
| 512 | except Exception: |
| 513 | utils.run('scp -rpq %s "%s"' % ( |
| 514 | ' '.join(sources), dest)) |
| 515 | |
| 516 | |
mbligh | dcd57a8 | 2007-07-11 23:06:47 +0000 | [diff] [blame] | 517 | def get_file(self, source, dest): |
mbligh | 7d2bde8 | 2007-08-02 16:26:10 +0000 | [diff] [blame] | 518 | """ |
| 519 | Copy files from the remote host to a local path. |
mbligh | dcd57a8 | 2007-07-11 23:06:47 +0000 | [diff] [blame] | 520 | |
| 521 | Directories will be copied recursively. |
| 522 | If a source component is a directory with a trailing slash, |
| 523 | the content of the directory will be copied, otherwise, the |
| 524 | directory itself and its content will be copied. This |
| 525 | behavior is similar to that of the program 'rsync'. |
| 526 | |
| 527 | Args: |
| 528 | source: either |
| 529 | 1) a single file or directory, as a string |
| 530 | 2) a list of one or more (possibly mixed) |
| 531 | files or directories |
| 532 | dest: a file or a directory (if source contains a |
| 533 | directory or more than one element, you must |
| 534 | supply a directory dest) |
| 535 | |
| 536 | Raises: |
| 537 | AutoservRunError: the scp command failed |
| 538 | """ |
| 539 | if isinstance(source, types.StringTypes): |
| 540 | source= [source] |
| 541 | |
| 542 | processed_source= [] |
| 543 | for entry in source: |
| 544 | if entry.endswith('/'): |
| 545 | format_string= '%s@%s:"%s*"' |
| 546 | else: |
| 547 | format_string= '%s@%s:"%s"' |
| 548 | entry= format_string % (self.user, self.hostname, |
| 549 | utils.scp_remote_escape(entry)) |
| 550 | processed_source.append(entry) |
| 551 | |
| 552 | processed_dest= os.path.abspath(dest) |
| 553 | if os.path.isdir(dest): |
| 554 | processed_dest= "%s/" % (utils.sh_escape(processed_dest),) |
| 555 | else: |
| 556 | processed_dest= utils.sh_escape(processed_dest) |
mbligh | cfc7ab3 | 2008-01-25 16:35:28 +0000 | [diff] [blame] | 557 | |
| 558 | self.__copy_files(processed_source, processed_dest) |
mbligh | 7d2bde8 | 2007-08-02 16:26:10 +0000 | [diff] [blame] | 559 | |
| 560 | |
mbligh | dcd57a8 | 2007-07-11 23:06:47 +0000 | [diff] [blame] | 561 | def send_file(self, source, dest): |
mbligh | 7d2bde8 | 2007-08-02 16:26:10 +0000 | [diff] [blame] | 562 | """ |
| 563 | Copy files from a local path to the remote host. |
mbligh | dcd57a8 | 2007-07-11 23:06:47 +0000 | [diff] [blame] | 564 | |
| 565 | Directories will be copied recursively. |
| 566 | If a source component is a directory with a trailing slash, |
| 567 | the content of the directory will be copied, otherwise, the |
| 568 | directory itself and its content will be copied. This |
| 569 | behavior is similar to that of the program 'rsync'. |
| 570 | |
| 571 | Args: |
| 572 | source: either |
| 573 | 1) a single file or directory, as a string |
| 574 | 2) a list of one or more (possibly mixed) |
| 575 | files or directories |
| 576 | dest: a file or a directory (if source contains a |
| 577 | directory or more than one element, you must |
| 578 | supply a directory dest) |
| 579 | |
| 580 | Raises: |
| 581 | AutoservRunError: the scp command failed |
| 582 | """ |
| 583 | if isinstance(source, types.StringTypes): |
| 584 | source= [source] |
| 585 | |
| 586 | processed_source= [] |
| 587 | for entry in source: |
| 588 | if entry.endswith('/'): |
| 589 | format_string= '"%s/"*' |
| 590 | else: |
| 591 | format_string= '"%s"' |
| 592 | entry= format_string % (utils.sh_escape(os.path.abspath(entry)),) |
| 593 | processed_source.append(entry) |
mbligh | 7d2bde8 | 2007-08-02 16:26:10 +0000 | [diff] [blame] | 594 | |
mbligh | 0faf91f | 2007-10-18 03:10:48 +0000 | [diff] [blame] | 595 | remote_dest = '%s@%s:"%s"' % ( |
| 596 | self.user, self.hostname, |
| 597 | utils.scp_remote_escape(dest)) |
mbligh | cfc7ab3 | 2008-01-25 16:35:28 +0000 | [diff] [blame] | 598 | |
| 599 | self.__copy_files(processed_source, remote_dest) |
mbligh | a4eb0fa | 2008-04-11 15:16:50 +0000 | [diff] [blame] | 600 | self.run('find "%s" -type d | xargs -i -r chmod o+rx "{}"' % dest) |
| 601 | self.run('find "%s" -type f | xargs -i -r chmod o+r "{}"' % dest) |
mbligh | 7d2bde8 | 2007-08-02 16:26:10 +0000 | [diff] [blame] | 602 | |
mbligh | dcd57a8 | 2007-07-11 23:06:47 +0000 | [diff] [blame] | 603 | def get_tmp_dir(self): |
mbligh | 7d2bde8 | 2007-08-02 16:26:10 +0000 | [diff] [blame] | 604 | """ |
| 605 | Return the pathname of a directory on the host suitable |
mbligh | dcd57a8 | 2007-07-11 23:06:47 +0000 | [diff] [blame] | 606 | for temporary file storage. |
| 607 | |
| 608 | The directory and its content will be deleted automatically |
| 609 | on the destruction of the Host object that was used to obtain |
| 610 | it. |
| 611 | """ |
mbligh | a25b29e | 2007-08-26 13:58:04 +0000 | [diff] [blame] | 612 | dir_name= self.run("mktemp -d /tmp/autoserv-XXXXXX").stdout.rstrip(" \n") |
mbligh | dcd57a8 | 2007-07-11 23:06:47 +0000 | [diff] [blame] | 613 | self.tmp_dirs.append(dir_name) |
| 614 | return dir_name |
mbligh | 7d2bde8 | 2007-08-02 16:26:10 +0000 | [diff] [blame] | 615 | |
| 616 | |
mbligh | dcd57a8 | 2007-07-11 23:06:47 +0000 | [diff] [blame] | 617 | def is_up(self): |
mbligh | 7d2bde8 | 2007-08-02 16:26:10 +0000 | [diff] [blame] | 618 | """ |
| 619 | Check if the remote host is up. |
mbligh | dcd57a8 | 2007-07-11 23:06:47 +0000 | [diff] [blame] | 620 | |
| 621 | Returns: |
| 622 | True if the remote host is up, False otherwise |
| 623 | """ |
| 624 | try: |
mbligh | 4cfa76a | 2007-11-26 20:45:16 +0000 | [diff] [blame] | 625 | self.ssh_ping() |
mbligh | eadfbb1 | 2007-11-26 23:03:12 +0000 | [diff] [blame] | 626 | except: |
mbligh | dcd57a8 | 2007-07-11 23:06:47 +0000 | [diff] [blame] | 627 | return False |
mbligh | eadfbb1 | 2007-11-26 23:03:12 +0000 | [diff] [blame] | 628 | return True |
mbligh | 7d2bde8 | 2007-08-02 16:26:10 +0000 | [diff] [blame] | 629 | |
mbligh | 7d2bde8 | 2007-08-02 16:26:10 +0000 | [diff] [blame] | 630 | |
mbligh | dcd57a8 | 2007-07-11 23:06:47 +0000 | [diff] [blame] | 631 | def wait_up(self, timeout=None): |
mbligh | 7d2bde8 | 2007-08-02 16:26:10 +0000 | [diff] [blame] | 632 | """ |
| 633 | Wait until the remote host is up or the timeout expires. |
mbligh | dcd57a8 | 2007-07-11 23:06:47 +0000 | [diff] [blame] | 634 | |
| 635 | In fact, it will wait until an ssh connection to the remote |
| 636 | host can be established. |
| 637 | |
| 638 | Args: |
| 639 | timeout: time limit in seconds before returning even |
| 640 | if the host is not up. |
| 641 | |
| 642 | Returns: |
| 643 | True if the host was found to be up, False otherwise |
| 644 | """ |
| 645 | if timeout: |
| 646 | end_time= time.time() + timeout |
| 647 | |
| 648 | while not timeout or time.time() < end_time: |
| 649 | try: |
mbligh | 4cfa76a | 2007-11-26 20:45:16 +0000 | [diff] [blame] | 650 | self.ssh_ping() |
mbligh | eadfbb1 | 2007-11-26 23:03:12 +0000 | [diff] [blame] | 651 | except: |
mbligh | dcd57a8 | 2007-07-11 23:06:47 +0000 | [diff] [blame] | 652 | pass |
| 653 | else: |
mbligh | eadfbb1 | 2007-11-26 23:03:12 +0000 | [diff] [blame] | 654 | return True |
mbligh | dcd57a8 | 2007-07-11 23:06:47 +0000 | [diff] [blame] | 655 | time.sleep(1) |
| 656 | |
| 657 | return False |
mbligh | 7d2bde8 | 2007-08-02 16:26:10 +0000 | [diff] [blame] | 658 | |
| 659 | |
mbligh | dcd57a8 | 2007-07-11 23:06:47 +0000 | [diff] [blame] | 660 | def wait_down(self, timeout=None): |
mbligh | 7d2bde8 | 2007-08-02 16:26:10 +0000 | [diff] [blame] | 661 | """ |
| 662 | Wait until the remote host is down or the timeout expires. |
mbligh | dcd57a8 | 2007-07-11 23:06:47 +0000 | [diff] [blame] | 663 | |
| 664 | In fact, it will wait until an ssh connection to the remote |
| 665 | host fails. |
| 666 | |
| 667 | Args: |
| 668 | timeout: time limit in seconds before returning even |
| 669 | if the host is not up. |
| 670 | |
| 671 | Returns: |
| 672 | True if the host was found to be down, False otherwise |
| 673 | """ |
| 674 | if timeout: |
| 675 | end_time= time.time() + timeout |
| 676 | |
| 677 | while not timeout or time.time() < end_time: |
| 678 | try: |
mbligh | 4cfa76a | 2007-11-26 20:45:16 +0000 | [diff] [blame] | 679 | self.ssh_ping() |
mbligh | eadfbb1 | 2007-11-26 23:03:12 +0000 | [diff] [blame] | 680 | except: |
mbligh | dcd57a8 | 2007-07-11 23:06:47 +0000 | [diff] [blame] | 681 | return True |
mbligh | dcd57a8 | 2007-07-11 23:06:47 +0000 | [diff] [blame] | 682 | time.sleep(1) |
| 683 | |
| 684 | return False |
mbligh | 7d2bde8 | 2007-08-02 16:26:10 +0000 | [diff] [blame] | 685 | |
| 686 | |
mbligh | dbe4a38 | 2007-07-26 19:41:28 +0000 | [diff] [blame] | 687 | def ensure_up(self): |
mbligh | 7d2bde8 | 2007-08-02 16:26:10 +0000 | [diff] [blame] | 688 | """ |
| 689 | Ensure the host is up if it is not then do not proceed; |
| 690 | this prevents cacading failures of tests |
| 691 | """ |
mbligh | a0452c8 | 2007-08-08 20:24:57 +0000 | [diff] [blame] | 692 | print 'Ensuring that %s is up before continuing' % self.hostname |
| 693 | if hasattr(self, 'hardreset') and not self.wait_up(300): |
mbligh | dbe4a38 | 2007-07-26 19:41:28 +0000 | [diff] [blame] | 694 | print "Performing a hardreset on %s" % self.hostname |
mbligh | 4ba0b46 | 2007-11-05 23:05:40 +0000 | [diff] [blame] | 695 | try: |
| 696 | self.hardreset() |
mbligh | f5427bb | 2008-04-09 15:55:57 +0000 | [diff] [blame] | 697 | except error.AutoservUnsupportedError: |
mbligh | 4ba0b46 | 2007-11-05 23:05:40 +0000 | [diff] [blame] | 698 | print "Hardreset is unsupported on %s" % self.hostname |
mbligh | a9563b9 | 2007-10-25 14:45:56 +0000 | [diff] [blame] | 699 | if not self.wait_up(60 * 30): |
| 700 | # 30 minutes should be more than enough |
mbligh | f5427bb | 2008-04-09 15:55:57 +0000 | [diff] [blame] | 701 | raise error.AutoservHostError |
mbligh | a0452c8 | 2007-08-08 20:24:57 +0000 | [diff] [blame] | 702 | print 'Host up, continuing' |
mbligh | 7d2bde8 | 2007-08-02 16:26:10 +0000 | [diff] [blame] | 703 | |
| 704 | |
mbligh | dcd57a8 | 2007-07-11 23:06:47 +0000 | [diff] [blame] | 705 | def get_num_cpu(self): |
mbligh | 7d2bde8 | 2007-08-02 16:26:10 +0000 | [diff] [blame] | 706 | """ |
| 707 | Get the number of CPUs in the host according to |
mbligh | dcd57a8 | 2007-07-11 23:06:47 +0000 | [diff] [blame] | 708 | /proc/cpuinfo. |
| 709 | |
| 710 | Returns: |
| 711 | The number of CPUs |
| 712 | """ |
| 713 | |
mbligh | 0ba3579 | 2008-04-15 19:16:11 +0000 | [diff] [blame] | 714 | proc_cpuinfo = self.run("cat /proc/cpuinfo", |
| 715 | stdout_tee=open('/dev/null', 'w')).stdout |
mbligh | dcd57a8 | 2007-07-11 23:06:47 +0000 | [diff] [blame] | 716 | cpus = 0 |
| 717 | for line in proc_cpuinfo.splitlines(): |
| 718 | if line.startswith('processor'): |
| 719 | cpus += 1 |
| 720 | return cpus |
mbligh | 5f876ad | 2007-10-12 23:59:53 +0000 | [diff] [blame] | 721 | |
| 722 | |
| 723 | def check_uptime(self): |
| 724 | """ |
| 725 | Check that uptime is available and monotonically increasing. |
| 726 | """ |
| 727 | if not self.ping(): |
mbligh | f5427bb | 2008-04-09 15:55:57 +0000 | [diff] [blame] | 728 | raise error.AutoservHostError('Client is not pingable') |
mbligh | 5f876ad | 2007-10-12 23:59:53 +0000 | [diff] [blame] | 729 | result = self.run("/bin/cat /proc/uptime", 30) |
| 730 | return result.stdout.strip().split()[0] |
| 731 | |
| 732 | |
| 733 | def get_arch(self): |
| 734 | """ |
| 735 | Get the hardware architecture of the remote machine |
| 736 | """ |
| 737 | arch = self.run('/bin/uname -m').stdout.rstrip() |
| 738 | if re.match(r'i\d86$', arch): |
| 739 | arch = 'i386' |
| 740 | return arch |
| 741 | |
| 742 | |
| 743 | def get_kernel_ver(self): |
| 744 | """ |
| 745 | Get the kernel version of the remote machine |
| 746 | """ |
| 747 | return self.run('/bin/uname -r').stdout.rstrip() |
| 748 | |
| 749 | |
| 750 | def get_cmdline(self): |
| 751 | """ |
| 752 | Get the kernel command line of the remote machine |
| 753 | """ |
| 754 | return self.run('cat /proc/cmdline').stdout.rstrip() |
| 755 | |
| 756 | |
| 757 | def ping(self): |
| 758 | """ |
| 759 | Ping the remote system, and return whether it's available |
| 760 | """ |
| 761 | fpingcmd = "%s -q %s" % ('/usr/bin/fping', self.hostname) |
| 762 | rc = utils.system(fpingcmd, ignore_status = 1) |
| 763 | return (rc == 0) |
mbligh | d2e4605 | 2007-11-05 18:31:00 +0000 | [diff] [blame] | 764 | |
mbligh | f014ff4 | 2007-11-26 21:33:11 +0000 | [diff] [blame] | 765 | |
mbligh | 4cfa76a | 2007-11-26 20:45:16 +0000 | [diff] [blame] | 766 | def ssh_ping(self, timeout = 60): |
mbligh | 21d7e26 | 2008-05-02 23:04:50 +0000 | [diff] [blame] | 767 | try: |
| 768 | self.run('true', timeout = timeout, connect_timeout = timeout) |
| 769 | except error.AutoservSSHTimeout: |
| 770 | msg = "ssh ping timed out. timeout = %s" % timeout |
| 771 | raise error.AutoservSSHTimeout(msg) |
jadmanski | 4c6f15e | 2008-05-06 20:36:09 +0000 | [diff] [blame] | 772 | except error.AutoservRunError, exc: |
mbligh | 21d7e26 | 2008-05-02 23:04:50 +0000 | [diff] [blame] | 773 | msg = "command true failed in ssh ping" |
jadmanski | 4c6f15e | 2008-05-06 20:36:09 +0000 | [diff] [blame] | 774 | raise error.AutoservRunError(msg, exc.args[1]) |
mbligh | da13d54 | 2008-01-03 16:28:34 +0000 | [diff] [blame] | 775 | |
| 776 | |
| 777 | def get_autodir(self): |
| 778 | return self.autodir |