Rahul Chaudhry | 8ac4ec0 | 2014-11-01 09:31:15 -0700 | [diff] [blame] | 1 | # Copyright 2011 The Chromium OS Authors. All rights reserved. |
| 2 | # Use of this source code is governed by a BSD-style license that can be |
| 3 | # found in the LICENSE file. |
Ahmad Sharif | fd356fb | 2012-05-07 12:02:16 -0700 | [diff] [blame] | 4 | |
Luis Lozano | bd44704 | 2015-10-01 14:24:27 -0700 | [diff] [blame] | 5 | """Utilities to run commands in outside/inside chroot and on the board.""" |
| 6 | |
Ahmad Sharif | 70de27b | 2011-06-15 17:51:24 -0700 | [diff] [blame] | 7 | import getpass |
Ahmad Sharif | 70de27b | 2011-06-15 17:51:24 -0700 | [diff] [blame] | 8 | import os |
Ahmad Sharif | 70de27b | 2011-06-15 17:51:24 -0700 | [diff] [blame] | 9 | import re |
| 10 | import select |
Luis Lozano | 45b53c5 | 2015-09-30 11:36:27 -0700 | [diff] [blame] | 11 | import signal |
Ahmad Sharif | 70de27b | 2011-06-15 17:51:24 -0700 | [diff] [blame] | 12 | import subprocess |
Luis Lozano | bd44704 | 2015-10-01 14:24:27 -0700 | [diff] [blame] | 13 | import sys |
Ahmad Sharif | 0dcbc4b | 2012-02-02 16:37:18 -0800 | [diff] [blame] | 14 | import tempfile |
Ahmad Sharif | 70de27b | 2011-06-15 17:51:24 -0700 | [diff] [blame] | 15 | import time |
Ahmad Sharif | fd356fb | 2012-05-07 12:02:16 -0700 | [diff] [blame] | 16 | |
| 17 | import logger |
| 18 | import misc |
Ahmad Sharif | 70de27b | 2011-06-15 17:51:24 -0700 | [diff] [blame] | 19 | |
| 20 | mock_default = False |
| 21 | |
Luis Lozano | bd44704 | 2015-10-01 14:24:27 -0700 | [diff] [blame] | 22 | LOG_LEVEL = ("none", "quiet", "average", "verbose") |
Ahmad Sharif | 70de27b | 2011-06-15 17:51:24 -0700 | [diff] [blame] | 23 | |
| 24 | def InitCommandExecuter(mock=False): |
| 25 | global mock_default |
| 26 | # Whether to default to a mock command executer or not |
| 27 | mock_default = mock |
| 28 | |
| 29 | |
cmtice | 1390924 | 2014-03-11 13:38:07 -0700 | [diff] [blame] | 30 | def GetCommandExecuter(logger_to_set=None, mock=False, log_level="verbose"): |
Ahmad Sharif | 70de27b | 2011-06-15 17:51:24 -0700 | [diff] [blame] | 31 | # If the default is a mock executer, always return one. |
| 32 | if mock_default or mock: |
Luis Lozano | bd44704 | 2015-10-01 14:24:27 -0700 | [diff] [blame] | 33 | return MockCommandExecuter(log_level, logger_to_set) |
Ahmad Sharif | 70de27b | 2011-06-15 17:51:24 -0700 | [diff] [blame] | 34 | else: |
cmtice | 1390924 | 2014-03-11 13:38:07 -0700 | [diff] [blame] | 35 | return CommandExecuter(log_level, logger_to_set) |
Ahmad Sharif | 70de27b | 2011-06-15 17:51:24 -0700 | [diff] [blame] | 36 | |
| 37 | |
Luis Lozano | bd44704 | 2015-10-01 14:24:27 -0700 | [diff] [blame] | 38 | class CommandExecuter(object): |
| 39 | """Provides several methods to execute commands on several environments.""" |
| 40 | |
cmtice | 1390924 | 2014-03-11 13:38:07 -0700 | [diff] [blame] | 41 | def __init__(self, log_level, logger_to_set=None): |
| 42 | self.log_level = log_level |
cmtice | 3782857 | 2015-02-04 17:37:43 -0800 | [diff] [blame] | 43 | if log_level == "none": |
| 44 | self.logger = None |
Ahmad Sharif | 70de27b | 2011-06-15 17:51:24 -0700 | [diff] [blame] | 45 | else: |
cmtice | 3782857 | 2015-02-04 17:37:43 -0800 | [diff] [blame] | 46 | if logger_to_set is not None: |
| 47 | self.logger = logger_to_set |
| 48 | else: |
| 49 | self.logger = logger.GetLogger() |
Ahmad Sharif | 70de27b | 2011-06-15 17:51:24 -0700 | [diff] [blame] | 50 | |
cmtice | 1390924 | 2014-03-11 13:38:07 -0700 | [diff] [blame] | 51 | def GetLogLevel(self): |
| 52 | return self.log_level |
| 53 | |
| 54 | def SetLogLevel(self, log_level): |
| 55 | self.log_level = log_level |
| 56 | |
Ahmad Sharif | 70de27b | 2011-06-15 17:51:24 -0700 | [diff] [blame] | 57 | def RunCommand(self, cmd, return_output=False, machine=None, |
| 58 | username=None, command_terminator=None, |
| 59 | command_timeout=None, |
Ahmad Sharif | f395c26 | 2012-10-09 17:48:09 -0700 | [diff] [blame] | 60 | terminated_timeout=10, |
| 61 | print_to_console=True): |
Luis Lozano | 45b53c5 | 2015-09-30 11:36:27 -0700 | [diff] [blame] | 62 | """Run a command.""" |
Ahmad Sharif | 70de27b | 2011-06-15 17:51:24 -0700 | [diff] [blame] | 63 | |
Ahmad Sharif | 0dcbc4b | 2012-02-02 16:37:18 -0800 | [diff] [blame] | 64 | cmd = str(cmd) |
| 65 | |
cmtice | 1390924 | 2014-03-11 13:38:07 -0700 | [diff] [blame] | 66 | if self.log_level == "quiet": |
Luis Lozano | bd44704 | 2015-10-01 14:24:27 -0700 | [diff] [blame] | 67 | print_to_console = False |
cmtice | 1390924 | 2014-03-11 13:38:07 -0700 | [diff] [blame] | 68 | |
| 69 | if self.log_level == "verbose": |
| 70 | self.logger.LogCmd(cmd, machine, username, print_to_console) |
cmtice | 3782857 | 2015-02-04 17:37:43 -0800 | [diff] [blame] | 71 | elif self.logger: |
cmtice | 0537956 | 2014-04-07 13:19:01 -0700 | [diff] [blame] | 72 | self.logger.LogCmdToFileOnly(cmd, machine, username) |
Ahmad Sharif | 70de27b | 2011-06-15 17:51:24 -0700 | [diff] [blame] | 73 | if command_terminator and command_terminator.IsTerminated(): |
cmtice | 3782857 | 2015-02-04 17:37:43 -0800 | [diff] [blame] | 74 | if self.logger: |
| 75 | self.logger.LogError("Command was terminated!", print_to_console) |
Ahmad Sharif | 0dcbc4b | 2012-02-02 16:37:18 -0800 | [diff] [blame] | 76 | if return_output: |
| 77 | return [1, "", ""] |
| 78 | else: |
| 79 | return 1 |
Ahmad Sharif | 70de27b | 2011-06-15 17:51:24 -0700 | [diff] [blame] | 80 | |
| 81 | if machine is not None: |
| 82 | user = "" |
| 83 | if username is not None: |
| 84 | user = username + "@" |
| 85 | cmd = "ssh -t -t %s%s -- '%s'" % (user, machine, cmd) |
| 86 | |
Luis Lozano | 45b53c5 | 2015-09-30 11:36:27 -0700 | [diff] [blame] | 87 | # We use setsid so that the child will have a different session id |
| 88 | # and we can easily kill the process group. This is also important |
| 89 | # because the child will be disassociated from the parent terminal. |
| 90 | # In this way the child cannot mess the parent's terminal. |
| 91 | p = subprocess.Popen(cmd, stdout=subprocess.PIPE, |
| 92 | stderr=subprocess.PIPE, shell=True, |
| 93 | preexec_fn=os.setsid) |
Ahmad Sharif | 70de27b | 2011-06-15 17:51:24 -0700 | [diff] [blame] | 94 | |
| 95 | full_stdout = "" |
| 96 | full_stderr = "" |
| 97 | |
| 98 | # Pull output from pipes, send it to file/stdout/string |
| 99 | out = err = None |
| 100 | pipes = [p.stdout, p.stderr] |
| 101 | |
Luis Lozano | f81680c | 2013-03-15 14:44:13 -0700 | [diff] [blame] | 102 | my_poll = select.poll() |
| 103 | my_poll.register(p.stdout, select.POLLIN) |
| 104 | my_poll.register(p.stderr, select.POLLIN) |
| 105 | |
Ahmad Sharif | 70de27b | 2011-06-15 17:51:24 -0700 | [diff] [blame] | 106 | terminated_time = None |
| 107 | started_time = time.time() |
| 108 | |
| 109 | while len(pipes): |
Ahmad Sharif | 70de27b | 2011-06-15 17:51:24 -0700 | [diff] [blame] | 110 | if command_terminator and command_terminator.IsTerminated(): |
Luis Lozano | 45b53c5 | 2015-09-30 11:36:27 -0700 | [diff] [blame] | 111 | os.killpg(os.getpgid(p.pid), signal.SIGTERM) |
cmtice | 3782857 | 2015-02-04 17:37:43 -0800 | [diff] [blame] | 112 | if self.logger: |
Luis Lozano | 45b53c5 | 2015-09-30 11:36:27 -0700 | [diff] [blame] | 113 | self.logger.LogError("Command received termination request. " |
| 114 | "Killed child process group.", |
| 115 | print_to_console) |
| 116 | break |
Luis Lozano | f81680c | 2013-03-15 14:44:13 -0700 | [diff] [blame] | 117 | |
Luis Lozano | bd44704 | 2015-10-01 14:24:27 -0700 | [diff] [blame] | 118 | l = my_poll.poll(100) |
| 119 | for (fd, _) in l: |
Luis Lozano | f81680c | 2013-03-15 14:44:13 -0700 | [diff] [blame] | 120 | if fd == p.stdout.fileno(): |
Ahmad Sharif | 70de27b | 2011-06-15 17:51:24 -0700 | [diff] [blame] | 121 | out = os.read(p.stdout.fileno(), 16384) |
| 122 | if return_output: |
| 123 | full_stdout += out |
cmtice | 3782857 | 2015-02-04 17:37:43 -0800 | [diff] [blame] | 124 | if self.logger: |
| 125 | self.logger.LogCommandOutput(out, print_to_console) |
Ahmad Sharif | 70de27b | 2011-06-15 17:51:24 -0700 | [diff] [blame] | 126 | if out == "": |
| 127 | pipes.remove(p.stdout) |
Luis Lozano | f81680c | 2013-03-15 14:44:13 -0700 | [diff] [blame] | 128 | my_poll.unregister(p.stdout) |
| 129 | if fd == p.stderr.fileno(): |
Ahmad Sharif | 70de27b | 2011-06-15 17:51:24 -0700 | [diff] [blame] | 130 | err = os.read(p.stderr.fileno(), 16384) |
| 131 | if return_output: |
| 132 | full_stderr += err |
cmtice | 3782857 | 2015-02-04 17:37:43 -0800 | [diff] [blame] | 133 | if self.logger: |
| 134 | self.logger.LogCommandError(err, print_to_console) |
Ahmad Sharif | 70de27b | 2011-06-15 17:51:24 -0700 | [diff] [blame] | 135 | if err == "": |
| 136 | pipes.remove(p.stderr) |
Luis Lozano | f81680c | 2013-03-15 14:44:13 -0700 | [diff] [blame] | 137 | my_poll.unregister(p.stderr) |
Ahmad Sharif | 70de27b | 2011-06-15 17:51:24 -0700 | [diff] [blame] | 138 | |
| 139 | if p.poll() is not None: |
| 140 | if terminated_time is None: |
| 141 | terminated_time = time.time() |
| 142 | elif (terminated_timeout is not None and |
| 143 | time.time() - terminated_time > terminated_timeout): |
cmtice | 3782857 | 2015-02-04 17:37:43 -0800 | [diff] [blame] | 144 | if self.logger: |
Luis Lozano | 45b53c5 | 2015-09-30 11:36:27 -0700 | [diff] [blame] | 145 | self.logger.LogWarning("Timeout of %s seconds reached since " |
| 146 | "process termination." |
| 147 | % terminated_timeout, print_to_console) |
Ahmad Sharif | 70de27b | 2011-06-15 17:51:24 -0700 | [diff] [blame] | 148 | break |
| 149 | |
| 150 | if (command_timeout is not None and |
| 151 | time.time() - started_time > command_timeout): |
Luis Lozano | 45b53c5 | 2015-09-30 11:36:27 -0700 | [diff] [blame] | 152 | os.killpg(os.getpgid(p.pid), signal.SIGTERM) |
cmtice | 3782857 | 2015-02-04 17:37:43 -0800 | [diff] [blame] | 153 | if self.logger: |
Luis Lozano | 45b53c5 | 2015-09-30 11:36:27 -0700 | [diff] [blame] | 154 | self.logger.LogWarning("Timeout of %s seconds reached since process" |
| 155 | "started. Killed child process group." |
| 156 | % command_timeout, print_to_console) |
Ahmad Sharif | 70de27b | 2011-06-15 17:51:24 -0700 | [diff] [blame] | 157 | break |
| 158 | |
| 159 | if out == err == "": |
| 160 | break |
| 161 | |
| 162 | p.wait() |
Ahmad Sharif | 70de27b | 2011-06-15 17:51:24 -0700 | [diff] [blame] | 163 | if return_output: |
| 164 | return (p.returncode, full_stdout, full_stderr) |
| 165 | return p.returncode |
| 166 | |
| 167 | def RemoteAccessInitCommand(self, chromeos_root, machine): |
| 168 | command = "" |
| 169 | command += "\nset -- --remote=" + machine |
| 170 | command += "\n. " + chromeos_root + "/src/scripts/common.sh" |
| 171 | command += "\n. " + chromeos_root + "/src/scripts/remote_access.sh" |
| 172 | command += "\nTMP=$(mktemp -d)" |
| 173 | command += "\nFLAGS \"$@\" || exit 1" |
| 174 | command += "\nremote_access_init" |
| 175 | return command |
| 176 | |
Ahmad Sharif | 0dcbc4b | 2012-02-02 16:37:18 -0800 | [diff] [blame] | 177 | def WriteToTempShFile(self, contents): |
| 178 | handle, command_file = tempfile.mkstemp(prefix=os.uname()[1], |
| 179 | suffix=".sh") |
| 180 | os.write(handle, "#!/bin/bash\n") |
| 181 | os.write(handle, contents) |
| 182 | os.close(handle) |
| 183 | return command_file |
| 184 | |
Ahmad Sharif | 70de27b | 2011-06-15 17:51:24 -0700 | [diff] [blame] | 185 | |
| 186 | def CrosLearnBoard(self, chromeos_root, machine): |
| 187 | command = self.RemoteAccessInitCommand(chromeos_root, machine) |
| 188 | command += "\nlearn_board" |
| 189 | command += "\necho ${FLAGS_board}" |
Luis Lozano | bd44704 | 2015-10-01 14:24:27 -0700 | [diff] [blame] | 190 | retval, output, _ = self.RunCommand(command, True) |
cmtice | 3782857 | 2015-02-04 17:37:43 -0800 | [diff] [blame] | 191 | if self.logger: |
| 192 | self.logger.LogFatalIf(retval, "learn_board command failed") |
cmtice | 513d852 | 2015-02-06 09:17:25 -0800 | [diff] [blame] | 193 | elif retval: |
cmtice | 3782857 | 2015-02-04 17:37:43 -0800 | [diff] [blame] | 194 | sys.exit(1) |
Ahmad Sharif | 70de27b | 2011-06-15 17:51:24 -0700 | [diff] [blame] | 195 | return output.split()[-1] |
| 196 | |
| 197 | def CrosRunCommand(self, cmd, return_output=False, machine=None, |
Luis Lozano | bd44704 | 2015-10-01 14:24:27 -0700 | [diff] [blame] | 198 | command_terminator=None, |
| 199 | chromeos_root=None, command_timeout=None, |
| 200 | terminated_timeout=10, print_to_console=True): |
Ahmad Sharif | 70de27b | 2011-06-15 17:51:24 -0700 | [diff] [blame] | 201 | """Run a command on a chromeos box""" |
cmtice | 1390924 | 2014-03-11 13:38:07 -0700 | [diff] [blame] | 202 | |
| 203 | if self.log_level != "verbose": |
Luis Lozano | bd44704 | 2015-10-01 14:24:27 -0700 | [diff] [blame] | 204 | print_to_console = False |
cmtice | 1390924 | 2014-03-11 13:38:07 -0700 | [diff] [blame] | 205 | |
cmtice | 3782857 | 2015-02-04 17:37:43 -0800 | [diff] [blame] | 206 | if self.logger: |
| 207 | self.logger.LogCmd(cmd, print_to_console=print_to_console) |
| 208 | self.logger.LogFatalIf(not machine, "No machine provided!") |
| 209 | self.logger.LogFatalIf(not chromeos_root, "chromeos_root not given!") |
| 210 | else: |
| 211 | if not chromeos_root or not machine: |
| 212 | sys.exit(1) |
Ahmad Sharif | 70de27b | 2011-06-15 17:51:24 -0700 | [diff] [blame] | 213 | chromeos_root = os.path.expanduser(chromeos_root) |
Ahmad Sharif | 0dcbc4b | 2012-02-02 16:37:18 -0800 | [diff] [blame] | 214 | |
| 215 | # Write all commands to a file. |
| 216 | command_file = self.WriteToTempShFile(cmd) |
Ahmad Sharif | 4467f00 | 2012-12-20 12:09:49 -0800 | [diff] [blame] | 217 | retval = self.CopyFiles(command_file, command_file, |
| 218 | dest_machine=machine, |
| 219 | command_terminator=command_terminator, |
| 220 | chromeos_root=chromeos_root, |
| 221 | dest_cros=True, |
| 222 | recursive=False, |
| 223 | print_to_console=print_to_console) |
| 224 | if retval: |
cmtice | 3782857 | 2015-02-04 17:37:43 -0800 | [diff] [blame] | 225 | if self.logger: |
| 226 | self.logger.LogError("Could not run remote command on machine." |
Luis Lozano | bd44704 | 2015-10-01 14:24:27 -0700 | [diff] [blame] | 227 | " Is the machine up?") |
Ahmad Sharif | 4467f00 | 2012-12-20 12:09:49 -0800 | [diff] [blame] | 228 | return retval |
Ahmad Sharif | 0dcbc4b | 2012-02-02 16:37:18 -0800 | [diff] [blame] | 229 | |
Ahmad Sharif | 70de27b | 2011-06-15 17:51:24 -0700 | [diff] [blame] | 230 | command = self.RemoteAccessInitCommand(chromeos_root, machine) |
Ahmad Sharif | 0dcbc4b | 2012-02-02 16:37:18 -0800 | [diff] [blame] | 231 | command += "\nremote_sh bash %s" % command_file |
Luis Lozano | dd75bad | 2014-04-21 13:58:16 -0700 | [diff] [blame] | 232 | command += "\nl_retval=$?; echo \"$REMOTE_OUT\"; exit $l_retval" |
Ahmad Sharif | 0dcbc4b | 2012-02-02 16:37:18 -0800 | [diff] [blame] | 233 | retval = self.RunCommand(command, return_output, |
| 234 | command_terminator=command_terminator, |
Ahmad Sharif | fd356fb | 2012-05-07 12:02:16 -0700 | [diff] [blame] | 235 | command_timeout=command_timeout, |
Ahmad Sharif | 4467f00 | 2012-12-20 12:09:49 -0800 | [diff] [blame] | 236 | terminated_timeout=terminated_timeout, |
| 237 | print_to_console=print_to_console) |
Ahmad Sharif | 70de27b | 2011-06-15 17:51:24 -0700 | [diff] [blame] | 238 | if return_output: |
| 239 | connect_signature = ("Initiating first contact with remote host\n" + |
| 240 | "Connection OK\n") |
| 241 | connect_signature_re = re.compile(connect_signature) |
Luis Lozano | 54db538 | 2015-05-20 15:57:19 -0700 | [diff] [blame] | 242 | modded_retval = list(retval) |
| 243 | modded_retval[1] = connect_signature_re.sub("", retval[1]) |
| 244 | return modded_retval |
Ahmad Sharif | 70de27b | 2011-06-15 17:51:24 -0700 | [diff] [blame] | 245 | return retval |
| 246 | |
Ahmad Sharif | 0dcbc4b | 2012-02-02 16:37:18 -0800 | [diff] [blame] | 247 | def ChrootRunCommand(self, chromeos_root, command, return_output=False, |
Ahmad Sharif | fd356fb | 2012-05-07 12:02:16 -0700 | [diff] [blame] | 248 | command_terminator=None, command_timeout=None, |
Ahmad Sharif | f395c26 | 2012-10-09 17:48:09 -0700 | [diff] [blame] | 249 | terminated_timeout=10, |
Ahmad Sharif | 4467f00 | 2012-12-20 12:09:49 -0800 | [diff] [blame] | 250 | print_to_console=True, |
| 251 | cros_sdk_options=""): |
cmtice | 1390924 | 2014-03-11 13:38:07 -0700 | [diff] [blame] | 252 | |
| 253 | if self.log_level != "verbose": |
| 254 | print_to_console = False |
| 255 | |
cmtice | 3782857 | 2015-02-04 17:37:43 -0800 | [diff] [blame] | 256 | if self.logger: |
| 257 | self.logger.LogCmd(command, print_to_console=print_to_console) |
Ahmad Sharif | 0dcbc4b | 2012-02-02 16:37:18 -0800 | [diff] [blame] | 258 | |
| 259 | handle, command_file = tempfile.mkstemp(dir=os.path.join(chromeos_root, |
Luis Lozano | bd44704 | 2015-10-01 14:24:27 -0700 | [diff] [blame] | 260 | "src/scripts"), |
| 261 | suffix=".sh", |
| 262 | prefix="in_chroot_cmd") |
Ahmad Sharif | 0dcbc4b | 2012-02-02 16:37:18 -0800 | [diff] [blame] | 263 | os.write(handle, "#!/bin/bash\n") |
| 264 | os.write(handle, command) |
Rahul Chaudhry | 8ac4ec0 | 2014-11-01 09:31:15 -0700 | [diff] [blame] | 265 | os.write(handle, "\n") |
Ahmad Sharif | 0dcbc4b | 2012-02-02 16:37:18 -0800 | [diff] [blame] | 266 | os.close(handle) |
| 267 | |
| 268 | os.chmod(command_file, 0777) |
| 269 | |
Rahul Chaudhry | 8d77427 | 2014-11-05 12:50:45 -0800 | [diff] [blame] | 270 | # if return_output is set, run a dummy command first to make sure that |
| 271 | # the chroot already exists. We want the final returned output to skip |
| 272 | # the output from chroot creation steps. |
| 273 | if return_output: |
| 274 | ret = self.RunCommand("cd %s; cros_sdk %s -- true" % |
| 275 | (chromeos_root, cros_sdk_options)) |
| 276 | if ret: |
| 277 | return (ret, "", "") |
| 278 | |
Rahul Chaudhry | 8ac4ec0 | 2014-11-01 09:31:15 -0700 | [diff] [blame] | 279 | # Run command_file inside the chroot, making sure that any "~" is expanded |
| 280 | # by the shell inside the chroot, not outside. |
| 281 | command = ("cd %s; cros_sdk %s -- bash -c '%s/%s'" % |
| 282 | (chromeos_root, |
| 283 | cros_sdk_options, |
| 284 | misc.CHROMEOS_SCRIPTS_DIR, |
| 285 | os.path.basename(command_file))) |
Ahmad Sharif | 0dcbc4b | 2012-02-02 16:37:18 -0800 | [diff] [blame] | 286 | ret = self.RunCommand(command, return_output, |
Ahmad Sharif | fd356fb | 2012-05-07 12:02:16 -0700 | [diff] [blame] | 287 | command_terminator=command_terminator, |
| 288 | command_timeout=command_timeout, |
Ahmad Sharif | 4467f00 | 2012-12-20 12:09:49 -0800 | [diff] [blame] | 289 | terminated_timeout=terminated_timeout, |
| 290 | print_to_console=print_to_console) |
Ahmad Sharif | 0dcbc4b | 2012-02-02 16:37:18 -0800 | [diff] [blame] | 291 | os.remove(command_file) |
| 292 | return ret |
| 293 | |
| 294 | |
Ahmad Sharif | 70de27b | 2011-06-15 17:51:24 -0700 | [diff] [blame] | 295 | def RunCommands(self, cmdlist, return_output=False, machine=None, |
| 296 | username=None, command_terminator=None): |
| 297 | cmd = " ;\n" .join(cmdlist) |
| 298 | return self.RunCommand(cmd, return_output, machine, username, |
| 299 | command_terminator) |
| 300 | |
| 301 | def CopyFiles(self, src, dest, src_machine=None, dest_machine=None, |
| 302 | src_user=None, dest_user=None, recursive=True, |
| 303 | command_terminator=None, |
Ahmad Sharif | 4467f00 | 2012-12-20 12:09:49 -0800 | [diff] [blame] | 304 | chromeos_root=None, src_cros=False, dest_cros=False, |
| 305 | print_to_console=True): |
Ahmad Sharif | 70de27b | 2011-06-15 17:51:24 -0700 | [diff] [blame] | 306 | src = os.path.expanduser(src) |
| 307 | dest = os.path.expanduser(dest) |
| 308 | |
| 309 | if recursive: |
| 310 | src = src + "/" |
| 311 | dest = dest + "/" |
| 312 | |
| 313 | if src_cros == True or dest_cros == True: |
cmtice | 3782857 | 2015-02-04 17:37:43 -0800 | [diff] [blame] | 314 | if self.logger: |
Luis Lozano | bd44704 | 2015-10-01 14:24:27 -0700 | [diff] [blame] | 315 | self.logger.LogFatalIf(src_cros == dest_cros, |
| 316 | "Only one of src_cros and desc_cros can " |
| 317 | "be True.") |
cmtice | 3782857 | 2015-02-04 17:37:43 -0800 | [diff] [blame] | 318 | self.logger.LogFatalIf(not chromeos_root, "chromeos_root not given!") |
Luis Lozano | bd44704 | 2015-10-01 14:24:27 -0700 | [diff] [blame] | 319 | elif src_cros == dest_cros or not chromeos_root: |
cmtice | 3782857 | 2015-02-04 17:37:43 -0800 | [diff] [blame] | 320 | sys.exit(1) |
Ahmad Sharif | 70de27b | 2011-06-15 17:51:24 -0700 | [diff] [blame] | 321 | if src_cros == True: |
| 322 | cros_machine = src_machine |
| 323 | else: |
| 324 | cros_machine = dest_machine |
| 325 | |
| 326 | command = self.RemoteAccessInitCommand(chromeos_root, cros_machine) |
Ahmad Sharif | 70de27b | 2011-06-15 17:51:24 -0700 | [diff] [blame] | 327 | ssh_command = ("ssh -p ${FLAGS_ssh_port}" + |
| 328 | " -o StrictHostKeyChecking=no" + |
| 329 | " -o UserKnownHostsFile=$(mktemp)" + |
| 330 | " -i $TMP_PRIVATE_KEY") |
| 331 | rsync_prefix = "\nrsync -r -e \"%s\" " % ssh_command |
| 332 | if dest_cros == True: |
| 333 | command += rsync_prefix + "%s root@%s:%s" % (src, dest_machine, dest) |
| 334 | return self.RunCommand(command, |
| 335 | machine=src_machine, |
| 336 | username=src_user, |
Ahmad Sharif | 4467f00 | 2012-12-20 12:09:49 -0800 | [diff] [blame] | 337 | command_terminator=command_terminator, |
| 338 | print_to_console=print_to_console) |
Ahmad Sharif | 70de27b | 2011-06-15 17:51:24 -0700 | [diff] [blame] | 339 | else: |
| 340 | command += rsync_prefix + "root@%s:%s %s" % (src_machine, src, dest) |
| 341 | return self.RunCommand(command, |
| 342 | machine=dest_machine, |
| 343 | username=dest_user, |
Ahmad Sharif | 4467f00 | 2012-12-20 12:09:49 -0800 | [diff] [blame] | 344 | command_terminator=command_terminator, |
| 345 | print_to_console=print_to_console) |
Ahmad Sharif | 70de27b | 2011-06-15 17:51:24 -0700 | [diff] [blame] | 346 | |
| 347 | |
| 348 | if dest_machine == src_machine: |
Luis Lozano | bd44704 | 2015-10-01 14:24:27 -0700 | [diff] [blame] | 349 | command = "rsync -a %s %s" % (src, dest) |
Ahmad Sharif | 70de27b | 2011-06-15 17:51:24 -0700 | [diff] [blame] | 350 | else: |
| 351 | if src_machine is None: |
| 352 | src_machine = os.uname()[1] |
| 353 | src_user = getpass.getuser() |
Luis Lozano | bd44704 | 2015-10-01 14:24:27 -0700 | [diff] [blame] | 354 | command = "rsync -a %s@%s:%s %s" % (src_user, src_machine, src, dest) |
Ahmad Sharif | 70de27b | 2011-06-15 17:51:24 -0700 | [diff] [blame] | 355 | return self.RunCommand(command, |
| 356 | machine=dest_machine, |
| 357 | username=dest_user, |
Ahmad Sharif | 4467f00 | 2012-12-20 12:09:49 -0800 | [diff] [blame] | 358 | command_terminator=command_terminator, |
| 359 | print_to_console=print_to_console) |
Ahmad Sharif | 70de27b | 2011-06-15 17:51:24 -0700 | [diff] [blame] | 360 | |
| 361 | |
Han Shen | 9b55238 | 2015-03-26 11:45:52 -0700 | [diff] [blame] | 362 | def RunCommand2(self, cmd, cwd=None, line_consumer=None, |
| 363 | timeout=None, shell=True, join_stderr=True, env=None): |
| 364 | """Run the command with an extra feature line_consumer. |
| 365 | |
| 366 | This version allow developers to provide a line_consumer which will be |
| 367 | fed execution output lines. |
| 368 | |
| 369 | A line_consumer is a callback, which is given a chance to run for each |
| 370 | line the execution outputs (either to stdout or stderr). The |
| 371 | line_consumer must accept one and exactly one dict argument, the dict |
| 372 | argument has these items - |
| 373 | 'line' - The line output by the binary. Notice, this string includes |
| 374 | the trailing '\n'. |
| 375 | 'output' - Whether this is a stdout or stderr output, values are either |
| 376 | 'stdout' or 'stderr'. When join_stderr is True, this value |
| 377 | will always be 'output'. |
| 378 | 'pobject' - The object used to control execution, for example, call |
| 379 | pobject.kill(). |
| 380 | |
Luis Lozano | 60a68a5 | 2015-09-28 15:13:57 -0700 | [diff] [blame] | 381 | Note: As this is written, the stdin for the process executed is |
| 382 | not associated with the stdin of the caller of this routine. |
| 383 | |
Han Shen | 9b55238 | 2015-03-26 11:45:52 -0700 | [diff] [blame] | 384 | Args: |
Luis Lozano | bd44704 | 2015-10-01 14:24:27 -0700 | [diff] [blame] | 385 | cmd: Command in a single string. |
| 386 | cwd: Working directory for execution. |
Han Shen | 9b55238 | 2015-03-26 11:45:52 -0700 | [diff] [blame] | 387 | line_consumer: A function that will ba called by this function. See above |
Luis Lozano | bd44704 | 2015-10-01 14:24:27 -0700 | [diff] [blame] | 388 | for details. |
| 389 | timeout: terminate command after this timeout. |
| 390 | shell: Whether to use a shell for execution. |
| 391 | join_stderr: Whether join stderr to stdout stream. |
| 392 | env: Execution environment. |
Han Shen | 9b55238 | 2015-03-26 11:45:52 -0700 | [diff] [blame] | 393 | |
| 394 | Returns: |
| 395 | Execution return code. |
| 396 | |
| 397 | Raises: |
| 398 | child_exception: if fails to start the command process (missing |
| 399 | permission, no such file, etc) |
| 400 | """ |
| 401 | |
| 402 | class StreamHandler(object): |
Luis Lozano | bd44704 | 2015-10-01 14:24:27 -0700 | [diff] [blame] | 403 | """Internal utility class.""" |
Han Shen | 9b55238 | 2015-03-26 11:45:52 -0700 | [diff] [blame] | 404 | |
Luis Lozano | bd44704 | 2015-10-01 14:24:27 -0700 | [diff] [blame] | 405 | def __init__(self, pobject, fd, name, line_consumer): |
| 406 | self._pobject = pobject |
| 407 | self._fd = fd |
| 408 | self._name = name |
| 409 | self._buf = '' |
| 410 | self._line_consumer = line_consumer |
Han Shen | 9b55238 | 2015-03-26 11:45:52 -0700 | [diff] [blame] | 411 | |
Luis Lozano | bd44704 | 2015-10-01 14:24:27 -0700 | [diff] [blame] | 412 | def read_and_notify_line(self): |
| 413 | t = os.read(fd, 1024) |
| 414 | self._buf = self._buf + t |
| 415 | self.notify_line() |
Han Shen | 9b55238 | 2015-03-26 11:45:52 -0700 | [diff] [blame] | 416 | |
Luis Lozano | bd44704 | 2015-10-01 14:24:27 -0700 | [diff] [blame] | 417 | def notify_line(self): |
| 418 | p = self._buf.find('\n') |
| 419 | while p >= 0: |
| 420 | self._line_consumer(line=self._buf[:p+1], output=self._name, |
| 421 | pobject=self._pobject) |
| 422 | if p < len(self._buf) - 1: |
| 423 | self._buf = self._buf[p+1:] |
Han Shen | 9b55238 | 2015-03-26 11:45:52 -0700 | [diff] [blame] | 424 | p = self._buf.find('\n') |
Luis Lozano | bd44704 | 2015-10-01 14:24:27 -0700 | [diff] [blame] | 425 | else: |
| 426 | self._buf = '' |
| 427 | p = -1 |
| 428 | break |
Han Shen | 9b55238 | 2015-03-26 11:45:52 -0700 | [diff] [blame] | 429 | |
Luis Lozano | bd44704 | 2015-10-01 14:24:27 -0700 | [diff] [blame] | 430 | def notify_eos(self): |
| 431 | # Notify end of stream. The last line may not end with a '\n'. |
| 432 | if self._buf != '': |
| 433 | self._line_consumer(line=self._buf, output=self._name, |
| 434 | pobject=self._pobject) |
| 435 | self._buf = '' |
Han Shen | 9b55238 | 2015-03-26 11:45:52 -0700 | [diff] [blame] | 436 | |
| 437 | if self.log_level == "verbose": |
| 438 | self.logger.LogCmd(cmd) |
| 439 | elif self.logger: |
| 440 | self.logger.LogCmdToFileOnly(cmd) |
Luis Lozano | 45b53c5 | 2015-09-30 11:36:27 -0700 | [diff] [blame] | 441 | |
| 442 | # We use setsid so that the child will have a different session id |
| 443 | # and we can easily kill the process group. This is also important |
| 444 | # because the child will be disassociated from the parent terminal. |
| 445 | # In this way the child cannot mess the parent's terminal. |
Han Shen | 9b55238 | 2015-03-26 11:45:52 -0700 | [diff] [blame] | 446 | pobject = subprocess.Popen( |
| 447 | cmd, cwd=cwd, bufsize=1024, env=env, shell=shell, |
Luis Lozano | 45b53c5 | 2015-09-30 11:36:27 -0700 | [diff] [blame] | 448 | universal_newlines=True, stdout=subprocess.PIPE, |
| 449 | stderr=subprocess.STDOUT if join_stderr else subprocess.PIPE, |
| 450 | preexec_fn=os.setsid) |
Luis Lozano | 60a68a5 | 2015-09-28 15:13:57 -0700 | [diff] [blame] | 451 | |
Han Shen | 9b55238 | 2015-03-26 11:45:52 -0700 | [diff] [blame] | 452 | # We provide a default line_consumer |
| 453 | if line_consumer is None: |
Luis Lozano | bd44704 | 2015-10-01 14:24:27 -0700 | [diff] [blame] | 454 | line_consumer = lambda **d: None |
Han Shen | 9b55238 | 2015-03-26 11:45:52 -0700 | [diff] [blame] | 455 | start_time = time.time() |
| 456 | poll = select.poll() |
| 457 | outfd = pobject.stdout.fileno() |
| 458 | poll.register(outfd, select.POLLIN | select.POLLPRI) |
| 459 | handlermap = {outfd: StreamHandler(pobject, outfd, 'stdout', line_consumer)} |
| 460 | if not join_stderr: |
Luis Lozano | bd44704 | 2015-10-01 14:24:27 -0700 | [diff] [blame] | 461 | errfd = pobject.stderr.fileno() |
| 462 | poll.register(errfd, select.POLLIN | select.POLLPRI) |
| 463 | handlermap[errfd] = StreamHandler( |
| 464 | pobject, errfd, 'stderr', line_consumer) |
Han Shen | 9b55238 | 2015-03-26 11:45:52 -0700 | [diff] [blame] | 465 | while len(handlermap): |
Luis Lozano | bd44704 | 2015-10-01 14:24:27 -0700 | [diff] [blame] | 466 | readables = poll.poll(300) |
| 467 | for (fd, evt) in readables: |
| 468 | handler = handlermap[fd] |
| 469 | if evt & (select.POLLPRI | select.POLLIN): |
| 470 | handler.read_and_notify_line() |
| 471 | elif evt & (select.POLLHUP | select.POLLERR | select.POLLNVAL): |
| 472 | handler.notify_eos() |
| 473 | poll.unregister(fd) |
| 474 | del handlermap[fd] |
Han Shen | 9b55238 | 2015-03-26 11:45:52 -0700 | [diff] [blame] | 475 | |
Luis Lozano | bd44704 | 2015-10-01 14:24:27 -0700 | [diff] [blame] | 476 | if timeout is not None and (time.time() - start_time > timeout): |
| 477 | os.killpg(os.getpgid(pobject.pid), signal.SIGTERM) |
Han Shen | 9b55238 | 2015-03-26 11:45:52 -0700 | [diff] [blame] | 478 | |
| 479 | return pobject.wait() |
| 480 | |
| 481 | |
Ahmad Sharif | 70de27b | 2011-06-15 17:51:24 -0700 | [diff] [blame] | 482 | class MockCommandExecuter(CommandExecuter): |
Luis Lozano | bd44704 | 2015-10-01 14:24:27 -0700 | [diff] [blame] | 483 | """Mock class for class CommandExecuter.""" |
| 484 | def __init__(self, log_level, logger_to_set=None): |
| 485 | super(MockCommandExecuter, self).__init__(log_level, logger_to_set) |
Ahmad Sharif | 70de27b | 2011-06-15 17:51:24 -0700 | [diff] [blame] | 486 | |
Luis Lozano | bd44704 | 2015-10-01 14:24:27 -0700 | [diff] [blame] | 487 | def RunCommand(self, cmd, return_output=False, machine=None, |
| 488 | username=None, command_terminator=None, |
| 489 | command_timeout=None, terminated_timeout=10, |
| 490 | print_to_console=True): |
| 491 | assert not command_timeout |
Ahmad Sharif | 0dcbc4b | 2012-02-02 16:37:18 -0800 | [diff] [blame] | 492 | cmd = str(cmd) |
Ahmad Sharif | 70de27b | 2011-06-15 17:51:24 -0700 | [diff] [blame] | 493 | if machine is None: |
| 494 | machine = "localhost" |
| 495 | if username is None: |
| 496 | username = "current" |
Luis Lozano | bd44704 | 2015-10-01 14:24:27 -0700 | [diff] [blame] | 497 | logger.GetLogger().LogCmd("(Mock) " + cmd, machine, |
| 498 | username, print_to_console) |
Ahmad Sharif | 70de27b | 2011-06-15 17:51:24 -0700 | [diff] [blame] | 499 | return 0 |
| 500 | |
| 501 | |
Luis Lozano | bd44704 | 2015-10-01 14:24:27 -0700 | [diff] [blame] | 502 | class CommandTerminator(object): |
| 503 | """Object to request termination of a command in execution.""" |
Ahmad Sharif | 70de27b | 2011-06-15 17:51:24 -0700 | [diff] [blame] | 504 | def __init__(self): |
| 505 | self.terminated = False |
| 506 | |
| 507 | def Terminate(self): |
| 508 | self.terminated = True |
| 509 | |
| 510 | def IsTerminated(self): |
| 511 | return self.terminated |