Ahmad Sharif | 4467f00 | 2012-12-20 12:09:49 -0800 | [diff] [blame] | 1 | #!/usr/bin/python |
Yunlian Jiang | 36f91ad | 2013-03-28 17:13:29 -0700 | [diff] [blame] | 2 | |
| 3 | # Copyright (c) 2013 The Chromium OS Authors. All rights reserved. |
| 4 | # Use of this source code is governed by a BSD-style license that can be |
| 5 | # found in the LICENSE file. |
Ahmad Sharif | 4467f00 | 2012-12-20 12:09:49 -0800 | [diff] [blame] | 6 | |
Ahmad Sharif | f395c26 | 2012-10-09 17:48:09 -0700 | [diff] [blame] | 7 | import hashlib |
| 8 | import image_chromeos |
| 9 | import lock_machine |
| 10 | import math |
| 11 | import os.path |
Ahmad Sharif | 4467f00 | 2012-12-20 12:09:49 -0800 | [diff] [blame] | 12 | import re |
Ahmad Sharif | 0dcbc4b | 2012-02-02 16:37:18 -0800 | [diff] [blame] | 13 | import sys |
| 14 | import threading |
| 15 | import time |
Ahmad Sharif | 4467f00 | 2012-12-20 12:09:49 -0800 | [diff] [blame] | 16 | |
Ahmad Sharif | 0dcbc4b | 2012-02-02 16:37:18 -0800 | [diff] [blame] | 17 | from utils import command_executer |
| 18 | from utils import logger |
Ahmad Sharif | f1d70cb | 2012-02-06 21:51:59 -0800 | [diff] [blame] | 19 | from utils.file_utils import FileUtils |
Ahmad Sharif | 0dcbc4b | 2012-02-02 16:37:18 -0800 | [diff] [blame] | 20 | |
Ahmad Sharif | 4467f00 | 2012-12-20 12:09:49 -0800 | [diff] [blame] | 21 | from image_checksummer import ImageChecksummer |
| 22 | |
Ahmad Sharif | 0dcbc4b | 2012-02-02 16:37:18 -0800 | [diff] [blame] | 23 | CHECKSUM_FILE = "/usr/local/osimage_checksum_file" |
| 24 | |
| 25 | |
| 26 | class CrosMachine(object): |
Ahmad Sharif | f395c26 | 2012-10-09 17:48:09 -0700 | [diff] [blame] | 27 | def __init__(self, name, chromeos_root): |
Ahmad Sharif | 0dcbc4b | 2012-02-02 16:37:18 -0800 | [diff] [blame] | 28 | self.name = name |
| 29 | self.image = None |
| 30 | self.checksum = None |
| 31 | self.locked = False |
| 32 | self.released_time = time.time() |
Yunlian Jiang | 04dc5dc | 2013-04-23 15:05:05 -0700 | [diff] [blame] | 33 | self.test_run = None |
Ahmad Sharif | f395c26 | 2012-10-09 17:48:09 -0700 | [diff] [blame] | 34 | self.chromeos_root = chromeos_root |
Yunlian Jiang | e5b673f | 2013-05-23 11:42:53 -0700 | [diff] [blame] | 35 | if not self.IsReachable(): |
Yunlian Jiang | 837e07a | 2013-05-22 16:23:28 -0700 | [diff] [blame] | 36 | self.machine_checksum = None |
| 37 | return |
Ahmad Sharif | f395c26 | 2012-10-09 17:48:09 -0700 | [diff] [blame] | 38 | self._GetMemoryInfo() |
| 39 | self._GetCPUInfo() |
| 40 | self._ComputeMachineChecksumString() |
Ahmad Sharif | 4467f00 | 2012-12-20 12:09:49 -0800 | [diff] [blame] | 41 | self._GetMachineID() |
| 42 | self.machine_checksum = self._GetMD5Checksum(self.checksum_string) |
| 43 | self.machine_id_checksum = self._GetMD5Checksum(self.machine_id) |
Ahmad Sharif | f395c26 | 2012-10-09 17:48:09 -0700 | [diff] [blame] | 44 | |
Yunlian Jiang | e5b673f | 2013-05-23 11:42:53 -0700 | [diff] [blame] | 45 | def IsReachable(self): |
Yunlian Jiang | 837e07a | 2013-05-22 16:23:28 -0700 | [diff] [blame] | 46 | ce = command_executer.GetCommandExecuter() |
| 47 | command = "ls" |
| 48 | ret = ce.CrosRunCommand(command, |
| 49 | machine=self.name, |
| 50 | chromeos_root=self.chromeos_root) |
| 51 | if ret: |
| 52 | return False |
| 53 | return True |
| 54 | |
Ahmad Sharif | f395c26 | 2012-10-09 17:48:09 -0700 | [diff] [blame] | 55 | def _ParseMemoryInfo(self): |
| 56 | line = self.meminfo.splitlines()[0] |
| 57 | usable_kbytes = int(line.split()[1]) |
Yunlian Jiang | 04dc5dc | 2013-04-23 15:05:05 -0700 | [diff] [blame] | 58 | # This code is from src/third_party/test/files/client/bin/base_utils.py |
Ahmad Sharif | f395c26 | 2012-10-09 17:48:09 -0700 | [diff] [blame] | 59 | # usable_kbytes is system's usable DRAM in kbytes, |
| 60 | # as reported by memtotal() from device /proc/meminfo memtotal |
| 61 | # after Linux deducts 1.5% to 9.5% for system table overhead |
| 62 | # Undo the unknown actual deduction by rounding up |
| 63 | # to next small multiple of a big power-of-two |
| 64 | # eg 12GB - 5.1% gets rounded back up to 12GB |
| 65 | mindeduct = 0.005 # 0.5 percent |
| 66 | maxdeduct = 0.095 # 9.5 percent |
| 67 | # deduction range 1.5% .. 9.5% supports physical mem sizes |
| 68 | # 6GB .. 12GB in steps of .5GB |
| 69 | # 12GB .. 24GB in steps of 1 GB |
| 70 | # 24GB .. 48GB in steps of 2 GB ... |
| 71 | # Finer granularity in physical mem sizes would require |
| 72 | # tighter spread between min and max possible deductions |
| 73 | |
| 74 | # increase mem size by at least min deduction, without rounding |
| 75 | min_kbytes = int(usable_kbytes / (1.0 - mindeduct)) |
| 76 | # increase mem size further by 2**n rounding, by 0..roundKb or more |
| 77 | round_kbytes = int(usable_kbytes / (1.0 - maxdeduct)) - min_kbytes |
| 78 | # find least binary roundup 2**n that covers worst-cast roundKb |
| 79 | mod2n = 1 << int(math.ceil(math.log(round_kbytes, 2))) |
| 80 | # have round_kbytes <= mod2n < round_kbytes*2 |
| 81 | # round min_kbytes up to next multiple of mod2n |
| 82 | phys_kbytes = min_kbytes + mod2n - 1 |
| 83 | phys_kbytes -= phys_kbytes % mod2n # clear low bits |
| 84 | self.phys_kbytes = phys_kbytes |
| 85 | |
| 86 | def _GetMemoryInfo(self): |
Ahmad Sharif | 4467f00 | 2012-12-20 12:09:49 -0800 | [diff] [blame] | 87 | #TODO yunlian: when the machine in rebooting, it will not return |
Ahmad Sharif | f395c26 | 2012-10-09 17:48:09 -0700 | [diff] [blame] | 88 | #meminfo, the assert does not catch it either |
| 89 | ce = command_executer.GetCommandExecuter() |
| 90 | command = "cat /proc/meminfo" |
| 91 | ret, self.meminfo, _ = ce.CrosRunCommand( |
| 92 | command, return_output=True, |
| 93 | machine=self.name, username="root", chromeos_root=self.chromeos_root) |
| 94 | assert ret == 0, "Could not get meminfo from machine: %s" % self.name |
| 95 | if ret == 0: |
| 96 | self._ParseMemoryInfo() |
| 97 | |
| 98 | #cpuinfo format is different across architecture |
| 99 | #need to find a better way to parse it. |
| 100 | def _ParseCPUInfo(self,cpuinfo): |
| 101 | return 0 |
| 102 | |
| 103 | def _GetCPUInfo(self): |
| 104 | ce = command_executer.GetCommandExecuter() |
| 105 | command = "cat /proc/cpuinfo" |
| 106 | ret, self.cpuinfo, _ = ce.CrosRunCommand( |
| 107 | command, return_output=True, |
| 108 | machine=self.name, username="root", chromeos_root=self.chromeos_root) |
| 109 | assert ret == 0, "Could not get cpuinfo from machine: %s" % self.name |
| 110 | if ret == 0: |
| 111 | self._ParseCPUInfo(self.cpuinfo) |
| 112 | |
| 113 | def _ComputeMachineChecksumString(self): |
| 114 | self.checksum_string = "" |
| 115 | exclude_lines_list = ["MHz", "BogoMIPS", "bogomips"] |
| 116 | for line in self.cpuinfo.splitlines(): |
| 117 | if not any([e in line for e in exclude_lines_list]): |
| 118 | self.checksum_string += line |
| 119 | self.checksum_string += " " + str(self.phys_kbytes) |
| 120 | |
Ahmad Sharif | 4467f00 | 2012-12-20 12:09:49 -0800 | [diff] [blame] | 121 | def _GetMD5Checksum(self, ss): |
| 122 | if ss: |
| 123 | return hashlib.md5(ss).hexdigest() |
Ahmad Sharif | f395c26 | 2012-10-09 17:48:09 -0700 | [diff] [blame] | 124 | else: |
Ahmad Sharif | 4467f00 | 2012-12-20 12:09:49 -0800 | [diff] [blame] | 125 | return "" |
| 126 | |
| 127 | def _GetMachineID(self): |
| 128 | ce = command_executer.GetCommandExecuter() |
Luis Lozano | f81680c | 2013-03-15 14:44:13 -0700 | [diff] [blame] | 129 | command = "dump_vpd_log --full --stdout" |
Ahmad Sharif | 4467f00 | 2012-12-20 12:09:49 -0800 | [diff] [blame] | 130 | ret, if_out, _ = ce.CrosRunCommand( |
| 131 | command, return_output=True, |
| 132 | machine=self.name, chromeos_root=self.chromeos_root) |
| 133 | b = if_out.splitlines() |
Luis Lozano | f81680c | 2013-03-15 14:44:13 -0700 | [diff] [blame] | 134 | a = [l for l in b if "Product" in l] |
Yunlian Jiang | 9fc9919 | 2013-05-29 16:29:51 -0700 | [diff] [blame] | 135 | if len(a): |
| 136 | self.machine_id = a[0] |
| 137 | return |
| 138 | command = "ifconfig" |
| 139 | ret, if_out, _ = ce.CrosRunCommand( |
| 140 | command, return_output=True, |
| 141 | machine=self.name, chromeos_root=self.chromeos_root) |
| 142 | b = if_out.splitlines() |
| 143 | a = [l for l in b if "HWaddr" in l] |
| 144 | if len(a): |
| 145 | self.machine_id = "_".join(a) |
| 146 | return |
| 147 | assert 0, "Could not get machine_id from machine: %s" % self.name |
Ahmad Sharif | 0dcbc4b | 2012-02-02 16:37:18 -0800 | [diff] [blame] | 148 | |
| 149 | def __str__(self): |
| 150 | l = [] |
| 151 | l.append(self.name) |
| 152 | l.append(str(self.image)) |
| 153 | l.append(str(self.checksum)) |
| 154 | l.append(str(self.locked)) |
| 155 | l.append(str(self.released_time)) |
| 156 | return ", ".join(l) |
| 157 | |
| 158 | |
| 159 | class MachineManager(object): |
Luis Lozano | f81680c | 2013-03-15 14:44:13 -0700 | [diff] [blame] | 160 | def __init__(self, chromeos_root, acquire_timeout): |
Ahmad Sharif | 0dcbc4b | 2012-02-02 16:37:18 -0800 | [diff] [blame] | 161 | self._lock = threading.RLock() |
| 162 | self._all_machines = [] |
| 163 | self._machines = [] |
| 164 | self.image_lock = threading.Lock() |
| 165 | self.num_reimages = 0 |
| 166 | self.chromeos_root = None |
Ahmad Sharif | 4467f00 | 2012-12-20 12:09:49 -0800 | [diff] [blame] | 167 | self.machine_checksum = {} |
| 168 | self.machine_checksum_string = {} |
Luis Lozano | f81680c | 2013-03-15 14:44:13 -0700 | [diff] [blame] | 169 | self.acquire_timeout = acquire_timeout |
Ahmad Sharif | 4467f00 | 2012-12-20 12:09:49 -0800 | [diff] [blame] | 170 | |
| 171 | if os.path.isdir(lock_machine.Machine.LOCKS_DIR): |
Ahmad Sharif | f395c26 | 2012-10-09 17:48:09 -0700 | [diff] [blame] | 172 | self.no_lock = False |
| 173 | else: |
| 174 | self.no_lock = True |
Ahmad Sharif | 4467f00 | 2012-12-20 12:09:49 -0800 | [diff] [blame] | 175 | self._initialized_machines = [] |
Ahmad Sharif | 0dcbc4b | 2012-02-02 16:37:18 -0800 | [diff] [blame] | 176 | self.chromeos_root = chromeos_root |
| 177 | |
Ahmad Sharif | 4467f00 | 2012-12-20 12:09:49 -0800 | [diff] [blame] | 178 | def ImageMachine(self, machine, label): |
Luis Lozano | f81680c | 2013-03-15 14:44:13 -0700 | [diff] [blame] | 179 | checksum = ImageChecksummer().Checksum(label) |
Ahmad Sharif | 0dcbc4b | 2012-02-02 16:37:18 -0800 | [diff] [blame] | 180 | if machine.checksum == checksum: |
| 181 | return |
Ahmad Sharif | 4467f00 | 2012-12-20 12:09:49 -0800 | [diff] [blame] | 182 | chromeos_root = label.chromeos_root |
Ahmad Sharif | f1d70cb | 2012-02-06 21:51:59 -0800 | [diff] [blame] | 183 | if not chromeos_root: |
| 184 | chromeos_root = self.chromeos_root |
Ahmad Sharif | 4467f00 | 2012-12-20 12:09:49 -0800 | [diff] [blame] | 185 | image_chromeos_args = [image_chromeos.__file__, |
| 186 | "--chromeos_root=%s" % chromeos_root, |
| 187 | "--image=%s" % label.chromeos_image, |
| 188 | "--image_args=%s" % label.image_args, |
| 189 | "--remote=%s" % machine.name] |
| 190 | if label.board: |
| 191 | image_chromeos_args.append("--board=%s" % label.board) |
Ahmad Sharif | 0dcbc4b | 2012-02-02 16:37:18 -0800 | [diff] [blame] | 192 | |
| 193 | # Currently can't image two machines at once. |
| 194 | # So have to serialized on this lock. |
| 195 | ce = command_executer.GetCommandExecuter() |
| 196 | with self.image_lock: |
Ahmad Sharif | 4467f00 | 2012-12-20 12:09:49 -0800 | [diff] [blame] | 197 | retval = ce.RunCommand(" ".join(["python"] + image_chromeos_args)) |
Ahmad Sharif | 0dcbc4b | 2012-02-02 16:37:18 -0800 | [diff] [blame] | 198 | if retval: |
Yunlian Jiang | 36f91ad | 2013-03-28 17:13:29 -0700 | [diff] [blame] | 199 | cmd ="reboot && exit" |
| 200 | ce.CrosRunCommand(cmd, machine=machine.name, |
| 201 | chromeos_root=self.chromeos_root) |
| 202 | time.sleep(60) |
| 203 | retval = ce.RunCommand(" ".join(["python"] + image_chromeos_args)) |
| 204 | if retval: |
Ahmad Sharif | 0dcbc4b | 2012-02-02 16:37:18 -0800 | [diff] [blame] | 205 | raise Exception("Could not image machine: '%s'." % machine.name) |
Ahmad Sharif | 4467f00 | 2012-12-20 12:09:49 -0800 | [diff] [blame] | 206 | else: |
| 207 | self.num_reimages += 1 |
Ahmad Sharif | 0dcbc4b | 2012-02-02 16:37:18 -0800 | [diff] [blame] | 208 | machine.checksum = checksum |
Ahmad Sharif | 4467f00 | 2012-12-20 12:09:49 -0800 | [diff] [blame] | 209 | machine.image = label.chromeos_image |
Ahmad Sharif | 0dcbc4b | 2012-02-02 16:37:18 -0800 | [diff] [blame] | 210 | |
| 211 | return retval |
| 212 | |
Ahmad Sharif | 4467f00 | 2012-12-20 12:09:49 -0800 | [diff] [blame] | 213 | def ComputeCommonCheckSum(self, label): |
| 214 | for machine in self.GetMachines(label): |
Ahmad Sharif | f395c26 | 2012-10-09 17:48:09 -0700 | [diff] [blame] | 215 | if machine.machine_checksum: |
Ahmad Sharif | 4467f00 | 2012-12-20 12:09:49 -0800 | [diff] [blame] | 216 | self.machine_checksum[label.name] = machine.machine_checksum |
Ahmad Sharif | f395c26 | 2012-10-09 17:48:09 -0700 | [diff] [blame] | 217 | break |
| 218 | |
Ahmad Sharif | 4467f00 | 2012-12-20 12:09:49 -0800 | [diff] [blame] | 219 | def ComputeCommonCheckSumString(self, label): |
| 220 | for machine in self.GetMachines(label): |
Ahmad Sharif | f395c26 | 2012-10-09 17:48:09 -0700 | [diff] [blame] | 221 | if machine.checksum_string: |
Ahmad Sharif | 4467f00 | 2012-12-20 12:09:49 -0800 | [diff] [blame] | 222 | self.machine_checksum_string[label.name] = machine.checksum_string |
Ahmad Sharif | f395c26 | 2012-10-09 17:48:09 -0700 | [diff] [blame] | 223 | break |
| 224 | |
Ahmad Sharif | 0dcbc4b | 2012-02-02 16:37:18 -0800 | [diff] [blame] | 225 | def _TryToLockMachine(self, cros_machine): |
| 226 | with self._lock: |
| 227 | assert cros_machine, "Machine can't be None" |
| 228 | for m in self._machines: |
Luis Lozano | f81680c | 2013-03-15 14:44:13 -0700 | [diff] [blame] | 229 | if m.name == cros_machine.name: |
| 230 | return |
Ahmad Sharif | 0dcbc4b | 2012-02-02 16:37:18 -0800 | [diff] [blame] | 231 | if self.no_lock: |
| 232 | locked = True |
| 233 | else: |
| 234 | locked = lock_machine.Machine(cros_machine.name).Lock(True, sys.argv[0]) |
| 235 | if locked: |
| 236 | self._machines.append(cros_machine) |
| 237 | ce = command_executer.GetCommandExecuter() |
| 238 | command = "cat %s" % CHECKSUM_FILE |
| 239 | ret, out, _ = ce.CrosRunCommand( |
| 240 | command, return_output=True, chromeos_root=self.chromeos_root, |
| 241 | machine=cros_machine.name) |
| 242 | if ret == 0: |
| 243 | cros_machine.checksum = out.strip() |
| 244 | else: |
| 245 | logger.GetLogger().LogOutput("Couldn't lock: %s" % cros_machine.name) |
| 246 | |
| 247 | # This is called from single threaded mode. |
| 248 | def AddMachine(self, machine_name): |
| 249 | with self._lock: |
| 250 | for m in self._all_machines: |
| 251 | assert m.name != machine_name, "Tried to double-add %s" % machine_name |
Ahmad Sharif | f395c26 | 2012-10-09 17:48:09 -0700 | [diff] [blame] | 252 | cm = CrosMachine(machine_name, self.chromeos_root) |
Yunlian Jiang | 837e07a | 2013-05-22 16:23:28 -0700 | [diff] [blame] | 253 | if cm.machine_checksum: |
| 254 | self._all_machines.append(cm) |
Ahmad Sharif | f395c26 | 2012-10-09 17:48:09 -0700 | [diff] [blame] | 255 | |
Ahmad Sharif | 4467f00 | 2012-12-20 12:09:49 -0800 | [diff] [blame] | 256 | def AreAllMachineSame(self, label): |
| 257 | checksums = [m.machine_checksum for m in self.GetMachines(label)] |
Ahmad Sharif | f395c26 | 2012-10-09 17:48:09 -0700 | [diff] [blame] | 258 | return len(set(checksums)) == 1 |
Ahmad Sharif | 0dcbc4b | 2012-02-02 16:37:18 -0800 | [diff] [blame] | 259 | |
Yunlian Jiang | e5b673f | 2013-05-23 11:42:53 -0700 | [diff] [blame] | 260 | def RemoveMachine(self, machine_name): |
| 261 | with self._lock: |
| 262 | self._machines = [m for m in self._machines |
| 263 | if m.name != machine_name] |
| 264 | res = lock_machine.Machine(machine_name).Unlock(True) |
| 265 | if not res: |
| 266 | logger.GetLogger().LogError("Could not unlock machine: '%s'." |
| 267 | % m.name) |
| 268 | |
Ahmad Sharif | 4467f00 | 2012-12-20 12:09:49 -0800 | [diff] [blame] | 269 | def AcquireMachine(self, chromeos_image, label): |
Luis Lozano | f81680c | 2013-03-15 14:44:13 -0700 | [diff] [blame] | 270 | image_checksum = ImageChecksummer().Checksum(label) |
Ahmad Sharif | 4467f00 | 2012-12-20 12:09:49 -0800 | [diff] [blame] | 271 | machines = self.GetMachines(label) |
Luis Lozano | f81680c | 2013-03-15 14:44:13 -0700 | [diff] [blame] | 272 | check_interval_time = 120 |
Ahmad Sharif | 0dcbc4b | 2012-02-02 16:37:18 -0800 | [diff] [blame] | 273 | with self._lock: |
| 274 | # Lazily external lock machines |
Luis Lozano | f81680c | 2013-03-15 14:44:13 -0700 | [diff] [blame] | 275 | while self.acquire_timeout >= 0: |
| 276 | for m in machines: |
| 277 | new_machine = m not in self._all_machines |
Ahmad Sharif | 4467f00 | 2012-12-20 12:09:49 -0800 | [diff] [blame] | 278 | self._TryToLockMachine(m) |
Luis Lozano | f81680c | 2013-03-15 14:44:13 -0700 | [diff] [blame] | 279 | if new_machine: |
| 280 | m.released_time = time.time() |
| 281 | if not self.AreAllMachineSame(label): |
| 282 | logger.GetLogger().LogFatal("-- not all the machine are identical") |
| 283 | if self.GetAvailableMachines(label): |
| 284 | break |
| 285 | else: |
| 286 | sleep_time = max(1, min(self.acquire_timeout, check_interval_time)) |
| 287 | time.sleep(sleep_time) |
| 288 | self.acquire_timeout -= sleep_time |
| 289 | |
| 290 | if self.acquire_timeout < 0: |
Ahmad Sharif | 0dcbc4b | 2012-02-02 16:37:18 -0800 | [diff] [blame] | 291 | machine_names = [] |
Ahmad Sharif | 4467f00 | 2012-12-20 12:09:49 -0800 | [diff] [blame] | 292 | for machine in machines: |
Ahmad Sharif | 0dcbc4b | 2012-02-02 16:37:18 -0800 | [diff] [blame] | 293 | machine_names.append(machine.name) |
Ahmad Sharif | 4467f00 | 2012-12-20 12:09:49 -0800 | [diff] [blame] | 294 | logger.GetLogger().LogFatal("Could not acquire any of the " |
Luis Lozano | f81680c | 2013-03-15 14:44:13 -0700 | [diff] [blame] | 295 | "following machines: '%s'" |
| 296 | % ", ".join(machine_names)) |
Ahmad Sharif | 0dcbc4b | 2012-02-02 16:37:18 -0800 | [diff] [blame] | 297 | |
| 298 | ### for m in self._machines: |
| 299 | ### if (m.locked and time.time() - m.released_time < 10 and |
| 300 | ### m.checksum == image_checksum): |
| 301 | ### return None |
Ahmad Sharif | 4467f00 | 2012-12-20 12:09:49 -0800 | [diff] [blame] | 302 | for m in [machine for machine in self.GetAvailableMachines(label) |
| 303 | if not machine.locked]: |
Ahmad Sharif | 0dcbc4b | 2012-02-02 16:37:18 -0800 | [diff] [blame] | 304 | if m.checksum == image_checksum: |
| 305 | m.locked = True |
Yunlian Jiang | 04dc5dc | 2013-04-23 15:05:05 -0700 | [diff] [blame] | 306 | m.test_run = threading.current_thread() |
Ahmad Sharif | 0dcbc4b | 2012-02-02 16:37:18 -0800 | [diff] [blame] | 307 | return m |
Ahmad Sharif | 4467f00 | 2012-12-20 12:09:49 -0800 | [diff] [blame] | 308 | for m in [machine for machine in self.GetAvailableMachines(label) |
| 309 | if not machine.locked]: |
Ahmad Sharif | 0dcbc4b | 2012-02-02 16:37:18 -0800 | [diff] [blame] | 310 | if not m.checksum: |
| 311 | m.locked = True |
Yunlian Jiang | 04dc5dc | 2013-04-23 15:05:05 -0700 | [diff] [blame] | 312 | m.test_run = threading.current_thread() |
Ahmad Sharif | 0dcbc4b | 2012-02-02 16:37:18 -0800 | [diff] [blame] | 313 | return m |
| 314 | # This logic ensures that threads waiting on a machine will get a machine |
| 315 | # with a checksum equal to their image over other threads. This saves time |
| 316 | # when crosperf initially assigns the machines to threads by minimizing |
| 317 | # the number of re-images. |
| 318 | # TODO(asharif): If we centralize the thread-scheduler, we wont need this |
| 319 | # code and can implement minimal reimaging code more cleanly. |
Ahmad Sharif | 4467f00 | 2012-12-20 12:09:49 -0800 | [diff] [blame] | 320 | for m in [machine for machine in self.GetAvailableMachines(label) |
| 321 | if not machine.locked]: |
Ahmad Sharif | 0dcbc4b | 2012-02-02 16:37:18 -0800 | [diff] [blame] | 322 | if time.time() - m.released_time > 20: |
| 323 | m.locked = True |
Yunlian Jiang | 04dc5dc | 2013-04-23 15:05:05 -0700 | [diff] [blame] | 324 | m.test_run = threading.current_thread() |
Ahmad Sharif | 0dcbc4b | 2012-02-02 16:37:18 -0800 | [diff] [blame] | 325 | return m |
| 326 | return None |
| 327 | |
Ahmad Sharif | 4467f00 | 2012-12-20 12:09:49 -0800 | [diff] [blame] | 328 | def GetAvailableMachines(self, label=None): |
| 329 | if not label: |
| 330 | return self._machines |
| 331 | return [m for m in self._machines if m.name in label.remote] |
| 332 | |
| 333 | def GetMachines(self, label=None): |
| 334 | if not label: |
| 335 | return self._all_machines |
| 336 | return [m for m in self._all_machines if m.name in label.remote] |
Ahmad Sharif | 0dcbc4b | 2012-02-02 16:37:18 -0800 | [diff] [blame] | 337 | |
| 338 | def ReleaseMachine(self, machine): |
| 339 | with self._lock: |
| 340 | for m in self._machines: |
| 341 | if machine.name == m.name: |
| 342 | assert m.locked == True, "Tried to double-release %s" % m.name |
| 343 | m.released_time = time.time() |
| 344 | m.locked = False |
| 345 | m.status = "Available" |
| 346 | break |
| 347 | |
| 348 | def Cleanup(self): |
| 349 | with self._lock: |
| 350 | # Unlock all machines. |
| 351 | for m in self._machines: |
| 352 | if not self.no_lock: |
| 353 | res = lock_machine.Machine(m.name).Unlock(True) |
| 354 | if not res: |
| 355 | logger.GetLogger().LogError("Could not unlock machine: '%s'." |
| 356 | % m.name) |
| 357 | |
| 358 | def __str__(self): |
| 359 | with self._lock: |
| 360 | l = ["MachineManager Status:"] |
| 361 | for m in self._machines: |
| 362 | l.append(str(m)) |
| 363 | return "\n".join(l) |
| 364 | |
| 365 | def AsString(self): |
| 366 | with self._lock: |
| 367 | stringify_fmt = "%-30s %-10s %-4s %-25s %-32s" |
| 368 | header = stringify_fmt % ("Machine", "Thread", "Lock", "Status", |
| 369 | "Checksum") |
| 370 | table = [header] |
| 371 | for m in self._machines: |
Yunlian Jiang | 04dc5dc | 2013-04-23 15:05:05 -0700 | [diff] [blame] | 372 | if m.test_run: |
| 373 | test_name = m.test_run.name |
| 374 | test_status = m.test_run.timeline.GetLastEvent() |
Ahmad Sharif | 0dcbc4b | 2012-02-02 16:37:18 -0800 | [diff] [blame] | 375 | else: |
Yunlian Jiang | 04dc5dc | 2013-04-23 15:05:05 -0700 | [diff] [blame] | 376 | test_name = "" |
| 377 | test_status = "" |
Ahmad Sharif | 0dcbc4b | 2012-02-02 16:37:18 -0800 | [diff] [blame] | 378 | |
| 379 | try: |
| 380 | machine_string = stringify_fmt % (m.name, |
Yunlian Jiang | 04dc5dc | 2013-04-23 15:05:05 -0700 | [diff] [blame] | 381 | test_name, |
Ahmad Sharif | 0dcbc4b | 2012-02-02 16:37:18 -0800 | [diff] [blame] | 382 | m.locked, |
Yunlian Jiang | 04dc5dc | 2013-04-23 15:05:05 -0700 | [diff] [blame] | 383 | test_status, |
Ahmad Sharif | 0dcbc4b | 2012-02-02 16:37:18 -0800 | [diff] [blame] | 384 | m.checksum) |
| 385 | except Exception: |
| 386 | machine_string = "" |
| 387 | table.append(machine_string) |
| 388 | return "Machine Status:\n%s" % "\n".join(table) |
| 389 | |
Ahmad Sharif | 4467f00 | 2012-12-20 12:09:49 -0800 | [diff] [blame] | 390 | def GetAllCPUInfo(self, labels): |
| 391 | """Get cpuinfo for labels, merge them if their cpuinfo are the same.""" |
| 392 | dic = {} |
| 393 | for label in labels: |
| 394 | for machine in self._all_machines: |
| 395 | if machine.name in label.remote: |
| 396 | if machine.cpuinfo not in dic: |
| 397 | dic[machine.cpuinfo] = [label.name] |
| 398 | else: |
| 399 | dic[machine.cpuinfo].append(label.name) |
| 400 | break |
| 401 | output = "" |
| 402 | for key, v in dic.items(): |
| 403 | output += " ".join(v) |
| 404 | output += "\n-------------------\n" |
| 405 | output += key |
| 406 | output += "\n\n\n" |
| 407 | return output |
Ahmad Sharif | 0dcbc4b | 2012-02-02 16:37:18 -0800 | [diff] [blame] | 408 | |
Ahmad Sharif | 0dcbc4b | 2012-02-02 16:37:18 -0800 | [diff] [blame] | 409 | |
Ahmad Sharif | 4467f00 | 2012-12-20 12:09:49 -0800 | [diff] [blame] | 410 | class MockCrosMachine(CrosMachine): |
| 411 | def __init__(self, name, chromeos_root): |
| 412 | self.name = name |
| 413 | self.image = None |
| 414 | self.checksum = None |
| 415 | self.locked = False |
| 416 | self.released_time = time.time() |
Yunlian Jiang | 04dc5dc | 2013-04-23 15:05:05 -0700 | [diff] [blame] | 417 | self.test_run = None |
Ahmad Sharif | 4467f00 | 2012-12-20 12:09:49 -0800 | [diff] [blame] | 418 | self.chromeos_root = chromeos_root |
| 419 | self.checksum_string = re.sub("\d", "", name) |
| 420 | #In test, we assume "lumpy1", "lumpy2" are the same machine. |
| 421 | self.machine_checksum = self._GetMD5Checksum(self.checksum_string) |
| 422 | |
| 423 | |
| 424 | class MockMachineManager(MachineManager): |
| 425 | |
Luis Lozano | f81680c | 2013-03-15 14:44:13 -0700 | [diff] [blame] | 426 | def __init__(self, chromeos_root, acquire_timeout): |
| 427 | super(MockMachineManager, self).__init__(chromeos_root, acquire_timeout) |
Ahmad Sharif | 4467f00 | 2012-12-20 12:09:49 -0800 | [diff] [blame] | 428 | |
| 429 | def _TryToLockMachine(self, cros_machine): |
| 430 | self._machines.append(cros_machine) |
| 431 | cros_machine.checksum = "" |
Ahmad Sharif | 0dcbc4b | 2012-02-02 16:37:18 -0800 | [diff] [blame] | 432 | |
| 433 | def AddMachine(self, machine_name): |
Ahmad Sharif | 4467f00 | 2012-12-20 12:09:49 -0800 | [diff] [blame] | 434 | with self._lock: |
| 435 | for m in self._all_machines: |
| 436 | assert m.name != machine_name, "Tried to double-add %s" % machine_name |
| 437 | cm = MockCrosMachine(machine_name, self.chromeos_root) |
| 438 | assert cm.machine_checksum, ("Could not find checksum for machine %s" % |
| 439 | machine_name) |
| 440 | self._all_machines.append(cm) |
Ahmad Sharif | 0dcbc4b | 2012-02-02 16:37:18 -0800 | [diff] [blame] | 441 | |
Ahmad Sharif | 4467f00 | 2012-12-20 12:09:49 -0800 | [diff] [blame] | 442 | def AcquireMachine(self, chromeos_image, label): |
| 443 | for machine in self._all_machines: |
Ahmad Sharif | 0dcbc4b | 2012-02-02 16:37:18 -0800 | [diff] [blame] | 444 | if not machine.locked: |
| 445 | machine.locked = True |
| 446 | return machine |
| 447 | return None |
| 448 | |
Ahmad Sharif | 4467f00 | 2012-12-20 12:09:49 -0800 | [diff] [blame] | 449 | def ImageMachine(self, machine_name, label): |
| 450 | return 0 |
| 451 | |
Ahmad Sharif | 0dcbc4b | 2012-02-02 16:37:18 -0800 | [diff] [blame] | 452 | def ReleaseMachine(self, machine): |
| 453 | machine.locked = False |
| 454 | |
Ahmad Sharif | 4467f00 | 2012-12-20 12:09:49 -0800 | [diff] [blame] | 455 | def GetMachines(self, label): |
| 456 | return self._all_machines |
| 457 | |
| 458 | def GetAvailableMachines(self, label): |
| 459 | return self._all_machines |