blob: 7c3343b14e383adc825419865d62337bbe59f014 [file] [log] [blame]
lmr1f61a922009-09-09 22:39:21 +00001"""
2High-level KVM test utility functions.
3
4This module is meant to reduce code size by performing common test procedures.
5Generally, code here should look like test code.
6More specifically:
7 - Functions in this module should raise exceptions if things go wrong
8 (unlike functions in kvm_utils.py and kvm_vm.py which report failure via
9 their returned values).
10 - Functions in this module may use logging.info(), in addition to
11 logging.debug() and logging.error(), to log messages the user may be
12 interested in (unlike kvm_utils.py and kvm_vm.py which use
13 logging.debug() for anything that isn't an error).
14 - Functions in this module typically use functions and classes from
15 lower-level modules (e.g. kvm_utils.py, kvm_vm.py, kvm_subprocess.py).
16 - Functions in this module should not be used by lower-level modules.
17 - Functions in this module should be used in the right context.
18 For example, a function should not be used where it may display
19 misleading or inaccurate info or debug messages.
20
21@copyright: 2008-2009 Red Hat Inc.
22"""
23
Eric Li861b2d52011-02-04 14:50:35 -080024import time, os, logging, re, signal
lmr0bee2342010-02-24 00:01:37 +000025from autotest_lib.client.common_lib import error
26from autotest_lib.client.bin import utils
Eric Li22434d42011-04-28 15:17:19 -070027from autotest_lib.client.tools import scan_results
28import aexpect, virt_utils, virt_vm
lmrb635b862009-09-10 14:53:21 +000029
lmr1f61a922009-09-09 22:39:21 +000030
31def get_living_vm(env, vm_name):
32 """
33 Get a VM object from the environment and make sure it's alive.
34
35 @param env: Dictionary with test environment.
36 @param vm_name: Name of the desired VM object.
37 @return: A VM object.
38 """
Eric Li861b2d52011-02-04 14:50:35 -080039 vm = env.get_vm(vm_name)
lmr1f61a922009-09-09 22:39:21 +000040 if not vm:
41 raise error.TestError("VM '%s' not found in environment" % vm_name)
42 if not vm.is_alive():
43 raise error.TestError("VM '%s' seems to be dead; test requires a "
44 "living VM" % vm_name)
45 return vm
46
47
Eric Li861b2d52011-02-04 14:50:35 -080048def wait_for_login(vm, nic_index=0, timeout=240, start=0, step=2, serial=None):
lmr1f61a922009-09-09 22:39:21 +000049 """
50 Try logging into a VM repeatedly. Stop on success or when timeout expires.
51
52 @param vm: VM object.
53 @param nic_index: Index of NIC to access in the VM.
54 @param timeout: Time to wait before giving up.
Eric Li861b2d52011-02-04 14:50:35 -080055 @param serial: Whether to use a serial connection instead of a remote
56 (ssh, rss) one.
lmr1f61a922009-09-09 22:39:21 +000057 @return: A shell session object.
58 """
Eric Li861b2d52011-02-04 14:50:35 -080059 end_time = time.time() + timeout
60 session = None
61 if serial:
62 type = 'serial'
63 logging.info("Trying to log into guest %s using serial connection,"
64 " timeout %ds", vm.name, timeout)
65 time.sleep(start)
66 while time.time() < end_time:
67 try:
68 session = vm.serial_login()
69 break
Eric Li22434d42011-04-28 15:17:19 -070070 except virt_utils.LoginError, e:
Eric Li861b2d52011-02-04 14:50:35 -080071 logging.debug(e)
72 time.sleep(step)
73 else:
74 type = 'remote'
75 logging.info("Trying to log into guest %s using remote connection,"
76 " timeout %ds", vm.name, timeout)
77 time.sleep(start)
78 while time.time() < end_time:
79 try:
80 session = vm.login(nic_index=nic_index)
81 break
Eric Li22434d42011-04-28 15:17:19 -070082 except (virt_utils.LoginError, virt_vm.VMError), e:
Eric Li861b2d52011-02-04 14:50:35 -080083 logging.debug(e)
84 time.sleep(step)
lmr1f61a922009-09-09 22:39:21 +000085 if not session:
Eric Li861b2d52011-02-04 14:50:35 -080086 raise error.TestFail("Could not log into guest %s using %s connection" %
87 (vm.name, type))
88 logging.info("Logged into guest %s using %s connection", vm.name, type)
lmr1f61a922009-09-09 22:39:21 +000089 return session
lmr60fc92b2009-10-13 13:31:50 +000090
91
lmrb9cb23f2009-10-13 13:38:55 +000092def reboot(vm, session, method="shell", sleep_before_reset=10, nic_index=0,
93 timeout=240):
94 """
95 Reboot the VM and wait for it to come back up by trying to log in until
96 timeout expires.
97
98 @param vm: VM object.
99 @param session: A shell session object.
100 @param method: Reboot method. Can be "shell" (send a shell reboot
101 command) or "system_reset" (send a system_reset monitor command).
102 @param nic_index: Index of NIC to access in the VM, when logging in after
103 rebooting.
104 @param timeout: Time to wait before giving up (after rebooting).
105 @return: A new shell session object.
106 """
107 if method == "shell":
108 # Send a reboot command to the guest's shell
109 session.sendline(vm.get_params().get("reboot_command"))
Dale Curtis456d3c12011-07-19 11:42:51 -0700110 logging.info("Reboot command sent. Waiting for guest to go down")
lmrb9cb23f2009-10-13 13:38:55 +0000111 elif method == "system_reset":
112 # Sleep for a while before sending the command
113 time.sleep(sleep_before_reset)
lmref30e6f2010-06-18 12:51:38 +0000114 # Clear the event list of all QMP monitors
115 monitors = [m for m in vm.monitors if m.protocol == "qmp"]
116 for m in monitors:
117 m.clear_events()
lmrb9cb23f2009-10-13 13:38:55 +0000118 # Send a system_reset monitor command
lmr9e964a02010-06-18 03:46:21 +0000119 vm.monitor.cmd("system_reset")
lmr133d11a2010-03-09 11:31:25 +0000120 logging.info("Monitor command system_reset sent. Waiting for guest to "
Dale Curtis456d3c12011-07-19 11:42:51 -0700121 "go down")
lmref30e6f2010-06-18 12:51:38 +0000122 # Look for RESET QMP events
123 time.sleep(1)
124 for m in monitors:
125 if not m.get_event("RESET"):
126 raise error.TestFail("RESET QMP event not received after "
127 "system_reset (monitor '%s')" % m.name)
128 else:
129 logging.info("RESET QMP event received")
lmrb9cb23f2009-10-13 13:38:55 +0000130 else:
lmr133d11a2010-03-09 11:31:25 +0000131 logging.error("Unknown reboot method: %s", method)
lmrb9cb23f2009-10-13 13:38:55 +0000132
133 # Wait for the session to become unresponsive and close it
Eric Li22434d42011-04-28 15:17:19 -0700134 if not virt_utils.wait_for(lambda: not session.is_responsive(timeout=30),
lmr15652542009-12-02 13:19:47 +0000135 120, 0, 1):
lmrb9cb23f2009-10-13 13:38:55 +0000136 raise error.TestFail("Guest refuses to go down")
137 session.close()
138
139 # Try logging into the guest until timeout expires
lmr133d11a2010-03-09 11:31:25 +0000140 logging.info("Guest is down. Waiting for it to go up again, timeout %ds",
141 timeout)
Eric Li861b2d52011-02-04 14:50:35 -0800142 session = vm.wait_for_login(nic_index, timeout=timeout)
lmrb9cb23f2009-10-13 13:38:55 +0000143 logging.info("Guest is up again")
144 return session
145
146
lmr1424f3e2010-06-17 13:57:09 +0000147def migrate(vm, env=None, mig_timeout=3600, mig_protocol="tcp",
Eric Li861b2d52011-02-04 14:50:35 -0800148 mig_cancel=False, offline=False, stable_check=False,
149 clean=False, save_path=None, dest_host='localhost', mig_port=None):
lmr60fc92b2009-10-13 13:31:50 +0000150 """
151 Migrate a VM locally and re-register it in the environment.
152
153 @param vm: The VM to migrate.
154 @param env: The environment dictionary. If omitted, the migrated VM will
155 not be registered.
lmr1424f3e2010-06-17 13:57:09 +0000156 @param mig_timeout: timeout value for migration.
157 @param mig_protocol: migration protocol
158 @param mig_cancel: Test migrate_cancel or not when protocol is tcp.
Eric Li861b2d52011-02-04 14:50:35 -0800159 @param dest_host: Destination host (defaults to 'localhost').
160 @param mig_port: Port that will be used for migration.
161 @return: The post-migration VM, in case of same host migration, True in
162 case of multi-host migration.
lmr60fc92b2009-10-13 13:31:50 +0000163 """
lmr60fc92b2009-10-13 13:31:50 +0000164 def mig_finished():
lmr9e964a02010-06-18 03:46:21 +0000165 o = vm.monitor.info("migrate")
lmr134e20c2010-06-18 03:49:34 +0000166 if isinstance(o, str):
167 return "status: active" not in o
168 else:
169 return o.get("status") != "active"
lmr60fc92b2009-10-13 13:31:50 +0000170
171 def mig_succeeded():
lmr9e964a02010-06-18 03:46:21 +0000172 o = vm.monitor.info("migrate")
lmr134e20c2010-06-18 03:49:34 +0000173 if isinstance(o, str):
174 return "status: completed" in o
175 else:
176 return o.get("status") == "completed"
lmr1424f3e2010-06-17 13:57:09 +0000177
lmr60fc92b2009-10-13 13:31:50 +0000178 def mig_failed():
lmr9e964a02010-06-18 03:46:21 +0000179 o = vm.monitor.info("migrate")
lmr134e20c2010-06-18 03:49:34 +0000180 if isinstance(o, str):
181 return "status: failed" in o
182 else:
183 return o.get("status") == "failed"
lmr60fc92b2009-10-13 13:31:50 +0000184
lmr9e964a02010-06-18 03:46:21 +0000185 def mig_cancelled():
186 o = vm.monitor.info("migrate")
lmr134e20c2010-06-18 03:49:34 +0000187 if isinstance(o, str):
Eric Lie0493a42010-11-15 13:05:43 -0800188 return ("Migration status: cancelled" in o or
189 "Migration status: canceled" in o)
lmr134e20c2010-06-18 03:49:34 +0000190 else:
Eric Lie0493a42010-11-15 13:05:43 -0800191 return (o.get("status") == "cancelled" or
192 o.get("status") == "canceled")
lmr60fc92b2009-10-13 13:31:50 +0000193
lmr9e964a02010-06-18 03:46:21 +0000194 def wait_for_migration():
Eric Li22434d42011-04-28 15:17:19 -0700195 if not virt_utils.wait_for(mig_finished, mig_timeout, 2, 2,
Dale Curtis456d3c12011-07-19 11:42:51 -0700196 "Waiting for migration to finish"):
lmr9e964a02010-06-18 03:46:21 +0000197 raise error.TestFail("Timeout expired while waiting for migration "
198 "to finish")
199
Eric Li861b2d52011-02-04 14:50:35 -0800200 if dest_host == 'localhost':
201 dest_vm = vm.clone()
lmr9e964a02010-06-18 03:46:21 +0000202
Eric Li861b2d52011-02-04 14:50:35 -0800203 if (dest_host == 'localhost') and stable_check:
204 # Pause the dest vm after creation
205 dest_vm.params['extra_params'] = (dest_vm.params.get('extra_params','')
206 + ' -S')
lmr1424f3e2010-06-17 13:57:09 +0000207
Eric Li861b2d52011-02-04 14:50:35 -0800208 if dest_host == 'localhost':
209 dest_vm.create(migration_mode=mig_protocol, mac_source=vm)
210
211 try:
Eric Lie0493a42010-11-15 13:05:43 -0800212 try:
213 if mig_protocol == "tcp":
Eric Li861b2d52011-02-04 14:50:35 -0800214 if dest_host == 'localhost':
215 uri = "tcp:localhost:%d" % dest_vm.migration_port
216 else:
217 uri = 'tcp:%s:%d' % (dest_host, mig_port)
Eric Lie0493a42010-11-15 13:05:43 -0800218 elif mig_protocol == "unix":
219 uri = "unix:%s" % dest_vm.migration_file
Eric Li861b2d52011-02-04 14:50:35 -0800220 elif mig_protocol == "exec":
221 uri = '"exec:nc localhost %s"' % dest_vm.migration_port
222
223 if offline:
224 vm.monitor.cmd("stop")
Eric Lie0493a42010-11-15 13:05:43 -0800225 vm.monitor.migrate(uri)
lmr60fc92b2009-10-13 13:31:50 +0000226
Eric Lie0493a42010-11-15 13:05:43 -0800227 if mig_cancel:
228 time.sleep(2)
229 vm.monitor.cmd("migrate_cancel")
Eric Li22434d42011-04-28 15:17:19 -0700230 if not virt_utils.wait_for(mig_cancelled, 60, 2, 2,
Eric Lie0493a42010-11-15 13:05:43 -0800231 "Waiting for migration "
232 "cancellation"):
233 raise error.TestFail("Failed to cancel migration")
Eric Li861b2d52011-02-04 14:50:35 -0800234 if offline:
235 vm.monitor.cmd("cont")
236 if dest_host == 'localhost':
237 dest_vm.destroy(gracefully=False)
Eric Lie0493a42010-11-15 13:05:43 -0800238 return vm
239 else:
240 wait_for_migration()
Eric Li861b2d52011-02-04 14:50:35 -0800241 if (dest_host == 'localhost') and stable_check:
242 save_path = None or "/tmp"
243 save1 = os.path.join(save_path, "src")
244 save2 = os.path.join(save_path, "dst")
245
246 vm.save_to_file(save1)
247 dest_vm.save_to_file(save2)
248
249 # Fail if we see deltas
250 md5_save1 = utils.hash_file(save1)
251 md5_save2 = utils.hash_file(save2)
252 if md5_save1 != md5_save2:
253 raise error.TestFail("Mismatch of VM state before "
254 "and after migration")
255
256 if (dest_host == 'localhost') and offline:
257 dest_vm.monitor.cmd("cont")
Eric Lie0493a42010-11-15 13:05:43 -0800258 except:
Eric Li861b2d52011-02-04 14:50:35 -0800259 if dest_host == 'localhost':
260 dest_vm.destroy()
Eric Lie0493a42010-11-15 13:05:43 -0800261 raise
lmr9ef2fe62010-06-23 16:52:16 +0000262
Eric Li861b2d52011-02-04 14:50:35 -0800263 finally:
264 if (dest_host == 'localhost') and stable_check and clean:
265 logging.debug("Cleaning the state files")
266 if os.path.isfile(save1):
267 os.remove(save1)
268 if os.path.isfile(save2):
269 os.remove(save2)
270
Eric Lie0493a42010-11-15 13:05:43 -0800271 # Report migration status
272 if mig_succeeded():
273 logging.info("Migration finished successfully")
274 elif mig_failed():
275 raise error.TestFail("Migration failed")
276 else:
277 raise error.TestFail("Migration ended with unknown status")
lmr1424f3e2010-06-17 13:57:09 +0000278
Eric Li861b2d52011-02-04 14:50:35 -0800279 if dest_host == 'localhost':
280 if "paused" in dest_vm.monitor.info("status"):
Dale Curtis456d3c12011-07-19 11:42:51 -0700281 logging.debug("Destination VM is paused, resuming it")
Eric Li861b2d52011-02-04 14:50:35 -0800282 dest_vm.monitor.cmd("cont")
lmr60fc92b2009-10-13 13:31:50 +0000283
Eric Lie0493a42010-11-15 13:05:43 -0800284 # Kill the source VM
285 vm.destroy(gracefully=False)
lmr60fc92b2009-10-13 13:31:50 +0000286
Eric Lie0493a42010-11-15 13:05:43 -0800287 # Replace the source VM with the new cloned VM
Eric Li861b2d52011-02-04 14:50:35 -0800288 if (dest_host == 'localhost') and (env is not None):
289 env.register_vm(vm.name, dest_vm)
lmr60fc92b2009-10-13 13:31:50 +0000290
Eric Lie0493a42010-11-15 13:05:43 -0800291 # Return the new cloned VM
Eric Li861b2d52011-02-04 14:50:35 -0800292 if dest_host == 'localhost':
293 return dest_vm
294 else:
295 return vm
Eric Lie0493a42010-11-15 13:05:43 -0800296
297
298def stop_windows_service(session, service, timeout=120):
299 """
300 Stop a Windows service using sc.
301 If the service is already stopped or is not installed, do nothing.
302
303 @param service: The name of the service
304 @param timeout: Time duration to wait for service to stop
305 @raise error.TestError: Raised if the service can't be stopped
306 """
307 end_time = time.time() + timeout
308 while time.time() < end_time:
Eric Li861b2d52011-02-04 14:50:35 -0800309 o = session.cmd_output("sc stop %s" % service, timeout=60)
Eric Lie0493a42010-11-15 13:05:43 -0800310 # FAILED 1060 means the service isn't installed.
311 # FAILED 1062 means the service hasn't been started.
312 if re.search(r"\bFAILED (1060|1062)\b", o, re.I):
313 break
314 time.sleep(1)
315 else:
316 raise error.TestError("Could not stop service '%s'" % service)
317
318
319def start_windows_service(session, service, timeout=120):
320 """
321 Start a Windows service using sc.
322 If the service is already running, do nothing.
323 If the service isn't installed, fail.
324
325 @param service: The name of the service
326 @param timeout: Time duration to wait for service to start
327 @raise error.TestError: Raised if the service can't be started
328 """
329 end_time = time.time() + timeout
330 while time.time() < end_time:
Eric Li861b2d52011-02-04 14:50:35 -0800331 o = session.cmd_output("sc start %s" % service, timeout=60)
Eric Lie0493a42010-11-15 13:05:43 -0800332 # FAILED 1060 means the service isn't installed.
333 if re.search(r"\bFAILED 1060\b", o, re.I):
334 raise error.TestError("Could not start service '%s' "
335 "(service not installed)" % service)
336 # FAILED 1056 means the service is already running.
337 if re.search(r"\bFAILED 1056\b", o, re.I):
338 break
339 time.sleep(1)
340 else:
341 raise error.TestError("Could not start service '%s'" % service)
lmraea0d492009-10-13 13:33:12 +0000342
343
344def get_time(session, time_command, time_filter_re, time_format):
345 """
346 Return the host time and guest time. If the guest time cannot be fetched
347 a TestError exception is raised.
348
349 Note that the shell session should be ready to receive commands
350 (i.e. should "display" a command prompt and should be done with all
351 previous commands).
352
353 @param session: A shell session.
354 @param time_command: Command to issue to get the current guest time.
355 @param time_filter_re: Regex filter to apply on the output of
356 time_command in order to get the current time.
357 @param time_format: Format string to pass to time.strptime() with the
358 result of the regex filter.
359 @return: A tuple containing the host time and guest time.
360 """
Eric Li6f27d4f2010-09-29 10:55:17 -0700361 if len(re.findall("ntpdate|w32tm", time_command)) == 0:
362 host_time = time.time()
Eric Li861b2d52011-02-04 14:50:35 -0800363 s = session.cmd_output(time_command)
lmrb40e8bb2010-06-14 17:32:59 +0000364
Eric Li6f27d4f2010-09-29 10:55:17 -0700365 try:
366 s = re.findall(time_filter_re, s)[0]
367 except IndexError:
Eric Li861b2d52011-02-04 14:50:35 -0800368 logging.debug("The time string from guest is:\n%s", s)
Eric Li6f27d4f2010-09-29 10:55:17 -0700369 raise error.TestError("The time string from guest is unexpected.")
370 except Exception, e:
Eric Li861b2d52011-02-04 14:50:35 -0800371 logging.debug("(time_filter_re, time_string): (%s, %s)",
372 time_filter_re, s)
Eric Li6f27d4f2010-09-29 10:55:17 -0700373 raise e
lmrb40e8bb2010-06-14 17:32:59 +0000374
Eric Li6f27d4f2010-09-29 10:55:17 -0700375 guest_time = time.mktime(time.strptime(s, time_format))
376 else:
Eric Li861b2d52011-02-04 14:50:35 -0800377 o = session.cmd(time_command)
Eric Li6f27d4f2010-09-29 10:55:17 -0700378 if re.match('ntpdate', time_command):
Eric Li861b2d52011-02-04 14:50:35 -0800379 offset = re.findall('offset (.*) sec', o)[0]
Eric Li6f27d4f2010-09-29 10:55:17 -0700380 host_main, host_mantissa = re.findall(time_filter_re, o)[0]
Eric Li861b2d52011-02-04 14:50:35 -0800381 host_time = (time.mktime(time.strptime(host_main, time_format)) +
382 float("0.%s" % host_mantissa))
Dale Curtis74a314b2011-06-23 14:55:46 -0700383 guest_time = host_time - float(offset)
Eric Li6f27d4f2010-09-29 10:55:17 -0700384 else:
385 guest_time = re.findall(time_filter_re, o)[0]
386 offset = re.findall("o:(.*)s", o)[0]
387 if re.match('PM', guest_time):
388 hour = re.findall('\d+ (\d+):', guest_time)[0]
389 hour = str(int(hour) + 12)
390 guest_time = re.sub('\d+\s\d+:', "\d+\s%s:" % hour,
391 guest_time)[:-3]
392 else:
393 guest_time = guest_time[:-3]
394 guest_time = time.mktime(time.strptime(guest_time, time_format))
Dale Curtis74a314b2011-06-23 14:55:46 -0700395 host_time = guest_time + float(offset)
Eric Li6f27d4f2010-09-29 10:55:17 -0700396
lmraea0d492009-10-13 13:33:12 +0000397 return (host_time, guest_time)
lmr0bee2342010-02-24 00:01:37 +0000398
399
400def get_memory_info(lvms):
401 """
402 Get memory information from host and guests in format:
403 Host: memfree = XXXM; Guests memsh = {XXX,XXX,...}
404
405 @params lvms: List of VM objects
406 @return: String with memory info report
407 """
408 if not isinstance(lvms, list):
409 raise error.TestError("Invalid list passed to get_stat: %s " % lvms)
410
411 try:
412 meminfo = "Host: memfree = "
413 meminfo += str(int(utils.freememtotal()) / 1024) + "M; "
414 meminfo += "swapfree = "
415 mf = int(utils.read_from_meminfo("SwapFree")) / 1024
416 meminfo += str(mf) + "M; "
417 except Exception, e:
418 raise error.TestFail("Could not fetch host free memory info, "
419 "reason: %s" % e)
420
421 meminfo += "Guests memsh = {"
422 for vm in lvms:
423 shm = vm.get_shared_meminfo()
424 if shm is None:
425 raise error.TestError("Could not get shared meminfo from "
426 "VM %s" % vm)
427 meminfo += "%dM; " % shm
428 meminfo = meminfo[0:-2] + "}"
429
430 return meminfo
lmr3c11f252010-03-17 13:00:07 +0000431
432
Eric Li861b2d52011-02-04 14:50:35 -0800433def run_autotest(vm, session, control_path, timeout, outputdir, params):
lmr3c11f252010-03-17 13:00:07 +0000434 """
435 Run an autotest control file inside a guest (linux only utility).
436
437 @param vm: VM object.
438 @param session: A shell session on the VM provided.
lmrce013d62010-04-15 00:01:17 +0000439 @param control_path: A path to an autotest control file.
440 @param timeout: Timeout under which the autotest control file must complete.
lmr3c11f252010-03-17 13:00:07 +0000441 @param outputdir: Path on host where we should copy the guest autotest
442 results to.
Eric Li861b2d52011-02-04 14:50:35 -0800443
444 The following params is used by the migration
445 @param params: Test params used in the migration test
lmr3c11f252010-03-17 13:00:07 +0000446 """
lmr63ef61d2010-04-17 17:57:41 +0000447 def copy_if_hash_differs(vm, local_path, remote_path):
lmr3c11f252010-03-17 13:00:07 +0000448 """
lmr63ef61d2010-04-17 17:57:41 +0000449 Copy a file to a guest if it doesn't exist or if its MD5sum differs.
lmr3c11f252010-03-17 13:00:07 +0000450
451 @param vm: VM object.
452 @param local_path: Local path.
453 @param remote_path: Remote path.
Dale Curtis8adf7892011-09-08 16:13:36 -0700454
455 @return: Whether the hash differs (True) or not (False).
lmr3c11f252010-03-17 13:00:07 +0000456 """
Dale Curtis8adf7892011-09-08 16:13:36 -0700457 hash_differs = False
lmr63ef61d2010-04-17 17:57:41 +0000458 local_hash = utils.hash_file(local_path)
lmr3c11f252010-03-17 13:00:07 +0000459 basename = os.path.basename(local_path)
Eric Li861b2d52011-02-04 14:50:35 -0800460 output = session.cmd_output("md5sum %s" % remote_path)
lmr3c11f252010-03-17 13:00:07 +0000461 if "such file" in output:
lmr63ef61d2010-04-17 17:57:41 +0000462 remote_hash = "0"
463 elif output:
464 remote_hash = output.split()[0]
lmr3c11f252010-03-17 13:00:07 +0000465 else:
lmr63ef61d2010-04-17 17:57:41 +0000466 logging.warning("MD5 check for remote path %s did not return.",
467 remote_path)
468 # Let's be a little more lenient here and see if it wasn't a
469 # temporary problem
470 remote_hash = "0"
lmr63ef61d2010-04-17 17:57:41 +0000471 if remote_hash != local_hash:
Dale Curtis8adf7892011-09-08 16:13:36 -0700472 hash_differs = True
473 logging.debug("Copying %s to guest "
474 "(remote hash: %s, local hash:%s)",
475 basename, remote_hash, local_hash)
Eric Li861b2d52011-02-04 14:50:35 -0800476 vm.copy_files_to(local_path, remote_path)
Dale Curtis8adf7892011-09-08 16:13:36 -0700477 return hash_differs
lmr3c11f252010-03-17 13:00:07 +0000478
479
Dale Curtis8adf7892011-09-08 16:13:36 -0700480 def extract(vm, remote_path, dest_dir):
lmr3c11f252010-03-17 13:00:07 +0000481 """
Dale Curtis8adf7892011-09-08 16:13:36 -0700482 Extract the autotest .tar.bz2 file on the guest, ensuring the final
483 destination path will be dest_dir.
lmr3c11f252010-03-17 13:00:07 +0000484
485 @param vm: VM object
486 @param remote_path: Remote file path
487 @param dest_dir: Destination dir for the contents
488 """
489 basename = os.path.basename(remote_path)
Dale Curtis456d3c12011-07-19 11:42:51 -0700490 logging.debug("Extracting %s on VM %s", basename, vm.name)
Dale Curtis8adf7892011-09-08 16:13:36 -0700491 session.cmd("rm -rf %s" % dest_dir)
492 dirname = os.path.dirname(remote_path)
493 session.cmd("cd %s" % dirname)
494 session.cmd("mkdir -p %s" % os.path.dirname(dest_dir))
495 e_cmd = "tar xjvf %s -C %s" % (basename, os.path.dirname(dest_dir))
496 output = session.cmd(e_cmd, timeout=120)
497 autotest_dirname = ""
498 for line in output.splitlines():
499 autotest_dirname = line.split("/")[0]
500 break
501 if autotest_dirname != os.path.basename(dest_dir):
502 session.cmd("cd %s" % os.path.dirname(dest_dir))
503 session.cmd("mv %s %s" %
504 (autotest_dirname, os.path.basename(dest_dir)))
lmr3c11f252010-03-17 13:00:07 +0000505
lmrce013d62010-04-15 00:01:17 +0000506
Dale Curtis8adf7892011-09-08 16:13:36 -0700507 def get_results(guest_autotest_path):
lmrce013d62010-04-15 00:01:17 +0000508 """
509 Copy autotest results present on the guest back to the host.
510 """
Dale Curtis456d3c12011-07-19 11:42:51 -0700511 logging.debug("Trying to copy autotest results from guest")
lmrce013d62010-04-15 00:01:17 +0000512 guest_results_dir = os.path.join(outputdir, "guest_autotest_results")
513 if not os.path.exists(guest_results_dir):
514 os.mkdir(guest_results_dir)
Dale Curtis8adf7892011-09-08 16:13:36 -0700515 vm.copy_files_from("%s/results/default/*" % guest_autotest_path,
Eric Li861b2d52011-02-04 14:50:35 -0800516 guest_results_dir)
lmrce013d62010-04-15 00:01:17 +0000517
518
Dale Curtis8adf7892011-09-08 16:13:36 -0700519 def get_results_summary(guest_autotest_path):
lmrce013d62010-04-15 00:01:17 +0000520 """
521 Get the status of the tests that were executed on the host and close
522 the session where autotest was being executed.
523 """
Dale Curtis8adf7892011-09-08 16:13:36 -0700524 session.cmd("cd %s" % guest_autotest_path)
Eric Li861b2d52011-02-04 14:50:35 -0800525 output = session.cmd_output("cat results/*/status")
lmr63ef61d2010-04-17 17:57:41 +0000526 try:
527 results = scan_results.parse_results(output)
528 # Report test results
529 logging.info("Results (test, status, duration, info):")
530 for result in results:
531 logging.info(str(result))
532 session.close()
533 return results
534 except Exception, e:
535 logging.error("Error processing guest autotest results: %s", e)
536 return None
lmrce013d62010-04-15 00:01:17 +0000537
538
lmr3c11f252010-03-17 13:00:07 +0000539 if not os.path.isfile(control_path):
540 raise error.TestError("Invalid path to autotest control file: %s" %
541 control_path)
542
Eric Li861b2d52011-02-04 14:50:35 -0800543 migrate_background = params.get("migrate_background") == "yes"
544 if migrate_background:
545 mig_timeout = float(params.get("mig_timeout", "3600"))
546 mig_protocol = params.get("migration_protocol", "tcp")
547
Dale Curtis8adf7892011-09-08 16:13:36 -0700548 compressed_autotest_path = "/tmp/autotest.tar.bz2"
549 destination_autotest_path = "/usr/local/autotest"
lmr3c11f252010-03-17 13:00:07 +0000550
551 # To avoid problems, let's make the test use the current AUTODIR
552 # (autotest client path) location
Dale Curtis8adf7892011-09-08 16:13:36 -0700553 autotest_path = os.environ['AUTODIR']
554 autotest_basename = os.path.basename(autotest_path)
555 autotest_parentdir = os.path.dirname(autotest_path)
lmr3c11f252010-03-17 13:00:07 +0000556
557 # tar the contents of bindir/autotest
Dale Curtis8adf7892011-09-08 16:13:36 -0700558 cmd = ("cd %s; tar cvjf %s %s/*" %
559 (autotest_parentdir, compressed_autotest_path, autotest_basename))
lmrce013d62010-04-15 00:01:17 +0000560 # Until we have nested virtualization, we don't need the kvm test :)
Dale Curtis8adf7892011-09-08 16:13:36 -0700561 cmd += " --exclude=%s/tests/kvm" % autotest_basename
562 cmd += " --exclude=%s/results" % autotest_basename
563 cmd += " --exclude=%s/tmp" % autotest_basename
564 cmd += " --exclude=%s/control*" % autotest_basename
lmr3c11f252010-03-17 13:00:07 +0000565 cmd += " --exclude=*.pyc"
566 cmd += " --exclude=*.svn"
567 cmd += " --exclude=*.git"
568 utils.run(cmd)
569
lmr3c11f252010-03-17 13:00:07 +0000570 # Copy autotest.tar.bz2
Dale Curtis8adf7892011-09-08 16:13:36 -0700571 update = copy_if_hash_differs(vm, compressed_autotest_path,
572 compressed_autotest_path)
lmr3c11f252010-03-17 13:00:07 +0000573
574 # Extract autotest.tar.bz2
Dale Curtis8adf7892011-09-08 16:13:36 -0700575 if update:
576 extract(vm, compressed_autotest_path, destination_autotest_path)
lmr3c11f252010-03-17 13:00:07 +0000577
Dale Curtis8adf7892011-09-08 16:13:36 -0700578 vm.copy_files_to(control_path,
579 os.path.join(destination_autotest_path, 'control'))
lmr3c11f252010-03-17 13:00:07 +0000580
581 # Run the test
lmr63ef61d2010-04-17 17:57:41 +0000582 logging.info("Running autotest control file %s on guest, timeout %ss",
583 os.path.basename(control_path), timeout)
Dale Curtis8adf7892011-09-08 16:13:36 -0700584 session.cmd("cd %s" % destination_autotest_path)
Eric Li861b2d52011-02-04 14:50:35 -0800585 try:
586 session.cmd("rm -f control.state")
587 session.cmd("rm -rf results/*")
Eric Li22434d42011-04-28 15:17:19 -0700588 except aexpect.ShellError:
Eric Li861b2d52011-02-04 14:50:35 -0800589 pass
590 try:
591 bg = None
592 try:
593 logging.info("---------------- Test output ----------------")
594 if migrate_background:
595 mig_timeout = float(params.get("mig_timeout", "3600"))
596 mig_protocol = params.get("migration_protocol", "tcp")
597
Eric Li22434d42011-04-28 15:17:19 -0700598 bg = virt_utils.Thread(session.cmd_output,
Eric Li861b2d52011-02-04 14:50:35 -0800599 kwargs={'cmd': "bin/autotest control",
600 'timeout': timeout,
601 'print_func': logging.info})
602
603 bg.start()
604
605 while bg.is_alive():
Dale Curtis8adf7892011-09-08 16:13:36 -0700606 logging.info("Autotest job did not end, start a round of "
Dale Curtis456d3c12011-07-19 11:42:51 -0700607 "migration")
Eric Li861b2d52011-02-04 14:50:35 -0800608 vm.migrate(timeout=mig_timeout, protocol=mig_protocol)
609 else:
610 session.cmd_output("bin/autotest control", timeout=timeout,
611 print_func=logging.info)
612 finally:
613 logging.info("------------- End of test output ------------")
614 if migrate_background and bg:
615 bg.join()
Eric Li22434d42011-04-28 15:17:19 -0700616 except aexpect.ShellTimeoutError:
Eric Li861b2d52011-02-04 14:50:35 -0800617 if vm.is_alive():
Dale Curtis8adf7892011-09-08 16:13:36 -0700618 get_results(destination_autotest_path)
619 get_results_summary(destination_autotest_path)
Eric Li861b2d52011-02-04 14:50:35 -0800620 raise error.TestError("Timeout elapsed while waiting for job to "
621 "complete")
622 else:
lmr63ef61d2010-04-17 17:57:41 +0000623 raise error.TestError("Autotest job on guest failed "
624 "(VM terminated during job)")
Eric Li22434d42011-04-28 15:17:19 -0700625 except aexpect.ShellProcessTerminatedError:
Dale Curtis8adf7892011-09-08 16:13:36 -0700626 get_results(destination_autotest_path)
Eric Li861b2d52011-02-04 14:50:35 -0800627 raise error.TestError("Autotest job on guest failed "
628 "(Remote session terminated during job)")
lmr3c11f252010-03-17 13:00:07 +0000629
Dale Curtis8adf7892011-09-08 16:13:36 -0700630 results = get_results_summary(destination_autotest_path)
631 get_results(destination_autotest_path)
lmr3c11f252010-03-17 13:00:07 +0000632
633 # Make a list of FAIL/ERROR/ABORT results (make sure FAIL results appear
634 # before ERROR results, and ERROR results appear before ABORT results)
lmrce013d62010-04-15 00:01:17 +0000635 bad_results = [r[0] for r in results if r[1] == "FAIL"]
636 bad_results += [r[0] for r in results if r[1] == "ERROR"]
637 bad_results += [r[0] for r in results if r[1] == "ABORT"]
lmr3c11f252010-03-17 13:00:07 +0000638
639 # Fail the test if necessary
640 if not results:
lmrce013d62010-04-15 00:01:17 +0000641 raise error.TestFail("Autotest control file run did not produce any "
642 "recognizable results")
lmr3c11f252010-03-17 13:00:07 +0000643 if bad_results:
lmrce013d62010-04-15 00:01:17 +0000644 if len(bad_results) == 1:
lmr63ef61d2010-04-17 17:57:41 +0000645 e_msg = ("Test %s failed during control file execution" %
646 bad_results[0])
lmrce013d62010-04-15 00:01:17 +0000647 else:
648 e_msg = ("Tests %s failed during control file execution" %
649 " ".join(bad_results))
650 raise error.TestFail(e_msg)
Eric Lie0493a42010-11-15 13:05:43 -0800651
652
653def get_loss_ratio(output):
654 """
655 Get the packet loss ratio from the output of ping
656.
657 @param output: Ping output.
658 """
659 try:
660 return int(re.findall('(\d+)% packet loss', output)[0])
661 except IndexError:
662 logging.debug(output)
663 return -1
664
665
666def raw_ping(command, timeout, session, output_func):
667 """
668 Low-level ping command execution.
669
670 @param command: Ping command.
671 @param timeout: Timeout of the ping command.
672 @param session: Local executon hint or session to execute the ping command.
673 """
674 if session is None:
Eric Li22434d42011-04-28 15:17:19 -0700675 process = aexpect.run_bg(command, output_func=output_func,
Eric Lie0493a42010-11-15 13:05:43 -0800676 timeout=timeout)
677
678 # Send SIGINT signal to notify the timeout of running ping process,
679 # Because ping have the ability to catch the SIGINT signal so we can
680 # always get the packet loss ratio even if timeout.
681 if process.is_alive():
Eric Li22434d42011-04-28 15:17:19 -0700682 virt_utils.kill_process_tree(process.get_pid(), signal.SIGINT)
Eric Lie0493a42010-11-15 13:05:43 -0800683
684 status = process.get_status()
685 output = process.get_output()
686
687 process.close()
688 return status, output
689 else:
Eric Li861b2d52011-02-04 14:50:35 -0800690 output = ""
691 try:
692 output = session.cmd_output(command, timeout=timeout,
693 print_func=output_func)
Eric Li22434d42011-04-28 15:17:19 -0700694 except aexpect.ShellTimeoutError:
Eric Lie0493a42010-11-15 13:05:43 -0800695 # Send ctrl+c (SIGINT) through ssh session
696 session.send("\003")
Eric Li861b2d52011-02-04 14:50:35 -0800697 try:
698 output2 = session.read_up_to_prompt(print_func=output_func)
699 output += output2
Eric Li22434d42011-04-28 15:17:19 -0700700 except aexpect.ExpectTimeoutError, e:
Eric Li861b2d52011-02-04 14:50:35 -0800701 output += e.output
Eric Lie0493a42010-11-15 13:05:43 -0800702 # We also need to use this session to query the return value
703 session.send("\003")
704
705 session.sendline(session.status_test_command)
Eric Li861b2d52011-02-04 14:50:35 -0800706 try:
707 o2 = session.read_up_to_prompt()
Eric Li22434d42011-04-28 15:17:19 -0700708 except aexpect.ExpectError:
Eric Lie0493a42010-11-15 13:05:43 -0800709 status = -1
710 else:
711 try:
712 status = int(re.findall("\d+", o2)[0])
713 except:
714 status = -1
715
716 return status, output
717
718
719def ping(dest=None, count=None, interval=None, interface=None,
720 packetsize=None, ttl=None, hint=None, adaptive=False,
721 broadcast=False, flood=False, timeout=0,
722 output_func=logging.debug, session=None):
723 """
724 Wrapper of ping.
725
726 @param dest: Destination address.
727 @param count: Count of icmp packet.
728 @param interval: Interval of two icmp echo request.
729 @param interface: Specified interface of the source address.
730 @param packetsize: Packet size of icmp.
731 @param ttl: IP time to live.
732 @param hint: Path mtu discovery hint.
733 @param adaptive: Adaptive ping flag.
734 @param broadcast: Broadcast ping flag.
735 @param flood: Flood ping flag.
736 @param timeout: Timeout for the ping command.
737 @param output_func: Function used to log the result of ping.
738 @param session: Local executon hint or session to execute the ping command.
739 """
740 if dest is not None:
741 command = "ping %s " % dest
742 else:
743 command = "ping localhost "
744 if count is not None:
745 command += " -c %s" % count
746 if interval is not None:
747 command += " -i %s" % interval
748 if interface is not None:
749 command += " -I %s" % interface
750 if packetsize is not None:
751 command += " -s %s" % packetsize
752 if ttl is not None:
753 command += " -t %s" % ttl
754 if hint is not None:
755 command += " -M %s" % hint
756 if adaptive:
757 command += " -A"
758 if broadcast:
759 command += " -b"
760 if flood:
761 command += " -f -q"
762 output_func = None
763
764 return raw_ping(command, timeout, session, output_func)
765
766
767def get_linux_ifname(session, mac_address):
768 """
769 Get the interface name through the mac address.
770
771 @param session: session to the virtual machine
772 @mac_address: the macaddress of nic
773 """
774
Eric Li861b2d52011-02-04 14:50:35 -0800775 output = session.cmd_output("ifconfig -a")
Eric Lie0493a42010-11-15 13:05:43 -0800776
777 try:
778 ethname = re.findall("(\w+)\s+Link.*%s" % mac_address, output,
779 re.IGNORECASE)[0]
780 return ethname
781 except:
782 return None