Merge remote branch 'cros/upstream' into autotest-rebase
Merged to upstream trunk@5066, from trunk@4749.
There is no way I could enlist each individual CL from the upstream here since it will blow up the changelist description field.
BUG=
TEST=
Had patched this CL into a fresh cut client to avoid any side effect.
run_remote_test bvt from both emerged location and third_party/autotest/file.
Both test passed!
We should also keep any eye on this to see how it gets propagated into cautotest server.
TBR=dalecurtis
Change-Id: I72f2bc7a9de530178484aea1bfb5ace68bcad029
diff --git a/server/autotest.py b/server/autotest.py
index eb25095..74a76c7 100644
--- a/server/autotest.py
+++ b/server/autotest.py
@@ -76,6 +76,7 @@
try:
autotest_binary = os.path.join(path, 'bin', 'autotest')
host.run('test -x %s' % utils.sh_escape(autotest_binary))
+ host.run('test -w %s' % utils.sh_escape(path))
logging.debug('Found existing autodir at %s', path)
return path
except error.AutoservRunError:
@@ -109,6 +110,7 @@
for path in client_autodir_paths:
try:
host.run('mkdir -p %s' % utils.sh_escape(path))
+ host.run('test -w %s' % utils.sh_escape(path))
return path
except error.AutoservRunError:
logging.debug('Failed to create %s', path)
diff --git a/server/autotest_unittest.py b/server/autotest_unittest.py
index 6025af6..04023e9 100755
--- a/server/autotest_unittest.py
+++ b/server/autotest_unittest.py
@@ -256,6 +256,7 @@
self.host.get_autodir.expect_call().and_return(None)
self._expect_failed_run('test -x /some/path/bin/autotest')
self.host.run.expect_call('test -x /another/path/bin/autotest')
+ self.host.run.expect_call('test -w /another/path')
autodir = autotest.Autotest.get_installed_autodir(self.host)
self.assertEquals(autodir, '/another/path')
@@ -268,6 +269,7 @@
self._expect_failed_run('test -x /another/path/bin/autotest')
self._expect_failed_run('mkdir -p /some/path')
self.host.run.expect_call('mkdir -p /another/path')
+ self.host.run.expect_call('test -w /another/path')
install_dir = autotest.Autotest.get_install_dir(self.host)
self.assertEquals(install_dir, '/another/path')
diff --git a/server/base_utils.py b/server/base_utils.py
index 6d683b2..1c58609 100644
--- a/server/base_utils.py
+++ b/server/base_utils.py
@@ -244,45 +244,27 @@
return (ntuples, failures)
-def parse_machine(machine, user = 'root', port = 22, password = ''):
+def parse_machine(machine, user='root', password='', port=22):
"""
Parse the machine string user:pass@host:port and return it separately,
if the machine string is not complete, use the default parameters
when appropriate.
"""
- user = user
- port = port
- password = password
+ if '@' in machine:
+ user, machine = machine.split('@', 1)
- if re.search('@', machine):
- machine = machine.split('@')
+ if ':' in user:
+ user, password = user.split(':', 1)
- if re.search(':', machine[0]):
- machine[0] = machine[0].split(':')
- user = machine[0][0]
- password = machine[0][1]
+ if ':' in machine:
+ machine, port = machine.split(':', 1)
+ port = int(port)
- else:
- user = machine[0]
+ if not machine or not user:
+ raise ValueError
- if re.search(':', machine[1]):
- machine[1] = machine[1].split(':')
- hostname = machine[1][0]
- port = int(machine[1][1])
-
- else:
- hostname = machine[1]
-
- elif re.search(':', machine):
- machine = machine.split(':')
- hostname = machine[0]
- port = int(machine[1])
-
- else:
- hostname = machine
-
- return hostname, user, password, port
+ return machine, user, password, port
def get_public_key():
diff --git a/server/base_utils_unittest.py b/server/base_utils_unittest.py
index 111dfbe..feb9ace 100755
--- a/server/base_utils_unittest.py
+++ b/server/base_utils_unittest.py
@@ -26,5 +26,37 @@
self.assertEquals(self.failures, failures)
+ # parse_machine() test cases
+ def test_parse_machine_good(self):
+ '''test that parse_machine() is outputting the correct data'''
+ gooddata = (('host', ('host', 'root', '', 22)),
+ ('host:21', ('host', 'root', '', 21)),
+ ('user@host', ('host', 'user', '', 22)),
+ ('user:pass@host', ('host', 'user', 'pass', 22)),
+ ('user:pass@host:1234', ('host', 'user', 'pass', 1234)),
+ )
+ for machine, result in gooddata:
+ self.assertEquals(utils.parse_machine(machine), result)
+
+
+ def test_parse_machine_override(self):
+ '''Test that parse_machine() defaults can be overridden'''
+ self.assertEquals(utils.parse_machine('host', 'bob', 'foo', 1234),
+ ('host', 'bob', 'foo', 1234))
+
+
+ def test_parse_machine_bad(self):
+ '''test that bad data passed to parse_machine() will raise an exception'''
+ baddata = (('host:port', ValueError), # pass a non-integer string for port
+ ('host:22:33', ValueError), # pass two ports
+ (':22', ValueError), # neglect to pass a hostname #1
+ ('user@', ValueError), # neglect to pass a hostname #2
+ ('user@:22', ValueError), # neglect to pass a hostname #3
+ (':pass@host', ValueError), # neglect to pass a username
+ )
+ for machine, exception in baddata:
+ self.assertRaises(exception, utils.parse_machine, machine)
+
+
if __name__ == "__main__":
unittest.main()
diff --git a/server/control_segments/cleanup b/server/control_segments/cleanup
index 1a201bd..6ac6788 100644
--- a/server/control_segments/cleanup
+++ b/server/control_segments/cleanup
@@ -1,8 +1,5 @@
def cleanup(machine):
- hostname, user, passwd, port = parse_machine(machine, ssh_user,
- ssh_port, ssh_pass)
- host = hosts.create_host(hostname, user=user, port=port, initialize=False,
- password=passwd, auto_monitor=False)
+ host = hosts.create_host(machine, initialize=False, auto_monitor=False)
host.cleanup()
diff --git a/server/control_segments/client_wrapper b/server/control_segments/client_wrapper
index f8acae3..0d29c7e 100644
--- a/server/control_segments/client_wrapper
+++ b/server/control_segments/client_wrapper
@@ -2,9 +2,7 @@
def run_client(machine):
- hostname, user, passwd, port = parse_machine(machine, ssh_user, ssh_port,
- ssh_pass)
- host = hosts.create_host(hostname, user=user, port=port, password=passwd)
+ host = hosts.create_host(machine)
host.log_kernel()
at.run(control, host=host)
diff --git a/server/control_segments/crashdumps b/server/control_segments/crashdumps
index 7bad63d..b5ebe78 100644
--- a/server/control_segments/crashdumps
+++ b/server/control_segments/crashdumps
@@ -2,10 +2,7 @@
def crashdumps(machine):
- hostname, user, passwd, port = parse_machine(machine, ssh_user,
- ssh_port, ssh_pass)
- host = hosts.create_host(hostname, user=user, port=port, initialize=False,
- password=passwd, auto_monitor=False)
+ host = hosts.create_host(machine, initialize=False, auto_monitor=False)
crashcollect.get_crashdumps(host, test_start_time)
diff --git a/server/control_segments/crashinfo b/server/control_segments/crashinfo
index c273620..c802832 100644
--- a/server/control_segments/crashinfo
+++ b/server/control_segments/crashinfo
@@ -2,10 +2,7 @@
def crashinfo(machine):
- hostname, user, passwd, port = parse_machine(machine, ssh_user,
- ssh_port, ssh_pass)
- host = hosts.create_host(hostname, user=user, port=port, initialize=False,
- password=passwd, auto_monitor=False)
+ host = hosts.create_host(machine, initialize=False, auto_monitor=False)
crashcollect.get_crashinfo(host, test_start_time)
diff --git a/server/control_segments/install b/server/control_segments/install
index aba79ae..20d6944 100644
--- a/server/control_segments/install
+++ b/server/control_segments/install
@@ -1,8 +1,5 @@
def install(machine):
- hostname, user, passwd, port = parse_machine(machine, ssh_user,
- ssh_port, ssh_pass)
- host = hosts.create_host(hostname, user=user, port=port, initialize=False,
- password=passwd, auto_monitor=False)
+ host = hosts.create_host(machine, initialize=False, auto_monitor=False)
host.machine_install()
diff --git a/server/hosts/abstract_ssh.py b/server/hosts/abstract_ssh.py
index 3d8d9e9..3723c46 100644
--- a/server/hosts/abstract_ssh.py
+++ b/server/hosts/abstract_ssh.py
@@ -115,6 +115,17 @@
" ".join(sources), dest)
+ def _make_ssh_cmd(self, cmd):
+ """
+ Create a base ssh command string for the host which can be used
+ to run commands directly on the machine
+ """
+ base_cmd = make_ssh_command(user=self.user, port=self.port,
+ opts=self.master_ssh_option,
+ hosts_file=self.known_hosts_fd)
+
+ return '%s %s "%s"' % (base_cmd, self.hostname, utils.sh_escape(cmd))
+
def _make_scp_cmd(self, sources, dest):
"""
Given a list of source paths and a destination path, produces the
diff --git a/server/hosts/factory.py b/server/hosts/factory.py
index f1a054f..7a2a724 100644
--- a/server/hosts/factory.py
+++ b/server/hosts/factory.py
@@ -68,9 +68,8 @@
site_factory.postprocess_classes(classes, hostname,
auto_monitor=auto_monitor, **args)
- args['user'] = ssh_user
- args['port'] = ssh_port
- args['password'] = ssh_pass
+ hostname, args['user'], args['password'], args['port'] = \
+ server_utils.parse_machine(hostname, ssh_user, ssh_pass, ssh_port)
# create a custom host class for this machine and return an instance of it
host_class = type("%s_host" % hostname, tuple(classes), {})
diff --git a/server/hosts/logfile_monitor.py b/server/hosts/logfile_monitor.py
index 1608a6b..9595cc8 100644
--- a/server/hosts/logfile_monitor.py
+++ b/server/hosts/logfile_monitor.py
@@ -26,15 +26,6 @@
"""Error occurred launching followfiles remotely."""
-def run_cmd_on_host(hostname, cmd, stdin, stdout, stderr):
- base_cmd = abstract_ssh.make_ssh_command()
- full_cmd = "%s %s \"%s\"" % (base_cmd, hostname,
- server_utils.sh_escape(cmd))
-
- return subprocess.Popen(full_cmd, stdin=stdin, stdout=stdout,
- stderr=stderr, shell=True)
-
-
def list_remote_pythons(host):
"""List out installed pythons on host."""
result = host.run('ls /usr/bin/python[0-9]*')
@@ -72,25 +63,24 @@
raise FollowFilesLaunchError('No supported Python on host.')
remote_monitordir = copy_monitordir(host)
- remote_script_path = os.path.join(
- remote_monitordir, 'followfiles.py')
+ remote_script_path = os.path.join(remote_monitordir, 'followfiles.py')
followfiles_cmd = '%s %s --lastlines_dirpath=%s %s' % (
supported_python, remote_script_path,
lastlines_dirpath, ' '.join(follow_paths))
- devnull_r = open(os.devnull, 'r')
- devnull_w = open(os.devnull, 'w')
- remote_followfiles_proc = run_cmd_on_host(
- host.hostname, followfiles_cmd, stdout=subprocess.PIPE,
- stdin=devnull_r, stderr=devnull_w)
+ remote_ff_proc = subprocess.Popen(host._make_ssh_cmd(followfiles_cmd),
+ stdin=open(os.devnull, 'r'),
+ stdout=subprocess.PIPE, shell=True)
+
+
# Give it enough time to crash if it's going to (it shouldn't).
time.sleep(5)
- doa = remote_followfiles_proc.poll()
+ doa = remote_ff_proc.poll()
if doa:
raise FollowFilesLaunchError('ssh command crashed.')
- return remote_followfiles_proc
+ return remote_ff_proc
def resolve_patterns_path(patterns_path):
diff --git a/server/hosts/remote.py b/server/hosts/remote.py
index e46bc1b..d1b4b46 100644
--- a/server/hosts/remote.py
+++ b/server/hosts/remote.py
@@ -27,7 +27,7 @@
LAST_BOOT_TAG = object()
DEFAULT_HALT_TIMEOUT = 2 * 60
- VAR_LOG_MESSAGES_COPY_PATH = "/var/log/messages.autotest_start"
+ VAR_LOG_MESSAGES_COPY_PATH = "/var/tmp/messages.autotest_start"
def _initialize(self, hostname, autodir=None, *args, **dargs):
super(RemoteHost, self)._initialize(*args, **dargs)
@@ -230,8 +230,8 @@
keyvals = utils.read_keyval(keyval_path)
all_labels = keyvals.get('labels', '')
if all_labels:
- all_labels = all_labels.split(',')
- return [urllib.unquote(label) for label in all_labels]
+ all_labels = all_labels.split(',')
+ return [urllib.unquote(label) for label in all_labels]
return []
diff --git a/server/hosts/serial.py b/server/hosts/serial.py
index d514dba..d363cb7 100644
--- a/server/hosts/serial.py
+++ b/server/hosts/serial.py
@@ -165,6 +165,7 @@
# Run on num_attempts=1 or last retry
try:
self.wait_for_restart(timeout,
+ old_boot_id=old_boot_id,
**wait_for_restart_kwargs)
except error.AutoservShutdownError:
logging.warning(warning_msg, num_attempts, num_attempts)
diff --git a/server/prebuild.py b/server/prebuild.py
index 4d78605..b13fd7b 100644
--- a/server/prebuild.py
+++ b/server/prebuild.py
@@ -61,4 +61,3 @@
# instantiate a client_test instance.
client_test = init_test(client_test_dir)
client_setup_job.setup_test(client_test)
-
diff --git a/server/server_job.py b/server/server_job.py
index efc88d6..8a16c6a 100644
--- a/server/server_job.py
+++ b/server/server_job.py
@@ -6,7 +6,7 @@
Copyright Martin J. Bligh, Andy Whitcroft 2007
"""
-import getpass, os, sys, re, stat, tempfile, time, select, subprocess
+import getpass, os, sys, re, stat, tempfile, time, select, subprocess, platform
import traceback, shutil, warnings, fcntl, pickle, logging, itertools, errno
from autotest_lib.client.bin import sysinfo
from autotest_lib.client.common_lib import base_job
@@ -204,6 +204,7 @@
job_data = {'label' : label, 'user' : user,
'hostname' : ','.join(machines),
+ 'drone' : platform.node(),
'status_version' : str(self._STATUS_VERSION),
'job_started' : str(int(time.time()))}
if group_name:
diff --git a/server/standalone_profiler.py b/server/standalone_profiler.py
index 62a11e1..6324551 100644
--- a/server/standalone_profiler.py
+++ b/server/standalone_profiler.py
@@ -9,6 +9,7 @@
__author__ = 'cranger@google.com (Colby Ranger)'
+import platform
import common
from autotest_lib.client.common_lib import barrier
@@ -16,7 +17,7 @@
_RUNTEST_PATTERN = ("job.run_test('profiler_sync', timeout_sync=%r,\n"
" timeout_start=%r, timeout_stop=%r,\n"
" hostid='%s', masterid='%s', all_ids=%r)")
-_PROF_MASTER = "PROF_MASTER"
+_PROF_MASTER = platform.node()
_PORT = 11920
diff --git a/server/tests/barriertest_2client/control.srv b/server/tests/barriertest_2client/control.srv
new file mode 100644
index 0000000..d6a70bb
--- /dev/null
+++ b/server/tests/barriertest_2client/control.srv
@@ -0,0 +1,78 @@
+AUTHOR = "gps@google.com (Gregory P. Smith)"
+TIME = "SHORT"
+NAME = "barrier_2client"
+TEST_CATEGORY = "Functional"
+TEST_CLASS = 'Network'
+TEST_TYPE = "Server"
+EXPERIMENTAL = True # This is functional a test of autotest itself.
+SYNC_COUNT = 2
+DOC = """
+A functional test of autotest's Barrier mechanisms for synchronizing
+events between two clients without the help of the server.
+"""
+
+from autotest_lib.server import utils
+
+def run(pair):
+ logging.info('Running on %s and %s', pair[0], pair[1])
+ host_objs = [hosts.create_host(machine) for machine in pair]
+ host_at_objs = [autotest.Autotest(host) for host in host_objs]
+
+ client_control_template = """
+import logging, platform, socket, traceback
+try:
+ client_hostnames = %r
+ master_hostname = client_hostnames[0]
+ client_hostname = client_hostnames[1]
+
+ logging.info('Testing hostname only barrier')
+ barrier = job.barrier(platform.node(), 'barriertest_2client', 120)
+ logging.info('rendezvous-ing')
+ barrier.rendezvous(master_hostname, client_hostname)
+ logging.info('done.')
+
+ logging.info('Testing local identifier barrier')
+ barrier = job.barrier(platform.node() + '#id0', 'barriertest_2client', 120)
+ logging.info('rendezvous-ing')
+ barrier.rendezvous(master_hostname + '#id0',
+ client_hostname + '#id0')
+ logging.info('done.')
+
+ logging.info('Testing IP@ barrier')
+ barrier = job.barrier(socket.gethostbyname(platform.node()),
+ 'barriertest_2client', 120)
+ logging.info('rendezvous-ing')
+ barrier.rendezvous(socket.gethostbyname(master_hostname),
+ socket.gethostbyname(client_hostname))
+ logging.info('done.')
+
+ logging.info('Testing IP@ barrier with ids')
+ barrier = job.barrier(socket.gethostbyname(platform.node()) + '#42',
+ 'barriertest_2client', 120)
+ logging.info('rendezvous-ing')
+ barrier.rendezvous(socket.gethostbyname(master_hostname) + '#42',
+ socket.gethostbyname(client_hostname) + '#42')
+ logging.info('done.')
+except:
+ traceback.print_exc()
+ raise
+"""
+ client_controls = [client_control_template % (pair,) for host in host_objs]
+
+ subcommand_list = []
+ for host, host_at, control in zip(host_objs, host_at_objs, client_controls):
+ subcommand_list.append(subcommand(host_at.run,
+ (control, host.hostname)))
+
+ parallel(subcommand_list)
+
+
+# grab the pairs (and failures)
+(pairs, failures) = utils.form_ntuples_from_machines(machines, 2)
+
+# log the failures
+for failure in failures:
+ job.record("FAIL", failure[0], "barrier_2client", failure[1])
+
+# now run through each pair and run
+job.parallel_simple(run, pairs, log=False)
diff --git a/server/tests/netperf2/netperf2.py b/server/tests/netperf2/netperf2.py
index 604e4c8..108dab8 100644
--- a/server/tests/netperf2/netperf2.py
+++ b/server/tests/netperf2/netperf2.py
@@ -33,7 +33,7 @@
"test_time=%d, stream_list=%s, tag='%s', ",
"iterations=%d)"])
- server_control_file = template % (server.ip, client.ip, 'server', test,
+ server_control_file = template % (server.ip, client.ip, 'server', test,
time, stream_list, test, cycles)
client_control_file = template % (server.ip, client.ip, 'client', test,
time, stream_list, test, cycles)