Whenever a new SSH connection is initiated to an SSH host (run, send_file,
get_file), try to start and use a shared master SSH connection. If master-SSH
support is enabled through the global_config enable_master_ssh variable, a
master SSH connection is initiated as a background job by start_master_ssh.
From: petkov@chromium.org
git-svn-id: http://test.kernel.org/svn/autotest/trunk@4087 592f7852-d20e-0410-864c-8624ca9c26a4
diff --git a/global_config.ini b/global_config.ini
index 5df8ed5..320ec74 100644
--- a/global_config.ini
+++ b/global_config.ini
@@ -81,6 +81,9 @@
# another one based on the python SSH library paramiko (paramiko).
# You can change the default 'raw_ssh' to 'paramiko' if you want to.
ssh_engine: raw_ssh
+# Set to True to take advantage of OpenSSH-based connection sharing. This would
+# have bigger performance impact when ssh_engine is 'raw_ssh'.
+enable_master_ssh: False
# Autotest server operators *really should* set this to True, specially if
# using ssh_engine 'paramiko'.
require_atfork_module: False
diff --git a/server/hosts/abstract_ssh.py b/server/hosts/abstract_ssh.py
index b14d972..d671ab5 100644
--- a/server/hosts/abstract_ssh.py
+++ b/server/hosts/abstract_ssh.py
@@ -1,16 +1,22 @@
import os, time, types, socket, shutil, glob, logging, traceback
-from autotest_lib.client.common_lib import error, logging_manager
+from autotest_lib.client.common_lib import autotemp, error, logging_manager
from autotest_lib.server import utils, autotest
from autotest_lib.server.hosts import remote
+from autotest_lib.client.common_lib.global_config import global_config
-def make_ssh_command(user="root", port=22, opts='', connect_timeout=30):
+enable_master_ssh = global_config.get_config_value(
+ 'AUTOSERV', 'enable_master_ssh', type=bool, default=False)
+
+
+def make_ssh_command(user="root", port=22, opts='', connect_timeout=30,
+ alive_interval=300):
base_command = ("/usr/bin/ssh -a -x %s -o BatchMode=yes "
- "-o ConnectTimeout=%d -o ServerAliveInterval=300 "
+ "-o ConnectTimeout=%d -o ServerAliveInterval=%d "
"-l %s -p %d")
assert isinstance(connect_timeout, (int, long))
assert connect_timeout > 0 # can't disable the timeout
- return base_command % (opts, connect_timeout, user, port)
+ return base_command % (opts, connect_timeout, alive_interval, user, port)
# import site specific Host class
@@ -36,6 +42,15 @@
self.port = port
self.password = password
+ """
+ Master SSH connection background job, socket temp directory and socket
+ control path option. If master-SSH is enabled, these fields will be
+ initialized by start_master_ssh when a new SSH connection is initiated.
+ """
+ self.master_ssh_job = None
+ self.master_ssh_tempdir = None
+ self.master_ssh_option = ''
+
# Check if rsync is available on the remote host. If it's not,
# don't try to use it for any future file transfers.
self.use_rsync = self._check_rsync()
@@ -71,7 +86,8 @@
appropriate rsync command for copying them. Remote paths must be
pre-encoded.
"""
- ssh_cmd = make_ssh_command(self.user, self.port)
+ ssh_cmd = make_ssh_command(self.user, self.port,
+ self.master_ssh_option)
if delete_dest:
delete_flag = "--delete"
else:
@@ -91,8 +107,9 @@
appropriate scp command for encoding it. Remote paths must be
pre-encoded.
"""
- command = "scp -rq -P %d %s '%s'"
- return command % (self.port, " ".join(sources), dest)
+ command = "scp -rq %s -P %d %s '%s'"
+ return command % (self.master_ssh_option,
+ self.port, " ".join(sources), dest)
def _make_rsync_compatible_globs(self, path, is_local):
@@ -218,6 +235,10 @@
Raises:
AutoservRunError: the scp command failed
"""
+
+ # Start a master SSH connection if necessary.
+ self.start_master_ssh()
+
if isinstance(source, basestring):
source = [source]
dest = os.path.abspath(dest)
@@ -290,6 +311,10 @@
Raises:
AutoservRunError: the scp command failed
"""
+
+ # Start a master SSH connection if necessary.
+ self.start_master_ssh()
+
if isinstance(source, basestring):
source = [source]
remote_dest = self._encode_remote_paths([dest])
@@ -453,3 +478,58 @@
# autotest dir may not exist, etc. ignore
logging.debug('autodir space check exception, this is probably '
'safe to ignore\n' + traceback.format_exc())
+
+
+ def close(self):
+ super(AbstractSSHHost, self).close()
+ self._cleanup_master_ssh()
+
+
+ def _cleanup_master_ssh(self):
+ """
+ Release all resources (process, temporary directory) used by an active
+ master SSH connection.
+ """
+ # If a master SSH connection is running, kill it.
+ if self.master_ssh_job is not None:
+ utils.nuke_subprocess(self.master_ssh_job.sp)
+ self.master_ssh_job = None
+
+ # Remove the temporary directory for the master SSH socket.
+ if self.master_ssh_tempdir is not None:
+ self.master_ssh_tempdir.clean()
+ self.master_ssh_tempdir = None
+ self.master_ssh_option = ''
+
+
+ def start_master_ssh(self):
+ """
+ Called whenever a slave SSH connection needs to be initiated (e.g., by
+ run, rsync, scp). If master SSH support is enabled and a master SSH
+ connection is not active already, start a new one in the background.
+ Also, cleanup any zombie master SSH connections (e.g., dead due to
+ reboot).
+ """
+ if not enable_master_ssh:
+ return
+
+ # If a previously started master SSH connection is not running
+ # anymore, it needs to be cleaned up and then restarted.
+ if self.master_ssh_job is not None:
+ if self.master_ssh_job.sp.poll() is not None:
+ logging.info("Master ssh connection to %s is down.",
+ self.hostname)
+ self._cleanup_master_ssh()
+
+ # Start a new master SSH connection.
+ if self.master_ssh_job is None:
+ # Create a shared socket in a temp location.
+ self.master_ssh_tempdir = autotemp.tempdir(unique_id='ssh-master')
+ self.master_ssh_option = ("-o ControlPath=%s/socket" %
+ self.master_ssh_tempdir.name)
+
+ # Start the master SSH connection in the background.
+ master_cmd = self.ssh_command(options="-N -o ControlMaster=yes",
+ alive_interval=5)
+ logging.info("Starting master ssh connection '%s'" % master_cmd)
+ self.master_ssh_job = utils.BgJob(master_cmd)
diff --git a/server/hosts/ssh_host.py b/server/hosts/ssh_host.py
index 5926c5b..32d42ba 100644
--- a/server/hosts/ssh_host.py
+++ b/server/hosts/ssh_host.py
@@ -49,10 +49,15 @@
self.setup_ssh()
- def ssh_command(self, connect_timeout=30, options=''):
- """Construct an ssh command with proper args for this host."""
+ def ssh_command(self, connect_timeout=30, options='', alive_interval=300):
+ """
+ Construct an ssh command with proper args for this host.
+ """
+ options = "%s %s" % (options, self.master_ssh_option)
base_cmd = abstract_ssh.make_ssh_command(self.user, self.port,
- options, connect_timeout)
+ options,
+ connect_timeout,
+ alive_interval)
return "%s %s" % (base_cmd, self.hostname)
@@ -106,6 +111,10 @@
"""
if verbose:
logging.debug("Running (ssh) '%s'" % command)
+
+ # Start a master SSH connection if necessary.
+ self.start_master_ssh()
+
env = " ".join("=".join(pair) for pair in self.env.iteritems())
try:
return self._run(command, timeout, ignore_status, stdout_tee,