server/cros/servo_test.py

# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

import httplib, logging, os, socket, subprocess, sys, time, xmlrpclib

from autotest_lib.client.common_lib import error
from autotest_lib.server import autotest, test
from autotest_lib.server.cros import servo

class ServoTest(test.test):
    """AutoTest test class that creates and destroys a servo object.

    Servo-based server side AutoTests can inherit from this object.
    There are 2 remote clients supported:
        If use_pyauto flag is True, a remote PyAuto client will be launched;
        If use_faft flag is Ture, a remote FAFT client will be launched.
    """
    version = 2

    # Exposes RPC access to a remote PyAuto client.
    pyauto = None
    # Exposes RPC access to a remote FAFT client.
    faft_client = None

    # Autotest references to the client.
    _autotest_client = None
    # Remote client info list.
    _remote_infos = {
        'pyauto': {
            # Used or not.
            'used': False,
            # Reference name of RPC object in this class.
            'ref_name': 'pyauto',
            # Port number of the remote RPC.
            'port': 9988,
            # The remote command to be run.
            'remote_command': 'python /usr/local/autotest/cros/remote_pyauto.py'
                              ' --no-http-server',
            # The short form of remote command, used by pkill.
            'remote_command_short': 'remote_pyauto',
            # The remote process info.
            'remote_process': None,
            # The ssh tunnel process info.
            'ssh_tunnel': None,
            # Polling RPC function name for testing the server availability.
            'polling_rpc': 'IsLinux',
            # Additional SSH options.
            'ssh_config': '-o StrictHostKeyChecking=no ',
        },
        'faft': {
            'used': False,
            'ref_name': 'faft_client',
            'port': 9990,
            'remote_command': '/usr/local/autotest/cros/faft_client.py',
            'remote_command_short': 'faft_client',
            'remote_log_file': '/tmp/faft_client.log',
            'remote_process': None,
            'ssh_tunnel': None,
            'polling_rpc': 'system.is_available',
            'ssh_config': '-o StrictHostKeyChecking=no '
                          '-o UserKnownHostsFile=/dev/null ',
        },
    }

    def _init_servo(self, host):
        """Initialize `self.servo`.

        If the host has an attached servo object, use that.
        Otherwise assume that there's a locally attached servo
        device, and use it.

        """
        if host.servo:
            self.servo = host.servo
        else:
            self.servo = servo.Servo()


    def initialize(self, host, _, use_pyauto=False, use_faft=False):
        """Create a Servo object and install the dependency.

        If use_pyauto/use_faft is True the PyAuto/FAFTClient dependency is
        installed on the client and a remote PyAuto/FAFTClient server is
        launched and connected.
        """
        # Initialize servotest args.
        self._client = host
        self._remote_infos['pyauto']['used'] = use_pyauto
        self._remote_infos['faft']['used'] = use_faft

        self._init_servo(host)

        # Initializes dut, may raise AssertionError if pre-defined gpio
        # sequence to set GPIO's fail.  Autotest does not handle exception
        # throwing in initialize and will cause a test to hang.
        try:
            self.servo.initialize_dut()
        except (AssertionError, xmlrpclib.Fault) as e:
            raise error.TestFail(e)

        # Install PyAuto/FAFTClient dependency.
        for info in self._remote_infos.itervalues():
            if info['used']:
                if not self._autotest_client:
                    self._autotest_client = autotest.Autotest(self._client)
                self._autotest_client.install()
                self.launch_client(info)


    def _ping_test(self, hostname, timeout=5):
        """Verify whether a host responds to a ping.

        Args:
          hostname: Hostname to ping.
          timeout: Time in seconds to wait for a response.
        """
        with open(os.devnull, 'w') as fnull:
            return subprocess.call(
                    ['ping', '-c', '1', '-W', str(timeout), hostname],
                    stdout=fnull, stderr=fnull) == 0


    def _sshd_test(self, hostname, timeout=5):
        """Verify whether sshd is running in host.

        Args:
          hostname: Hostname to verify.
          timeout: Time in seconds to wait for a response.
        """
        try:
            sock = socket.create_connection((hostname, 22), timeout=timeout)
            sock.close()
            return True
        except socket.timeout:
            return False
        except socket.error:
            time.sleep(timeout)
            return False


    def launch_client(self, info):
        """Launch a remote XML RPC connection on client with retrials.

        Args:
          info: A dict of remote info, see the definition of self._remote_infos.
        """
        retry = 3
        while retry:
            try:
                self._launch_client_once(info)
                break
            except AssertionError:
                retry -= 1
                if retry:
                    logging.info('Retry again...')
                    time.sleep(5)
                else:
                    raise


    def _launch_client_once(self, info):
        """Launch a remote process on client and set up an xmlrpc connection.

        Args:
          info: A dict of remote info, see the definition of self._remote_infos.
        """
        assert info['used'], \
            'Remote %s dependency not installed.' % info['ref_name']

        if info['ssh_tunnel']:
            info['ssh_tunnel'].terminate()
            info['ssh_tunnel'] = None

        # Launch RPC server remotely.
        self._kill_remote_process(info)
        self._launch_ssh_tunnel(info)

        logging.info('Client command: %s', info['remote_command'])
        log_file = info.get('remote_log_file', '/dev/null')
        logging.info("Logging to %s", log_file)
        full_cmd = ['ssh -n -q %s root@%s \'%s &> %s\'' % (
                info['ssh_config'],
                self._client.ip, info['remote_command'], log_file)]
        logging.info('Starting process %s', ' '.join(full_cmd))
        info['remote_process'] = subprocess.Popen(full_cmd, shell=True)

        # Connect to RPC object.
        logging.info('Connecting to client RPC server...')
        remote_url = 'http://localhost:%s' % info['port']
        setattr(self, info['ref_name'],
            xmlrpclib.ServerProxy(remote_url, allow_none=True))
        logging.info('Server proxy: %s', remote_url)

        # Poll for client RPC server to come online.
        timeout = 20
        succeed = False
        rpc_error = None
        while timeout > 0 and not succeed:
            time.sleep(1)
            try:
                remote_object = getattr(self, info['ref_name'])
                polling_rpc = getattr(remote_object, info['polling_rpc'])
                polling_rpc()
                succeed = True
            except (socket.error,
                    xmlrpclib.ProtocolError,
                    httplib.BadStatusLine) as e:
                logging.info('caught exception %s', e)
                # The client RPC server may not come online fast enough. Retry.
                timeout -= 1
                rpc_error = e
            except:
                logging.error('Unexpected error: %s', sys.exc_info()[0])
                raise

        if not succeed:
            if isinstance(rpc_error, xmlrpclib.ProtocolError):
                logging.info("A protocol error occurred")
                logging.info("URL: %s", rpc_error.url)
                logging.info("HTTP/HTTPS headers: %s", rpc_error.headers)
                logging.info("Error code: %d", rpc_error.errcode)
                logging.info("Error message: %s", rpc_error.errmsg)
            if 'remote_log_file' in info:
                p = subprocess.Popen([
                    'ssh -n -q %s root@%s \'cat %s\'' % (info['ssh_config'],
                    self._client.ip, info['remote_log_file'])], shell=True,
                    stdout=subprocess.PIPE)
                logging.info('Log of running remote %s:', info['ref_name'])
                logging.info(p.communicate()[0])
        assert succeed, 'Timed out connecting to client RPC server.'


    def wait_for_client(self, install_deps=False, timeout=100):
        """Wait for the client to come back online.

        New remote processes will be launched if their used flags are enabled.

        Args:
            install_deps: If True, install the Autotest dependency when ready.
            timeout: Time in seconds to wait for the client SSH daemon to
              come up.
        """
        # Ensure old ssh connections are terminated.
        self._terminate_all_ssh()
        # Wait for the client to come up.
        while timeout > 0 and not self._sshd_test(self._client.ip, timeout=2):
            timeout -= 2
        assert (timeout > 0), 'Timed out waiting for client to reboot.'
        logging.info('Server: Client machine is up.')
        # Relaunch remote clients.
        for name, info in self._remote_infos.iteritems():
            if info['used']:
                if install_deps:
                    if not self._autotest_client:
                        self._autotest_client = autotest.Autotest(self._client)
                    self._autotest_client.install()
                self.launch_client(info)
                logging.info('Server: Relaunched remote %s.', name)


    def wait_for_client_offline(self, timeout=60):
        """Wait for the client to come offline.

        Args:
          timeout: Time in seconds to wait the client to come offline.
        """
        # Wait for the client to come offline.
        while timeout > 0 and self._ping_test(self._client.ip, timeout=1):
            time.sleep(1)
            timeout -= 1
        assert timeout, 'Timed out waiting for client offline.'
        logging.info('Server: Client machine is offline.')


    def kill_remote(self):
        """Call remote cleanup and kill ssh."""
        for info in self._remote_infos.itervalues():
            if info['remote_process'] and info['remote_process'].poll() is None:
                remote_object = getattr(self, info['ref_name'])
                try:
                    remote_object.cleanup()
                    logging.info('Cleanup succeeded.')
                except xmlrpclib.ProtocolError, e:
                    logging.info('Cleanup returned protocol error: ' + str(e))
        self._terminate_all_ssh()


    def cleanup(self):
        """Delete the Servo object, call remote cleanup, and kill ssh."""
        self.kill_remote()


    def _launch_ssh_tunnel(self, info):
        """Establish an ssh tunnel for connecting to the remote RPC server.

        Args:
          info: A dict of remote info, see the definition of self._remote_infos.
        """
        if not info['ssh_tunnel'] or info['ssh_tunnel'].poll() is not None:
            info['ssh_tunnel'] = subprocess.Popen([
                'ssh -N -n -q %s -L %s:localhost:%s root@%s' %
                (info['ssh_config'], info['port'], info['port'],
                self._client.ip)], shell=True)
            assert info['ssh_tunnel'].poll() is None, \
                'The SSH tunnel on port %d is not up.' % info['port']


    def _kill_remote_process(self, info):
        """Ensure the remote process and local ssh process are terminated.

        Args:
          info: A dict of remote info, see the definition of self._remote_infos.
        """
        kill_cmd = 'pkill -f %s' % info['remote_command_short']
        subprocess.call(['ssh -n -q %s root@%s \'%s\'' %
                         (info['ssh_config'], self._client.ip, kill_cmd)],
                        shell=True)
        if info['remote_process'] and info['remote_process'].poll() is None:
            info['remote_process'].terminate()


    def _terminate_all_ssh(self):
        """Terminate all ssh connections associated with remote processes."""
        for info in self._remote_infos.itervalues():
            if info['ssh_tunnel'] and info['ssh_tunnel'].poll() is None:
                info['ssh_tunnel'].terminate()
            self._kill_remote_process(info)
            info['ssh_tunnel'] = None