server/cros/servo_test.py

# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

import httplib, logging, os, socket, subprocess, sys, time, xmlrpclib

from autotest_lib.client.common_lib import error, utils
from autotest_lib.server import autotest, test


class ServoTest(test.test):
    """AutoTest test class to serve as a parent class for FAFT tests.

    TODO(jrbarnette):  This class is a legacy, reflecting
    refactoring that has begun but not completed.  The long term
    plan is to move all function here into FAFT specific classes.
    http://crosbug.com/33305.
    """
    version = 2

    _REMOTE_PORT = 9990
    _REMOTE_COMMAND = '/usr/local/autotest/cros/faft_client.py'
    _REMOTE_COMMAND_SHORT = 'faft_client'
    _REMOTE_LOG_FILE = '/tmp/faft_client.log'
    _SSH_CONFIG = ('-o StrictHostKeyChecking=no '
                   '-o UserKnownHostsFile=/dev/null ')

    def initialize(self, host, _, use_pyauto=False, use_faft=False):
        """Create a Servo object and install the dependency.
        """
        # TODO(jrbarnette): Part of the incomplete refactoring:
        # assert here that there are no legacy callers passing
        # parameters for functionality that's been deprecated and
        # removed.
        assert use_faft and not use_pyauto

        self.servo = host.servo
        self.faft_client = None
        self._client = host
        self._ssh_tunnel = None
        self._remote_process = None
        self._local_port = None

        # Initializes dut, may raise AssertionError if pre-defined gpio
        # sequence to set GPIO's fail.  Autotest does not handle exception
        # throwing in initialize and will cause a test to hang.
        try:
            self.servo.initialize_dut()
        except (AssertionError, xmlrpclib.Fault) as e:
            raise error.TestFail(e)

        # Install faft_client dependency.
        self._autotest_client = autotest.Autotest(self._client)
        self._autotest_client.install()
        self._launch_client()

    def _ping_test(self, hostname, timeout=5):
        """Verify whether a host responds to a ping.

        Args:
          hostname: Hostname to ping.
          timeout: Time in seconds to wait for a response.
        """
        with open(os.devnull, 'w') as fnull:
            return subprocess.call(
                    ['ping', '-c', '1', '-W', str(timeout), hostname],
                    stdout=fnull, stderr=fnull) == 0

    def _sshd_test(self, hostname, timeout=5):
        """Verify whether sshd is running in host.

        Args:
          hostname: Hostname to verify.
          timeout: Time in seconds to wait for a response.
        """
        try:
            sock = socket.create_connection((hostname, 22), timeout=timeout)
            sock.close()
            return True
        except socket.timeout:
            return False
        except socket.error:
            time.sleep(timeout)
            return False

    def _launch_client(self):
        """Launch a remote XML RPC connection on client with retrials.
        """
        retry = 3
        while retry:
            try:
                self._launch_client_once()
                break
            except AssertionError:
                retry -= 1
                if retry:
                    logging.info('Retry again...')
                    time.sleep(5)
                else:
                    raise

    def _launch_client_once(self):
        """Launch a remote process on client and set up an xmlrpc connection.
        """
        if self._ssh_tunnel:
            self._ssh_tunnel.terminate()
            self._ssh_tunnel = None

        # Launch RPC server remotely.
        self._kill_remote_process()
        self._launch_ssh_tunnel()

        logging.info('Client command: %s', self._REMOTE_COMMAND)
        logging.info("Logging to %s", self._REMOTE_LOG_FILE)
        full_cmd = ['ssh -n %s root@%s \'%s &> %s\'' % (
                      self._SSH_CONFIG, self._client.ip,
                      self._REMOTE_COMMAND, self._REMOTE_LOG_FILE)]
        logging.info('Starting process %s', ' '.join(full_cmd))
        self._remote_process = subprocess.Popen(full_cmd, shell=True,
                                                stdout=subprocess.PIPE,
                                                stderr=subprocess.PIPE)

        # Connect to RPC object.
        logging.info('Connecting to client RPC server...')
        remote_url = 'http://localhost:%s' % self._local_port
        self.faft_client = xmlrpclib.ServerProxy(remote_url, allow_none=True)
        logging.info('Server proxy: %s', remote_url)

        # Poll for client RPC server to come online.
        timeout = 20
        succeed = False
        rpc_error = None
        while timeout > 0 and not succeed:
            time.sleep(1)
            if self._remote_process.poll() is not None:
                # The SSH process is gone. Log stderr.
                logging.error('Remote process died!')
                sout, serr = self._remote_process.communicate()
                logging.error('Stdout: %s', sout)
                logging.error('Stderr: %s', serr)
                break

            try:
                self.faft_client.system.is_available()
                succeed = True
            except (socket.error,
                    xmlrpclib.ProtocolError,
                    httplib.BadStatusLine) as e:
                logging.info('caught: %s %s, tries left: %s',
                             repr(e), str(e), timeout)
                # The client RPC server may not come online fast enough. Retry.
                timeout -= 1
                rpc_error = e
            except:
                logging.error('Unexpected error: %s', sys.exc_info()[0])
                raise

        if not succeed:
            if isinstance(rpc_error, xmlrpclib.ProtocolError):
                logging.info("A protocol error occurred")
                logging.info("URL: %s", rpc_error.url)
                logging.info("HTTP/HTTPS headers: %s", rpc_error.headers)
                logging.info("Error code: %d", rpc_error.errcode)
                logging.info("Error message: %s", rpc_error.errmsg)
            p = subprocess.Popen([
                'ssh -n -q %s root@%s \'cat %s\'' % (self._SSH_CONFIG,
                self._client.ip, self._REMOTE_LOG_FILE)], shell=True,
                stdout=subprocess.PIPE)
            logging.info('Log of running remote %s:',
                         self._REMOTE_COMMAND_SHORT)
            logging.info(p.communicate()[0])
        assert succeed, 'Timed out connecting to client RPC server.'

    def wait_for_client(self, install_deps=False, timeout=100):
        """Wait for the client to come back online.

        New remote processes will be launched if their used flags are enabled.

        @param install_deps: If True, install Autotest dependency when ready.
        @param timeout: Time in seconds to wait for the client SSH daemon to
                        come up.
        """
        # Ensure old ssh connections are terminated.
        self._terminate_all_ssh()
        # Wait for the client to come up.
        while timeout > 0 and not self._sshd_test(self._client.ip, timeout=2):
            timeout -= 2
        assert (timeout > 0), 'Timed out waiting for client to reboot.'
        logging.info('Server: Client machine is up.')
        # Relaunch remote clients.
        if install_deps:
            self._autotest_client.install()
        self._launch_client()
        logging.info('Server: Relaunched remote %s.', 'faft')

    def wait_for_client_offline(self, timeout=60, orig_boot_id=None):
        """Wait for the client to come offline.

        @param timeout: Time in seconds to wait the client to come offline.
        @param orig_boot_id: A string containing the original boot id.
        """
        # Wait for the client to come offline.
        while timeout > 0 and self._ping_test(self._client.ip, timeout=1):
            time.sleep(1)
            timeout -= 1

        # As get_boot_id() requires DUT online. So we move the comparison here.
        if timeout == 0 and orig_boot_id:
            if self._client.get_boot_id() != orig_boot_id:
                logging.warn('Reboot done very quickly.')
                return

        assert timeout, 'Timed out waiting for client offline.'
        logging.info('Server: Client machine is offline.')

    def kill_remote(self):
        """Call remote cleanup and kill ssh."""
        if self._remote_process and self._remote_process.poll() is None:
            try:
                self.faft_client.cleanup()
                logging.info('Cleanup succeeded.')
            except xmlrpclib.ProtocolError, e:
                logging.info('Cleanup returned protocol error: ' + str(e))
        self._terminate_all_ssh()

    def cleanup(self):
        """Delete the Servo object, call remote cleanup, and kill ssh."""
        self.kill_remote()

    def _launch_ssh_tunnel(self):
        """Establish an ssh tunnel for connecting to the remote RPC server.
        """
        if self._local_port is None:
            self._local_port = utils.get_unused_port()
        if not self._ssh_tunnel or self._ssh_tunnel.poll() is not None:
            self._ssh_tunnel = subprocess.Popen([
                'ssh -N -n -q %s -L %s:localhost:%s root@%s' %
                (self._SSH_CONFIG, self._local_port, self._REMOTE_PORT,
                self._client.ip)], shell=True)
            assert self._ssh_tunnel.poll() is None, \
                'The SSH tunnel on port %d is not up.' % self._local_port

    def _kill_remote_process(self):
        """Ensure the remote process and local ssh process are terminated.
        """
        kill_cmd = 'pkill -f %s' % self._REMOTE_COMMAND_SHORT
        subprocess.call(['ssh -n -q %s root@%s \'%s\'' %
                         (self._SSH_CONFIG, self._client.ip, kill_cmd)],
                        shell=True)
        if self._remote_process and self._remote_process.poll() is None:
            self._remote_process.terminate()

    def _terminate_all_ssh(self):
        """Terminate all ssh connections associated with remote processes."""
        if self._ssh_tunnel and self._ssh_tunnel.poll() is None:
            self._ssh_tunnel.terminate()
        self._kill_remote_process()
        self._ssh_tunnel = None