public/actions/common_operations.py

#!/usr/bin/env python
#
# Copyright 2018 - The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Common operations between managing GCE and Cuttlefish devices.

This module provides the common operations between managing GCE (device_driver)
and Cuttlefish (create_cuttlefish_action) devices. Should not be called
directly.
"""

from __future__ import print_function
import getpass
import logging
import os
import subprocess

from acloud.public import avd
from acloud.public import errors
from acloud.public import report
from acloud.internal import constants
from acloud.internal.lib import utils

logger = logging.getLogger(__name__)

#For the cuttlefish remote instances: adb port is 6520 and vnc is 6444.
_CF_TARGET_ADB_PORT = 6520
_CF_TARGET_VNC_PORT = 6444

def CreateSshKeyPairIfNecessary(cfg):
    """Create ssh key pair if necessary.

    Args:
        cfg: An Acloudconfig instance.

    Raises:
        error.DriverError: If it falls into an unexpected condition.
    """
    if not cfg.ssh_public_key_path:
        logger.warning(
            "ssh_public_key_path is not specified in acloud config. "
            "Project-wide public key will "
            "be used when creating AVD instances. "
            "Please ensure you have the correct private half of "
            "a project-wide public key if you want to ssh into the "
            "instances after creation.")
    elif cfg.ssh_public_key_path and not cfg.ssh_private_key_path:
        logger.warning(
            "Only ssh_public_key_path is specified in acloud config, "
            "but ssh_private_key_path is missing. "
            "Please ensure you have the correct private half "
            "if you want to ssh into the instances after creation.")
    elif cfg.ssh_public_key_path and cfg.ssh_private_key_path:
        utils.CreateSshKeyPairIfNotExist(cfg.ssh_private_key_path,
                                         cfg.ssh_public_key_path)
    else:
        # Should never reach here.
        raise errors.DriverError(
            "Unexpected error in CreateSshKeyPairIfNecessary")


class DevicePool(object):
    """A class that manages a pool of virtual devices.

    Attributes:
        devices: A list of devices in the pool.
    """

    def __init__(self, device_factory, devices=None):
        """Constructs a new DevicePool.

        Args:
            device_factory: A device factory capable of producing a goldfish or
                cuttlefish device. The device factory must expose an attribute with
                the credentials that can be used to retrieve information from the
                constructed device.
            devices: List of devices managed by this pool.
        """
        self._devices = devices or []
        self._device_factory = device_factory
        self._compute_client = device_factory.GetComputeClient()

    def _CollectAdbLogcats(self, output_dir):
        """Collect Adb logcats.

        Args:
            output_dir: String, the output file directory to store adb logcats.

        Returns:
            The file information dictionary with file path and file name.
        """
        file_dict = {}
        for device in self._devices:
            if not device.adb_port:
                # If device adb tunnel is not established, do not do adb logcat
                continue
            file_name = "%s_adb_logcat.log" % device.instance_name
            full_file_path = os.path.join(output_dir, file_name)
            logger.info("Get adb %s:%s logcat for instance %s",
                        constants.LOCALHOST, device.adb_port,
                        device.instance_name)
            try:
                subprocess.check_call(
                    ["adb -s %s:%s logcat -b all -d > %s" % (
                        constants.LOCALHOST, device.adb_port, full_file_path)],
                    shell=True)
                file_dict[full_file_path] = file_name
            except subprocess.CalledProcessError:
                logging.error("Failed to get adb logcat for %s for instance %s",
                              device.serial_number, device.instance_name)
        return file_dict

    def CreateDevices(self, num):
        """Creates |num| devices for given build_target and build_id.

        Args:
            num: Number of devices to create.
        """
        # Create host instances for cuttlefish/goldfish device.
        # Currently one instance supports only 1 device.
        for _ in range(num):
            instance = self._device_factory.CreateInstance()
            ip = self._compute_client.GetInstanceIP(instance)
            self.devices.append(
                avd.AndroidVirtualDevice(ip=ip, instance_name=instance))

    @utils.TimeExecute(function_description="Waiting for AVD(s) to boot up")
    def WaitForBoot(self):
        """Waits for all devices to boot up.

        Returns:
            A dictionary that contains all the failures.
            The key is the name of the instance that fails to boot,
            and the value is an errors.DeviceBootError object.
        """
        failures = {}
        for device in self._devices:
            try:
                self._compute_client.WaitForBoot(device.instance_name)
            except errors.DeviceBootError as e:
                failures[device.instance_name] = e
        return failures

    def PullLogs(self, source_files, output_dir, user=None, ssh_rsa_path=None):
        """Tar logs from GCE instance into output_dir.

        Args:
            source_files: List of file names to be pulled.
            output_dir: String. The output file dirtory
            user: String, the ssh username to access GCE
            ssh_rsa_path: String, the ssh rsa key path to access GCE

        Returns:
            The file dictionary with file_path and file_name
        """

        file_dict = {}
        for device in self._devices:
            if isinstance(source_files, basestring):
                source_files = [source_files]
            for source_file in source_files:
                file_name = "%s_%s" % (device.instance_name,
                                       os.path.basename(source_file))
                dst_file = os.path.join(output_dir, file_name)
                logger.info("Pull %s for instance %s with user %s to %s",
                            source_file, device.instance_name, user, dst_file)
                try:
                    utils.ScpPullFile(source_file, dst_file, device.ip,
                                      user_name=user, rsa_key_file=ssh_rsa_path)
                    file_dict[dst_file] = file_name
                except errors.DeviceConnectionError as e:
                    logger.warning("Failed to pull %s from instance %s: %s",
                                   source_file, device.instance_name, e)
        return file_dict

    def CollectSerialPortLogs(self, output_file,
                              port=constants.DEFAULT_SERIAL_PORT):
        """Tar the instance serial logs into specified output_file.

        Args:
            output_file: String, the output tar file path
            port: The serial port number to be collected
        """
        # For emulator, the serial log is the virtual host serial log.
        # For GCE AVD device, the serial log is the AVD device serial log.
        with utils.TempDir() as tempdir:
            src_dict = {}
            for device in self._devices:
                logger.info("Store instance %s serial port %s output to %s",
                            device.instance_name, port, output_file)
                serial_log = self._compute_client.GetSerialPortOutput(
                    instance=device.instance_name, port=port)
                file_name = "%s_serial_%s.log" % (device.instance_name, port)
                file_path = os.path.join(tempdir, file_name)
                src_dict[file_path] = file_name
                with open(file_path, "w") as f:
                    f.write(serial_log.encode("utf-8"))
            utils.MakeTarFile(src_dict, output_file)

    def CollectLogcats(self, output_file, ssh_user, ssh_rsa_path):
        """Tar the instances' logcat and other logs into specified output_file.

        Args:
            output_file: String, the output tar file path
            ssh_user: The ssh user name
            ssh_rsa_path: The ssh rsa key path
        """
        with utils.TempDir() as tempdir:
            file_dict = {}
            if getattr(self._device_factory, "LOG_FILES", None):
                file_dict = self.PullLogs(
                    self._device_factory.LOG_FILES, tempdir, user=ssh_user,
                    ssh_rsa_path=ssh_rsa_path)
            # If the device is auto-connected, get adb logcat
            for file_path, file_name in self._CollectAdbLogcats(
                    tempdir).items():
                file_dict[file_path] = file_name
            utils.MakeTarFile(file_dict, output_file)

    @property
    def devices(self):
        """Returns a list of devices in the pool.

        Returns:
            A list of devices in the pool.
        """
        return self._devices

# TODO: Delete unused-argument when b/119614469 is resolved.
# pylint: disable=unused-argument
# pylint: disable=too-many-locals
def CreateDevices(command, cfg, device_factory, num, report_internal_ip=False,
                  autoconnect=False, serial_log_file=None, logcat_file=None):
    """Create a set of devices using the given factory.

    Main jobs in create devices.
        1. Create GCE instance: Launch instance in GCP(Google Cloud Platform).
        2. Starting up AVD: Wait device boot up.

    Args:
        command: The name of the command, used for reporting.
        cfg: An AcloudConfig instance.
        device_factory: A factory capable of producing a single device.
        num: The number of devices to create.
        report_internal_ip: Boolean to report the internal ip instead of
                            external ip.
        serial_log_file: String, the file path to tar the serial logs.
        logcat_file: String, the file path to tar the logcats.
        autoconnect: Boolean, whether to auto connect to device.

    Raises:
        errors: Create instance fail.

    Returns:
        A Report instance.
    """
    reporter = report.Report(command=command)
    try:
        CreateSshKeyPairIfNecessary(cfg)
        device_pool = DevicePool(device_factory)
        device_pool.CreateDevices(num)
        failures = device_pool.WaitForBoot()
        if failures:
            reporter.SetStatus(report.Status.BOOT_FAIL)
        else:
            reporter.SetStatus(report.Status.SUCCESS)

        # Collect logs
        if serial_log_file:
            device_pool.CollectSerialPortLogs(
                serial_log_file, port=constants.DEFAULT_SERIAL_PORT)
        # TODO(b/119614469): Refactor CollectLogcats into a utils lib and
        #                    turn it on inside the reporting loop.
        # if logcat_file:
        #     device_pool.CollectLogcats(logcat_file, ssh_user, ssh_rsa_path)

        # Write result to report.
        for device in device_pool.devices:
            ip = (device.ip.internal if report_internal_ip
                  else device.ip.external)
            device_dict = {
                "ip": ip,
                "instance_name": device.instance_name
            }
            if autoconnect:
                forwarded_ports = utils.AutoConnect(ip,
                                                    cfg.ssh_private_key_path,
                                                    _CF_TARGET_VNC_PORT,
                                                    _CF_TARGET_ADB_PORT,
                                                    getpass.getuser())
                device_dict[constants.VNC_PORT] = forwarded_ports.vnc_port
                device_dict[constants.ADB_PORT] = forwarded_ports.adb_port
            if device.instance_name in failures:
                reporter.AddData(key="devices_failing_boot", value=device_dict)
                reporter.AddError(str(failures[device.instance_name]))
            else:
                reporter.AddData(key="devices", value=device_dict)
    except errors.DriverError as e:
        reporter.AddError(str(e))
        reporter.SetStatus(report.Status.FAIL)
    return reporter