public/acloud_main.py

#!/usr/bin/env python
#
# Copyright 2016 - The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
r"""Cloud Android Driver.

This CLI manages google compute engine project for android devices.

- Prerequisites:
  See: go/acloud-manual

- Configuration:
  The script takes a required configuration file, which should look like
     <Start of the file>
     # If using service account
     service_account_name: "your_account@developer.gserviceaccount.com"
     service_account_private_key_path: "/path/to/your-project.p12"
     # Or
     service_account_json_private_key_path: "/path/to/your-project.json"

     # If using OAuth2 authentication flow
     client_id: <client id created in the project>
     client_secret: <client secret for the client id>

     # Optional
     ssh_private_key_path: "~/.ssh/acloud_rsa"
     ssh_public_key_path: "~/.ssh/acloud_rsa.pub"
     orientation: "portrait"
     resolution: "800x1280x32x213"
     network: "default"
     machine_type: "n1-standard-1"
     extra_data_disk_size_gb: 10  # 4G or 10G

     # Required
     project: "your-project"
     zone: "us-central1-f"
     storage_bucket_name: "your_google_storage_bucket_name"
     <End of the file>

  Save it at /path/to/acloud.config

- Example calls:
  - Create two instances:
  $ acloud.par create_cf
    --build_target aosp_cf_x86_phone-userdebug \
    --build_id 3744001 --num 2 --config_file /path/to/acloud.config \
    --report_file /tmp/acloud_report.json --log_file /tmp/acloud.log

  - Delete two instances:
  $ acloud.par delete --instance_names
    ins-b638cdba-3744001-gce-x86-phone-userdebug-fastbuild3c-linux
    --config_file /path/to/acloud.config
    --report_file /tmp/acloud_report.json --log_file /tmp/acloud.log
"""
import argparse
import getpass
import logging
import sys

# Needed to silence oauth2client.
logging.basicConfig(level=logging.CRITICAL)

# pylint: disable=wrong-import-position
from acloud.internal import constants
from acloud.public import acloud_common
from acloud.public import config
from acloud.public import device_driver
from acloud.public import errors
from acloud.public.actions import create_cuttlefish_action
from acloud.public.actions import create_goldfish_action
from acloud.setup import setup
from acloud.setup import setup_args

LOGGING_FMT = "%(asctime)s |%(levelname)s| %(module)s:%(lineno)s| %(message)s"
LOGGER_NAME = "acloud_main"

# Commands
CMD_CREATE = "create"
CMD_CREATE_CUTTLEFISH = "create_cf"
CMD_CREATE_GOLDFISH = "create_gf"
CMD_DELETE = "delete"
CMD_CLEANUP = "cleanup"
CMD_SSHKEY = "project_sshkey"


# pylint: disable=too-many-statements
def _ParseArgs(args):
    """Parse args.

    Args:
        args: Argument list passed from main.

    Returns:
        Parsed args.
    """
    usage = ",".join([
        CMD_CREATE, CMD_CREATE_CUTTLEFISH, CMD_DELETE, CMD_CLEANUP, CMD_SSHKEY,
        setup_args.CMD_SETUP,
    ])
    parser = argparse.ArgumentParser(
        description=__doc__,
        formatter_class=argparse.RawDescriptionHelpFormatter,
        usage="acloud {" + usage + "} ...")
    subparsers = parser.add_subparsers()
    subparser_list = []

    # Command "create"
    create_parser = subparsers.add_parser(CMD_CREATE)
    create_parser.required = False
    create_parser.set_defaults(which=CMD_CREATE)
    create_parser.add_argument(
        "--build_target",
        type=str,
        dest="build_target",
        help="Android build target, e.g. aosp_cf_x86_phone-userdebug, "
        "or short names: phone, tablet, or tablet_mobile.")
    create_parser.add_argument(
        "--branch",
        type=str,
        dest="branch",
        help="Android branch, e.g. mnc-dev or git_mnc-dev")
    # TODO: Support HEAD (the latest build)
    create_parser.add_argument(
        "--build_id",
        type=str,
        dest="build_id",
        help="Android build id, e.g. 2145099, P2804227")
    create_parser.add_argument(
        "--spec",
        type=str,
        dest="spec",
        required=False,
        help="The name of a pre-configured device spec that we are "
        "going to use. Choose from: %s" % ", ".join(constants.SPEC_NAMES))
    create_parser.add_argument(
        "--num",
        type=int,
        dest="num",
        required=False,
        default=1,
        help="Number of instances to create.")
    create_parser.add_argument(
        "--gce_image",
        type=str,
        dest="gce_image",
        required=False,
        help="Name of an existing compute engine image to reuse.")
    create_parser.add_argument(
        "--local_disk_image",
        type=str,
        dest="local_disk_image",
        required=False,
        help="Path to a local disk image to use, "
        "e.g /tmp/avd-system.tar.gz")
    create_parser.add_argument(
        "--no_cleanup",
        dest="no_cleanup",
        default=False,
        action="store_true",
        help="Do not clean up temporary disk image and compute engine image. "
        "For debugging purposes.")
    create_parser.add_argument(
        "--serial_log_file",
        type=str,
        dest="serial_log_file",
        required=False,
        help="Path to a *tar.gz file where serial logs will be saved "
        "when a device fails on boot.")
    create_parser.add_argument(
        "--logcat_file",
        type=str,
        dest="logcat_file",
        required=False,
        help="Path to a *tar.gz file where logcat logs will be saved "
        "when a device fails on boot.")
    create_parser.add_argument(
        "--autoconnect",
        action="store_true",
        dest="autoconnect",
        required=False,
        help=
        "For each instance created, we will automatically creates both 2 ssh"
        " tunnels forwarding both adb & vnc. Then add the device to adb.")

    subparser_list.append(create_parser)

    # Command "create_cf", create cuttlefish instances
    create_cf_parser = subparsers.add_parser(CMD_CREATE_CUTTLEFISH)
    create_cf_parser.required = False
    create_cf_parser.set_defaults(which=CMD_CREATE_CUTTLEFISH)
    create_cf_parser.add_argument(
        "--build_target",
        type=str,
        dest="build_target",
        help="Android build target, should be a cuttlefish target name.")
    create_cf_parser.add_argument(
        "--branch",
        type=str,
        dest="branch",
        help="Android branch, e.g. git_master")
    create_cf_parser.add_argument(
        "--build_id",
        type=str,
        dest="build_id",
        help="Android build id, e.g. 2145099, P2804227")
    create_cf_parser.add_argument(
        "--kernel_build_id",
        type=str,
        dest="kernel_build_id",
        required=False,
        help="Android kernel build id, e.g. 4586590. This is to test a new"
        " kernel build with a particular Android build (--build_id). If not"
        " specified, the kernel that's bundled with the Android build would"
        " be used.")
    create_cf_parser.add_argument(
        "--num",
        type=int,
        dest="num",
        required=False,
        default=1,
        help="Number of instances to create.")
    create_cf_parser.add_argument(
        "--serial_log_file",
        type=str,
        dest="serial_log_file",
        required=False,
        help="Path to a *tar.gz file where serial logs will be saved "
        "when a device fails on boot.")
    create_cf_parser.add_argument(
        "--logcat_file",
        type=str,
        dest="logcat_file",
        required=False,
        help="Path to a *tar.gz file where logcat logs will be saved "
        "when a device fails on boot.")
    create_cf_parser.add_argument(
        "--autoconnect",
        action="store_true",
        dest="autoconnect",
        required=False,
        help=
        "For each instance created, we will automatically creates both 2 ssh"
        " tunnels forwarding both adb & vnc. Then add the device to adb.")

    subparser_list.append(create_cf_parser)

    # Command "create_gf", create goldfish instances
    # In order to create a goldfish device we need the following parameters:
    # 1. The emulator build we wish to use, this is the binary that emulates
    #    an android device. See go/emu-dev for more
    # 2. A system-image. This is the android release we wish to run on the
    #    emulated hardware.
    create_gf_parser = subparsers.add_parser(CMD_CREATE_GOLDFISH)
    create_gf_parser.required = False
    create_gf_parser.set_defaults(which=CMD_CREATE_GOLDFISH)
    create_gf_parser.add_argument(
        "--build_target",
        type=str,
        dest="build_target",
        help="Android build target, should be a goldfish target name.")
    create_gf_parser.add_argument(
        "--branch",
        type=str,
        dest="branch",
        help="Android branch, e.g. git_master")
    create_gf_parser.add_argument(
        "--build_id",
        type=str,
        dest="build_id",
        help="Android build id, e.g. 4669424, P2804227")
    create_gf_parser.add_argument(
        "--emulator_build_id",
        type=str,
        dest="emulator_build_id",
        required=False,
        help="Emulator build used to run the images. e.g. 4669466.")
    create_gf_parser.add_argument(
        "--gpu",
        type=str,
        dest="gpu",
        required=False,
        default=None,
        help="GPU accelerator to use if any."
        " e.g. nvidia-tesla-k80, omit to use swiftshader")
    create_gf_parser.add_argument(
        "--num",
        type=int,
        dest="num",
        required=False,
        default=1,
        help="Number of instances to create.")
    create_gf_parser.add_argument(
        "--serial_log_file",
        type=str,
        dest="serial_log_file",
        required=False,
        help="Path to a *tar.gz file where serial logs will be saved "
        "when a device fails on boot.")
    create_gf_parser.add_argument(
        "--logcat_file",
        type=str,
        dest="logcat_file",
        required=False,
        help="Path to a *tar.gz file where logcat logs will be saved "
        "when a device fails on boot.")
    create_gf_parser.add_argument(
        "--autoconnect",
        action="store_true",
        dest="autoconnect",
        required=False,
        help=
        "For each instance created, we will automatically creates both 2 ssh"
        " tunnels forwarding both adb & vnc. Then add the device to adb.")
    create_gf_parser.add_argument(
        "--base_image",
        type=str,
        dest="base_image",
        required=False,
        help="Name of the goldfish base image to be used to create the instance. "
        "This will override stable_goldfish_host_image_name from config. "
        "e.g. emu-dev-cts-061118")

    subparser_list.append(create_gf_parser)

    # Command "Delete"
    delete_parser = subparsers.add_parser(CMD_DELETE)
    delete_parser.required = False
    delete_parser.set_defaults(which=CMD_DELETE)
    delete_parser.add_argument(
        "--instance_names",
        dest="instance_names",
        nargs="+",
        required=True,
        help="The names of the instances that need to delete, "
        "separated by spaces, e.g. --instance_names instance-1 instance-2")
    subparser_list.append(delete_parser)

    # Command "cleanup"
    cleanup_parser = subparsers.add_parser(CMD_CLEANUP)
    cleanup_parser.required = False
    cleanup_parser.set_defaults(which=CMD_CLEANUP)
    cleanup_parser.add_argument(
        "--expiration_mins",
        type=int,
        dest="expiration_mins",
        required=True,
        help="Garbage collect all gce instances, gce images, cached disk "
        "images that are older than |expiration_mins|.")
    subparser_list.append(cleanup_parser)

    # Command "project_sshkey"
    sshkey_parser = subparsers.add_parser(CMD_SSHKEY)
    sshkey_parser.required = False
    sshkey_parser.set_defaults(which=CMD_SSHKEY)
    sshkey_parser.add_argument(
        "--user",
        type=str,
        dest="user",
        default=getpass.getuser(),
        help="The user name which the sshkey belongs to, default to: %s." %
        getpass.getuser())
    sshkey_parser.add_argument(
        "--ssh_rsa_path",
        type=str,
        dest="ssh_rsa_path",
        required=True,
        help="Absolute path to the file that contains the public rsa key "
        "that will be added as project-wide ssh key.")
    subparser_list.append(sshkey_parser)

    # Command "setup"
    subparser_list.append(setup_args.GetSetupArgParser(subparsers))

    # Add common arguments.
    for subparser in subparser_list:
        acloud_common.AddCommonArguments(subparser)

    return parser.parse_args(args)


def _TranslateAlias(parsed_args):
    """Translate alias to Launch Control compatible values.

    This method translates alias to Launch Control compatible values.
     - branch: "git_" prefix will be added if branch name doesn't have it.
     - build_target: For example, "phone" will be translated to full target
                          name "git_x86_phone-userdebug",

    Args:
        parsed_args: Parsed args.

    Returns:
        Parsed args with its values being translated.
    """
    if parsed_args.which == CMD_CREATE:
        if (parsed_args.branch and
                not parsed_args.branch.startswith(constants.BRANCH_PREFIX)):
            parsed_args.branch = constants.BRANCH_PREFIX + parsed_args.branch
        parsed_args.build_target = constants.BUILD_TARGET_MAPPING.get(
            parsed_args.build_target, parsed_args.build_target)
    return parsed_args


def _VerifyArgs(parsed_args):
    """Verify args.

    Args:
        parsed_args: Parsed args.

    Raises:
        errors.CommandArgError: If args are invalid.
    """
    if parsed_args.which == CMD_CREATE:
        if (parsed_args.spec and parsed_args.spec not in constants.SPEC_NAMES):
            raise errors.CommandArgError(
                "%s is not valid. Choose from: %s" %
                (parsed_args.spec, ", ".join(constants.SPEC_NAMES)))
        if not ((parsed_args.build_id and parsed_args.build_target)
                or parsed_args.gce_image or parsed_args.local_disk_image):
            raise errors.CommandArgError(
                "At least one of the following should be specified: "
                "--build_id and --build_target, or --gce_image, or "
                "--local_disk_image.")
        if bool(parsed_args.build_id) != bool(parsed_args.build_target):
            raise errors.CommandArgError(
                "Must specify --build_id and --build_target at the same time.")

    if parsed_args.which in [CMD_CREATE_CUTTLEFISH, CMD_CREATE_GOLDFISH]:
        if not parsed_args.build_id or not parsed_args.build_target:
            raise errors.CommandArgError(
                "Must specify --build_id and --build_target")

    if parsed_args.which == CMD_CREATE_GOLDFISH:
        if not parsed_args.emulator_build_id:
            raise errors.CommandArgError("Must specify --emulator_build_id")

    if parsed_args.which in [
            CMD_CREATE, CMD_CREATE_CUTTLEFISH, CMD_CREATE_GOLDFISH
    ]:
        if (parsed_args.serial_log_file
                and not parsed_args.serial_log_file.endswith(".tar.gz")):
            raise errors.CommandArgError(
                "--serial_log_file must ends with .tar.gz")
        if (parsed_args.logcat_file
                and not parsed_args.logcat_file.endswith(".tar.gz")):
            raise errors.CommandArgError(
                "--logcat_file must ends with .tar.gz")


def _SetupLogging(log_file, verbose, very_verbose):
    """Setup logging.

    Args:
        log_file: path to log file.
        verbose: If True, log at DEBUG level, otherwise log at INFO level.
        very_verbose: If True, log at DEBUG level and turn on logging on
                      all libraries. Take take precedence over |verbose|.
    """
    if very_verbose:
        logger = logging.getLogger()
    else:
        logger = logging.getLogger(LOGGER_NAME)

    logging_level = logging.DEBUG if verbose or very_verbose else logging.INFO
    logger.setLevel(logging_level)

    if not log_file:
        handler = logging.StreamHandler()
    else:
        handler = logging.FileHandler(filename=log_file)
    log_formatter = logging.Formatter(LOGGING_FMT)
    handler.setFormatter(log_formatter)
    logger.addHandler(handler)


def main(argv):
    """Main entry.

    Args:
        argv: A list of system arguments.

    Returns:
        0 if success. None-zero if fails.
    """
    args = _ParseArgs(argv)
    _SetupLogging(args.log_file, args.verbose, args.very_verbose)
    args = _TranslateAlias(args)
    _VerifyArgs(args)

    config_mgr = config.AcloudConfigManager(args.config_file)
    cfg = config_mgr.Load()
    cfg.OverrideWithArgs(args)

    # Check access.
    device_driver.CheckAccess(cfg)

    report = None
    if args.which == CMD_CREATE:
        report = device_driver.CreateAndroidVirtualDevices(
            cfg,
            args.build_target,
            args.build_id,
            args.num,
            args.gce_image,
            args.local_disk_image,
            cleanup=not args.no_cleanup,
            serial_log_file=args.serial_log_file,
            logcat_file=args.logcat_file,
            autoconnect=args.autoconnect)
    elif args.which == CMD_CREATE_CUTTLEFISH:
        report = create_cuttlefish_action.CreateDevices(
            cfg=cfg,
            build_target=args.build_target,
            build_id=args.build_id,
            kernel_build_id=args.kernel_build_id,
            num=args.num,
            serial_log_file=args.serial_log_file,
            logcat_file=args.logcat_file,
            autoconnect=args.autoconnect)
    elif args.which == CMD_CREATE_GOLDFISH:
        report = create_goldfish_action.CreateDevices(
            cfg=cfg,
            build_target=args.build_target,
            build_id=args.build_id,
            emulator_build_id=args.emulator_build_id,
            gpu=args.gpu,
            num=args.num,
            serial_log_file=args.serial_log_file,
            logcat_file=args.logcat_file,
            autoconnect=args.autoconnect)
    elif args.which == CMD_DELETE:
        report = device_driver.DeleteAndroidVirtualDevices(
            cfg, args.instance_names)
    elif args.which == CMD_CLEANUP:
        report = device_driver.Cleanup(cfg, args.expiration_mins)
    elif args.which == CMD_SSHKEY:
        report = device_driver.AddSshRsa(cfg, args.user, args.ssh_rsa_path)
    elif args.which == setup_args.CMD_SETUP:
        setup.Run(args)
    else:
        sys.stderr.write("Invalid command %s" % args.which)
        return 2

    if report:
        report.Dump(args.report_file)
        if report.errors:
            msg = "\n".join(report.errors)
            sys.stderr.write("Encountered the following errors:\n%s\n" % msg)
            return 1
    return 0


if __name__ == "__main__":
    main(sys.argv[1:])