initial commit of acloud for open sourcing
The Cloud Android Driver Binaries (namely, acloud) in this project
provide the standard APIs to access and control Cloud Android devices
(i.e., Android Virtual Devices on Google Compute Engine) instantiated
by using the Android source code (e.g., device/google/gce* projects).
No code change required in the initial commit which is to track
all the changes submitted after the initial commit.
Unit tests are not part of this initial commit and thus will be
submitted as the second commit due to their current dependencies
Test: no build rule defined for python yet
Change-Id: Ib6aaadf33fa110f4532ba2d5b7be91e8ddc632a9
diff --git a/public/__init__.py b/public/__init__.py
new file mode 100755
index 0000000..e69de29
--- /dev/null
+++ b/public/__init__.py
diff --git a/public/acloud.py b/public/acloud.py
new file mode 100755
index 0000000..5badcef
--- /dev/null
+++ b/public/acloud.py
@@ -0,0 +1,351 @@
+#!/usr/bin/env python
+#
+# Copyright 2016 - The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Cloud Android Driver.
+
+This CLI manages google compute engine project for android devices.
+
+- Prerequisites:
+ See: go/acloud-manual
+
+- Configuration:
+ The script takes a required configuration file, which should look like
+ <Start of the file>
+ # If using service account
+ service_account_name: "your_account@developer.gserviceaccount.com"
+ service_account_private_key_path: "/path/to/your-project.p12"
+
+ # If using OAuth2 authentication flow
+ client_id: <client id created in the project>
+ client_secret: <client secret for the client id>
+
+ # Optional
+ ssh_private_key_path: ""
+ orientation: "portrait"
+ resolution: "800x1280x32x213"
+ network: "default"
+ machine_type: "n1-standard-1"
+ extra_data_disk_size_gb: 10 # 4G or 10G
+
+ # Required
+ project: "your-project"
+ zone: "us-central1-f"
+ storage_bucket_name: "your_google_storage_bucket_name"
+ <End of the file>
+
+ Save it at /path/to/acloud.config
+
+- Example calls:
+ - Create two instances:
+ $ acloud.par create
+ --build_target gce_x86_phone-userdebug
+ --build_id 2305447 --num 2 --config_file /path/to/acloud.config
+ --report_file /tmp/acloud_report.json --log_file /tmp/acloud.log
+
+ - Delete two instances:
+ $ acloud.par delete --instance_names
+ gce-x86-userdebug-2272605-43f9b2c6 gce-x86-userdebug-2272605-20f93a5
+ --config_file /path/to/acloud.config
+ --report_file /tmp/acloud_report.json --log_file /tmp/acloud.log
+"""
+import argparse
+import getpass
+import logging
+import os
+import sys
+
+from acloud.internal import constants
+from acloud.public import acloud_common
+from acloud.public import config
+from acloud.public import device_driver
+from acloud.public import errors
+
+LOGGING_FMT = "%(asctime)s |%(levelname)s| %(module)s:%(lineno)s| %(message)s"
+LOGGER_NAME = "google3.cloud.android.driver"
+
+# Commands
+CMD_CREATE = "create"
+CMD_DELETE = "delete"
+CMD_CLEANUP = "cleanup"
+CMD_SSHKEY = "sshkey"
+
+
+def _ParseArgs(args):
+ """Parse args.
+
+ Args:
+ args: Argument list passed from main.
+
+ Returns:
+ Parsed args.
+ """
+ usage = ",".join([CMD_CREATE, CMD_DELETE, CMD_CLEANUP, CMD_SSHKEY])
+ parser = argparse.ArgumentParser(
+ description=__doc__,
+ formatter_class=argparse.RawDescriptionHelpFormatter,
+ usage="%(prog)s {" + usage + "} ...")
+ subparsers = parser.add_subparsers()
+ subparser_list = []
+
+ # Command "create"
+ create_parser = subparsers.add_parser(CMD_CREATE)
+ create_parser.required = False
+ create_parser.set_defaults(which=CMD_CREATE)
+ create_parser.add_argument(
+ "--build_target",
+ type=str,
+ dest="build_target",
+ help="Android build target, e.g. gce_x86-userdebug, "
+ "or short names: phone, tablet, or tablet_mobile.")
+ create_parser.add_argument(
+ "--branch",
+ type=str,
+ dest="branch",
+ help="Android branch, e.g. mnc-dev or git_mnc-dev")
+ # TODO(fdeng): Support HEAD (the latest build)
+ create_parser.add_argument("--build_id",
+ type=str,
+ dest="build_id",
+ help="Android build id, e.g. 2145099, P2804227")
+ create_parser.add_argument(
+ "--spec",
+ type=str,
+ dest="spec",
+ required=False,
+ help="The name of a pre-configured device spec that we are "
+ "going to use. Choose from: %s" % ", ".join(constants.SPEC_NAMES))
+ create_parser.add_argument("--num",
+ type=int,
+ dest="num",
+ required=False,
+ default=1,
+ help="Number of instances to create.")
+ create_parser.add_argument(
+ "--gce_image",
+ type=str,
+ dest="gce_image",
+ required=False,
+ help="Name of an existing compute engine image to reuse.")
+ create_parser.add_argument("--local_disk_image",
+ type=str,
+ dest="local_disk_image",
+ required=False,
+ help="Path to a local disk image to use, "
+ "e.g /tmp/avd-system.tar.gz")
+ create_parser.add_argument(
+ "--no_cleanup",
+ dest="no_cleanup",
+ default=False,
+ action="store_true",
+ help="Do not clean up temporary disk image and compute engine image. "
+ "For debugging purposes.")
+ create_parser.add_argument(
+ "--serial_log_file",
+ type=str,
+ dest="serial_log_file",
+ required=False,
+ help="Path to a *tar.gz file where serial logs will be saved "
+ "when a device fails on boot.")
+ create_parser.add_argument(
+ "--logcat_file",
+ type=str,
+ dest="logcat_file",
+ required=False,
+ help="Path to a *tar.gz file where logcat logs will be saved "
+ "when a device fails on boot.")
+
+ subparser_list.append(create_parser)
+
+ # Command "Delete"
+ delete_parser = subparsers.add_parser(CMD_DELETE)
+ delete_parser.required = False
+ delete_parser.set_defaults(which=CMD_DELETE)
+ delete_parser.add_argument(
+ "--instance_names",
+ dest="instance_names",
+ nargs="+",
+ required=True,
+ help="The names of the instances that need to delete, "
+ "separated by spaces, e.g. --instance_names instance-1 instance-2")
+ subparser_list.append(delete_parser)
+
+ # Command "cleanup"
+ cleanup_parser = subparsers.add_parser(CMD_CLEANUP)
+ cleanup_parser.required = False
+ cleanup_parser.set_defaults(which=CMD_CLEANUP)
+ cleanup_parser.add_argument(
+ "--expiration_mins",
+ type=int,
+ dest="expiration_mins",
+ required=True,
+ help="Garbage collect all gce instances, gce images, cached disk "
+ "images that are older than |expiration_mins|.")
+ subparser_list.append(cleanup_parser)
+
+ # Command "sshkey"
+ sshkey_parser = subparsers.add_parser(CMD_SSHKEY)
+ sshkey_parser.required = False
+ sshkey_parser.set_defaults(which=CMD_SSHKEY)
+ sshkey_parser.add_argument(
+ "--user",
+ type=str,
+ dest="user",
+ default=getpass.getuser(),
+ help="The user name which the sshkey belongs to, default to: %s." %
+ getpass.getuser())
+ sshkey_parser.add_argument(
+ "--ssh_rsa_path",
+ type=str,
+ dest="ssh_rsa_path",
+ required=True,
+ help="Absolute path to the file that contains the public rsa key.")
+ subparser_list.append(sshkey_parser)
+
+ # Add common arguments.
+ for p in subparser_list:
+ acloud_common.AddCommonArguments(p)
+
+ return parser.parse_args(args)
+
+
+def _TranslateAlias(parsed_args):
+ """Translate alias to Launch Control compatible values.
+
+ This method translates alias to Launch Control compatible values.
+ - branch: "git_" prefix will be added if branch name doesn't have it.
+ - build_target: For example, "phone" will be translated to full target
+ name "git_x86_phone-userdebug",
+
+ Args:
+ parsed_args: Parsed args.
+
+ Returns:
+ Parsed args with its values being translated.
+ """
+ if parsed_args.which == CMD_CREATE:
+ if (parsed_args.branch and
+ not parsed_args.branch.startswith(constants.BRANCH_PREFIX)):
+ parsed_args.branch = constants.BRANCH_PREFIX + parsed_args.branch
+ parsed_args.build_target = constants.BUILD_TARGET_MAPPING.get(
+ parsed_args.build_target, parsed_args.build_target)
+ return parsed_args
+
+
+def _VerifyArgs(parsed_args):
+ """Verify args.
+
+ Args:
+ parsed_args: Parsed args.
+
+ Raises:
+ errors.CommandArgError: If args are invalid.
+ """
+ if parsed_args.which == CMD_CREATE:
+ if (parsed_args.spec and parsed_args.spec not in constants.SPEC_NAMES):
+ raise errors.CommandArgError(
+ "%s is not valid. Choose from: %s" %
+ (parsed_args.spec, ", ".join(constants.SPEC_NAMES)))
+ if not ((parsed_args.build_id and parsed_args.build_target) or
+ parsed_args.gce_image or parsed_args.local_disk_image):
+ raise errors.CommandArgError(
+ "At least one of the following should be specified: "
+ "--build_id and --build_target, or --gce_image, or "
+ "--local_disk_image.")
+ if bool(parsed_args.build_id) != bool(parsed_args.build_target):
+ raise errors.CommandArgError(
+ "Must specify --build_id and --build_target at the same time.")
+ if (parsed_args.serial_log_file and
+ not parsed_args.serial_log_file.endswith(".tar.gz")):
+ raise errors.CommandArgError(
+ "--serial_log_file must ends with .tar.gz")
+ if (parsed_args.logcat_file and
+ not parsed_args.logcat_file.endswith(".tar.gz")):
+ raise errors.CommandArgError(
+ "--logcat_file must ends with .tar.gz")
+
+
+def _SetupLogging(log_file, verbose, very_verbose):
+ """Setup logging.
+
+ Args:
+ log_file: path to log file.
+ verbose: If True, log at DEBUG level, otherwise log at INFO level.
+ very_verbose: If True, log at DEBUG level and turn on logging on
+ all libraries. Take take precedence over |verbose|.
+ """
+ if very_verbose:
+ logger = logging.getLogger()
+ else:
+ logger = logging.getLogger(LOGGER_NAME)
+
+ logging_level = logging.DEBUG if verbose or very_verbose else logging.INFO
+ logger.setLevel(logging_level)
+
+ if not log_file:
+ handler = logging.StreamHandler()
+ else:
+ handler = logging.FileHandler(filename=log_file)
+ log_formatter = logging.Formatter(LOGGING_FMT)
+ handler.setFormatter(log_formatter)
+ logger.addHandler(handler)
+
+
+def main(argv):
+ """Main entry.
+
+ Args:
+ argv: A list of system arguments.
+
+ Returns:
+ 0 if success. None-zero if fails.
+ """
+ args = _ParseArgs(argv)
+ _SetupLogging(args.log_file, args.verbose, args.very_verbose)
+ args = _TranslateAlias(args)
+ _VerifyArgs(args)
+
+ config_mgr = config.AcloudConfigManager(args.config_file)
+ cfg = config_mgr.Load()
+ cfg.OverrideWithArgs(args)
+
+ if args.which == CMD_CREATE:
+ report = device_driver.CreateAndroidVirtualDevices(
+ cfg,
+ args.build_target,
+ args.build_id,
+ args.num,
+ args.gce_image,
+ args.local_disk_image,
+ cleanup=not args.no_cleanup,
+ serial_log_file=args.serial_log_file,
+ logcat_file=args.logcat_file)
+ elif args.which == CMD_DELETE:
+ report = device_driver.DeleteAndroidVirtualDevices(cfg,
+ args.instance_names)
+ elif args.which == CMD_CLEANUP:
+ report = device_driver.Cleanup(cfg, args.expiration_mins)
+ elif args.which == CMD_SSHKEY:
+ report = device_driver.AddSshRsa(cfg, args.user, args.ssh_rsa_path)
+ else:
+ sys.stderr.write("Invalid command %s" % args.which)
+ return 2
+
+ report.Dump(args.report_file)
+ if report.errors:
+ msg = "\n".join(report.errors)
+ sys.stderr.write("Encountered the following errors:\n%s\n" % msg)
+ return 1
+ return 0
diff --git a/public/acloud_common.py b/public/acloud_common.py
new file mode 100755
index 0000000..a75cd8f
--- /dev/null
+++ b/public/acloud_common.py
@@ -0,0 +1,59 @@
+#!/usr/bin/env python
+#
+# Copyright 2016 - The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Common code used by both acloud and acloud_kernel tools."""
+
+DEFAULT_CONFIG_FILE = "acloud.config"
+
+
+def AddCommonArguments(parser):
+ """Adds arguments common to parsers.
+
+ Args:
+ parser: ArgumentParser object, used to parse flags.
+ """
+ parser.add_argument("--email",
+ type=str,
+ dest="email",
+ help="Email account to use for authentcation.")
+ parser.add_argument(
+ "--config_file",
+ type=str,
+ dest="config_file",
+ default=DEFAULT_CONFIG_FILE,
+ help="Path to the config file, default to acloud.config"
+ "in the current working directory")
+ parser.add_argument("--report_file",
+ type=str,
+ dest="report_file",
+ default=None,
+ help="Dump the report this file in json format. "
+ "If not specified, just log the report")
+ parser.add_argument("--log_file",
+ dest="log_file",
+ type=str,
+ default=None,
+ help="Path to log file.")
+ parser.add_argument("-v",
+ dest="verbose",
+ action="store_true",
+ default=False,
+ help="Verbose mode")
+ parser.add_argument("-vv",
+ dest="very_verbose",
+ action="store_true",
+ default=False,
+ help="Very verbose mode")
diff --git a/public/acloud_kernel/__init__.py b/public/acloud_kernel/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/public/acloud_kernel/__init__.py
diff --git a/public/acloud_kernel/acloud_kernel.py b/public/acloud_kernel/acloud_kernel.py
new file mode 100755
index 0000000..39361c4
--- /dev/null
+++ b/public/acloud_kernel/acloud_kernel.py
@@ -0,0 +1,90 @@
+#!/usr/bin/env python
+#
+# Copyright 2016 - The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Acloud Kernel Utility.
+
+This CLI implements additional functionality to acloud CLI.
+"""
+import argparse
+import os
+import sys
+
+from acloud.public import acloud_common
+from acloud.public import config
+from acloud.public.acloud_kernel import kernel_swapper
+
+DEFAULT_CONFIG_FILE = "acloud.config"
+
+# Commands
+CMD_SWAP_KERNEL = "swap_kernel"
+
+
+def _ParseArgs(args):
+ """Parse args.
+
+ Args:
+ args: argument list passed from main.
+
+ Returns:
+ Parsed args.
+ """
+ parser = argparse.ArgumentParser()
+ subparsers = parser.add_subparsers()
+
+ swap_kernel_parser = subparsers.add_parser(CMD_SWAP_KERNEL)
+ swap_kernel_parser.required = False
+ swap_kernel_parser.set_defaults(which=CMD_SWAP_KERNEL)
+ swap_kernel_parser.add_argument(
+ "--instance_name",
+ type=str,
+ dest="instance_name",
+ required=True,
+ help="The names of the instances that will have their kernels swapped, "
+ "separated by spaces, e.g. --instance_names instance-1 instance-2")
+ swap_kernel_parser.add_argument(
+ "--local_kernel_image",
+ type=str,
+ dest="local_kernel_image",
+ required=True,
+ help="Path to a local disk image to use, e.g /tmp/bzImage")
+ acloud_common.AddCommonArguments(swap_kernel_parser)
+
+ return parser.parse_args(args)
+
+
+def main(argv):
+ """Main entry.
+
+ Args:
+ argv: list of system arguments.
+
+ Returns:
+ 0 if success. Non-zero otherwise.
+ """
+ args = _ParseArgs(argv)
+ config_mgr = config.AcloudConfigManager(args.config_file)
+ cfg = config_mgr.Load()
+ cfg.OverrideWithArgs(args)
+
+ k_swapper = kernel_swapper.KernelSwapper(cfg, args.instance_name)
+ report = k_swapper.SwapKernel(args.local_kernel_image)
+
+ report.Dump(args.report_file)
+ if report.errors:
+ msg = "\n".join(report.errors)
+ sys.stderr.write("Encountered the following errors:\n%s\n" % msg)
+ return 1
+ return 0
diff --git a/public/acloud_kernel/kernel_swapper.py b/public/acloud_kernel/kernel_swapper.py
new file mode 100755
index 0000000..ca48c26
--- /dev/null
+++ b/public/acloud_kernel/kernel_swapper.py
@@ -0,0 +1,154 @@
+#!/usr/bin/env python
+#
+# Copyright 2016 - The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Kernel Swapper.
+
+This class manages swapping kernel images for a Cloud Android instance.
+"""
+import os
+import subprocess
+
+from acloud.public import errors
+from acloud.public import report
+from acloud.internal.lib import android_build_client
+from acloud.internal.lib import android_compute_client
+from acloud.internal.lib import auth
+from acloud.internal.lib import gstorage_client
+from acloud.internal.lib import utils
+
+ALL_SCOPES = ' '.join([android_build_client.AndroidBuildClient.SCOPE,
+ gstorage_client.StorageClient.SCOPE,
+ android_compute_client.AndroidComputeClient.SCOPE])
+
+# ssh flags used to communicate with the Cloud Android instance.
+SSH_FLAGS = [
+ '-q', '-o UserKnownHostsFile=/dev/null', '-o "StrictHostKeyChecking no"',
+ '-o ServerAliveInterval=10'
+]
+
+# Shell commands run on target.
+MOUNT_CMD = ('if mountpoint -q /boot ; then umount /boot ; fi ; '
+ 'mount -t ext4 /dev/block/sda1 /boot')
+REBOOT_CMD = 'nohup reboot > /dev/null 2>&1 &'
+
+
+class KernelSwapper(object):
+ """A class that manages swapping a kernel image on a Cloud Android instance.
+
+ Attributes:
+ _compute_client: AndroidCopmuteClient object, manages AVD.
+ _instance_name: string, name of Cloud Android Instance.
+ _target_ip: string, IP address of Cloud Android instance.
+ _ssh_flags: string list, flags to be used with ssh and scp.
+ """
+
+ def __init__(self, cfg, instance_name):
+ """Initialize.
+
+ Args:
+ cfg: AcloudConfig object, used to create credentials.
+ instance_name: string, instance name.
+ """
+ credentials = auth.CreateCredentials(cfg, ALL_SCOPES)
+ self._compute_client = android_compute_client.AndroidComputeClient(
+ cfg, credentials)
+ # Name of the Cloud Android instance.
+ self._instance_name = instance_name
+ # IP of the Cloud Android instance.
+ self._target_ip = self._compute_client.GetInstanceIP(instance_name)
+
+ def SwapKernel(self, local_kernel_image):
+ """Swaps the kernel image on target AVD with given kernel.
+
+ Mounts boot image containing the kernel image to the filesystem, then
+ overwrites that kernel image with a new kernel image, then reboots the
+ Cloud Android instance.
+
+ Args:
+ local_kernel_image: string, local path to a kernel image.
+
+ Returns:
+ A Report instance.
+ """
+ r = report.Report(command='swap_kernel')
+ try:
+ self._ShellCmdOnTarget(MOUNT_CMD)
+ self.PushFile(local_kernel_image, '/boot')
+ self.RebootTarget()
+ except subprocess.CalledProcessError as e:
+ r.AddError(str(e))
+ r.SetStatus(report.Status.FAIL)
+ return r
+ except errors.DeviceBootTimeoutError as e:
+ r.AddError(str(e))
+ r.SetStatus(report.Status.BOOT_FAIL)
+ return r
+
+ r.SetStatus(report.Status.SUCCESS)
+ return r
+
+ def PushFile(self, src_path, dest_path):
+ """Pushes local file to target Cloud Android instance.
+
+ Args:
+ src_path: string, local path to file to be pushed.
+ dest_path: string, path on target where to push the file to.
+
+ Raises:
+ subprocess.CalledProcessError: see _ShellCmd.
+ """
+ cmd = 'scp %s %s root@%s:%s' % (' '.join(SSH_FLAGS), src_path,
+ self._target_ip, dest_path)
+ self._ShellCmd(cmd)
+
+ def RebootTarget(self):
+ """Reboots the target Cloud Android instance and waits for boot.
+
+ Raises:
+ subprocess.CalledProcessError: see _ShellCmd.
+ errors.DeviceBootTimeoutError: if booting times out.
+ """
+ self._ShellCmdOnTarget(REBOOT_CMD)
+ self._compute_client.WaitForBoot(self._instance_name)
+
+ def _ShellCmdOnTarget(self, target_cmd):
+ """Runs a shell command on target Cloud Android instance.
+
+ Args:
+ target_cmd: string, shell command to be run on target.
+
+ Raises:
+ subprocess.CalledProcessError: see _ShellCmd.
+ """
+ ssh_cmd = 'ssh %s root@%s' % (' '.join(SSH_FLAGS), self._target_ip)
+ host_cmd = ' '.join([ssh_cmd, '"%s"' % target_cmd])
+ self._ShellCmd(host_cmd)
+
+ def _ShellCmd(self, host_cmd):
+ """Runs a shell command on host device.
+
+ Args:
+ host_cmd: string, shell command to be run on host.
+
+ Raises:
+ subprocess.CalledProcessError: For any non-zero return code of
+ host_cmd.
+ """
+ utils.GenericRetry(
+ handler=lambda e: isinstance(e, subprocess.CalledProcessError),
+ max_retry=2,
+ functor=lambda cmd: subprocess.check_call(cmd, shell=True),
+ cmd=host_cmd)
diff --git a/public/avd.py b/public/avd.py
new file mode 100755
index 0000000..d2e9c8d
--- /dev/null
+++ b/public/avd.py
@@ -0,0 +1,68 @@
+#!/usr/bin/env python
+#
+# Copyright 2016 - The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""This module defines an AVD instance.
+
+TODO(fdeng):
+ The current implementation only initialize an object
+ with IP and instance name. A complete implementation
+ will include the following features.
+ - Connect
+ - Disconnect
+ - HasApp
+ - InstallApp
+ - UninstallApp
+ - GrantAppPermission
+ Merge cloud/android/platform/devinfra/caci/framework/app_manager.py
+ with this module and updated any callers.
+"""
+
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+class AndroidVirtualDevice(object):
+ """Represent an Android device."""
+
+ def __init__(self, instance_name, ip=None):
+ """Initialize.
+
+ Args:
+ instance_name: Name of the gce instance, e.g. "instance-1"
+ ip: Ip address of the gce instance, e.g. "140.110.20.1"
+ """
+ self._ip = ip
+ self._instance_name = instance_name
+
+ @property
+ def ip(self):
+ if not self._ip:
+ raise ValueError("IP of instance %s is unknown yet." %
+ self._instance_name)
+ return self._ip
+
+ @ip.setter
+ def ip(self, value):
+ self._ip = value
+
+ @property
+ def instance_name(self):
+ return self._instance_name
+
+ def __str__(self):
+ """Return a string representation."""
+ return "<ip: %s, instance_name: %s >" % (self._ip, self._instance_name)
diff --git a/public/config.py b/public/config.py
new file mode 100755
index 0000000..c88aa41
--- /dev/null
+++ b/public/config.py
@@ -0,0 +1,228 @@
+#!/usr/bin/env python
+#
+# Copyright 2016 - The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Config manager.
+
+Three protobuf messages are defined in
+ driver/internal/config/proto/internal_config.proto
+ driver/internal/config/proto/user_config.proto
+
+Internal config file User config file
+ | |
+ v v
+ InternalConfig UserConfig
+ (proto message) (proto message)
+ | |
+ | |
+ |-> AcloudConfig <-|
+
+At runtime, AcloudConfigManager performs the following steps.
+- Load driver config file into a InternalConfig message instance.
+- Load user config file into a UserConfig message instance.
+- Create AcloudConfig using InternalConfig and UserConfig.
+
+TODO(fdeng):
+ 1. Add support for override configs with command line args.
+ 2. Scan all configs to find the right config for given branch and build_id.
+ Raise an error if the given build_id is smaller than min_build_id
+ only applies to release build id.
+ Raise an error if the branch is not supported.
+
+"""
+
+import logging
+import os
+
+from acloud.internal.proto import internal_config_pb2
+from acloud.internal.proto import user_config_pb2
+from acloud.public import errors
+from google.protobuf import text_format
+_CONFIG_DATA_PATH = os.path.join(
+ os.path.dirname(os.path.abspath(__file__)), "data")
+
+logger = logging.getLogger(__name__)
+
+
+class AcloudConfig(object):
+ """A class that holds all configurations for acloud."""
+
+ REQUIRED_FIELD = [
+ "project", "zone", "machine_type", "network", "storage_bucket_name",
+ "min_machine_size", "disk_image_name", "disk_image_mime_type"
+ ]
+
+ def __init__(self, usr_cfg, internal_cfg):
+ """Initialize.
+
+ Args:
+ usr_cfg: A protobuf object that holds the user configurations.
+ internal_cfg: A protobuf object that holds internal configurations.
+ """
+ self.service_account_name = usr_cfg.service_account_name
+ self.service_account_private_key_path = (
+ usr_cfg.service_account_private_key_path)
+ self.creds_cache_file = internal_cfg.creds_cache_file
+ self.user_agent = internal_cfg.user_agent
+ self.client_id = usr_cfg.client_id
+ self.client_secret = usr_cfg.client_secret
+
+ self.project = usr_cfg.project
+ self.zone = usr_cfg.zone
+ self.machine_type = (usr_cfg.machine_type or
+ internal_cfg.default_usr_cfg.machine_type)
+ self.network = (usr_cfg.network or
+ internal_cfg.default_usr_cfg.network)
+ self.ssh_private_key_path = usr_cfg.ssh_private_key_path
+ self.storage_bucket_name = usr_cfg.storage_bucket_name
+ self.metadata_variable = {
+ key: val
+ for key, val in
+ internal_cfg.default_usr_cfg.metadata_variable.iteritems()
+ }
+ self.metadata_variable.update(usr_cfg.metadata_variable)
+
+ self.device_resolution_map = {
+ device: resolution
+ for device, resolution in
+ internal_cfg.device_resolution_map.iteritems()
+ }
+ self.device_default_orientation_map = {
+ device: orientation
+ for device, orientation in
+ internal_cfg.device_default_orientation_map.iteritems()
+ }
+ self.min_machine_size = internal_cfg.min_machine_size
+ self.disk_image_name = internal_cfg.disk_image_name
+ self.disk_image_mime_type = internal_cfg.disk_image_mime_type
+ self.disk_image_extension = internal_cfg.disk_image_extension
+ self.disk_raw_image_name = internal_cfg.disk_raw_image_name
+ self.disk_raw_image_extension = internal_cfg.disk_raw_image_extension
+ self.valid_branch_and_min_build_id = {
+ branch: min_build_id
+ for branch, min_build_id in
+ internal_cfg.valid_branch_and_min_build_id.iteritems()
+ }
+ self.precreated_data_image_map = {
+ size_gb: image_name
+ for size_gb, image_name in
+ internal_cfg.precreated_data_image.iteritems()
+ }
+ self.extra_data_disk_size_gb = (
+ usr_cfg.extra_data_disk_size_gb or
+ internal_cfg.default_usr_cfg.extra_data_disk_size_gb)
+ if self.extra_data_disk_size_gb > 0:
+ if "cfg_sta_persistent_data_device" not in usr_cfg.metadata_variable:
+ # If user did not set it explicity, use default.
+ self.metadata_variable["cfg_sta_persistent_data_device"] = (
+ internal_cfg.default_extra_data_disk_device)
+ if "cfg_sta_ephemeral_data_size_mb" in usr_cfg.metadata_variable:
+ raise errors.ConfigError(
+ "The following settings can't be set at the same time: "
+ "extra_data_disk_size_gb and"
+ "metadata variable cfg_sta_ephemeral_data_size_mb.")
+ if "cfg_sta_ephemeral_data_size_mb" in self.metadata_variable:
+ del self.metadata_variable["cfg_sta_ephemeral_data_size_mb"]
+
+ # Fields that can be overriden by args
+ self.orientation = usr_cfg.orientation
+ self.resolution = usr_cfg.resolution
+
+ # Verify validity of configurations.
+ self.Verify()
+
+ def OverrideWithArgs(self, parsed_args):
+ """Override configuration values with args passed in from cmd line.
+
+ Args:
+ parsed_args: Args parsed from command line.
+ """
+ if parsed_args.which == "create" and parsed_args.spec:
+ if not self.resolution:
+ self.resolution = self.device_resolution_map.get(
+ parsed_args.spec, "")
+ if not self.orientation:
+ self.orientation = self.device_default_orientation_map.get(
+ parsed_args.spec, "")
+ if parsed_args.email:
+ self.service_account_name = parsed_args.email
+
+ def Verify(self):
+ """Verify configuration fields."""
+ missing = [f for f in self.REQUIRED_FIELD if not getattr(self, f)]
+ if missing:
+ raise errors.ConfigError(
+ "Missing required configuration fields: %s" % missing)
+ if (self.extra_data_disk_size_gb and self.extra_data_disk_size_gb
+ not in self.precreated_data_image_map):
+ raise errors.ConfigError(
+ "Supported extra_data_disk_size_gb options(gb): %s, "
+ "invalid value: %d" % (self.precreated_data_image_map.keys(),
+ self.extra_data_disk_size_gb))
+
+
+class AcloudConfigManager(object):
+ """A class that loads configurations."""
+
+ _DEFAULT_INTERNAL_CONFIG_PATH = os.path.join(_CONFIG_DATA_PATH,
+ "default.config")
+
+ def __init__(self,
+ user_config_path,
+ internal_config_path=_DEFAULT_INTERNAL_CONFIG_PATH):
+ """Initialize.
+
+ Args:
+ user_config_path: path to the user config.
+ internal_config_path: path to the internal conifg.
+ """
+ self._user_config_path = user_config_path
+ self._internal_config_path = internal_config_path
+
+ def Load(self):
+ """Load the configurations."""
+ internal_cfg = None
+ usr_cfg = None
+ try:
+ with open(self._internal_config_path) as config_file:
+ internal_cfg = self.LoadConfigFromProtocolBuffer(
+ config_file, internal_config_pb2.InternalConfig)
+
+ with open(self._user_config_path, "r") as config_file:
+ usr_cfg = self.LoadConfigFromProtocolBuffer(
+ config_file, user_config_pb2.UserConfig)
+ except OSError as e:
+ raise errors.ConfigError("Could not load config files: %s" %
+ str(e))
+ return AcloudConfig(usr_cfg, internal_cfg)
+
+ @staticmethod
+ def LoadConfigFromProtocolBuffer(config_file, message_type):
+ """Load config from a text-based protocol buffer file.
+
+ Args:
+ config_file: A python File object.
+ message_type: A proto message class.
+
+ Returns:
+ An instance of type "message_type" populated with data
+ from the file.
+ """
+ try:
+ config = message_type()
+ text_format.Merge(config_file.read(), config)
+ return config
+ except text_format.ParseError as e:
+ raise errors.ConfigError("Could not parse config: %s" % str(e))
diff --git a/public/device_driver.py b/public/device_driver.py
new file mode 100755
index 0000000..b0fb139
--- /dev/null
+++ b/public/device_driver.py
@@ -0,0 +1,566 @@
+#!/usr/bin/env python
+#
+# Copyright 2016 - The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Public Device Driver APIs.
+
+This module provides public device driver APIs that can be called
+as a Python library.
+
+TODO(fdeng): The following APIs have not been implemented
+ - RebootAVD(ip):
+ - RegisterSshPubKey(username, key):
+ - UnregisterSshPubKey(username, key):
+ - CleanupStaleImages():
+ - CleanupStaleDevices():
+"""
+
+import datetime
+import logging
+import os
+
+import google3
+
+import dateutil.parser
+import dateutil.tz
+
+from acloud.public import avd
+from acloud.public import errors
+from acloud.public import report
+from acloud.internal import constants
+from acloud.internal.lib import auth
+from acloud.internal.lib import android_build_client
+from acloud.internal.lib import android_compute_client
+from acloud.internal.lib import gstorage_client
+from acloud.internal.lib import utils
+
+logger = logging.getLogger(__name__)
+
+ALL_SCOPES = " ".join([android_build_client.AndroidBuildClient.SCOPE,
+ gstorage_client.StorageClient.SCOPE,
+ android_compute_client.AndroidComputeClient.SCOPE])
+
+MAX_BATCH_CLEANUP_COUNT = 100
+
+
+class AndroidVirtualDevicePool(object):
+ """A class that manages a pool of devices."""
+
+ def __init__(self, cfg, devices=None):
+ self._devices = devices or []
+ self._cfg = cfg
+ credentials = auth.CreateCredentials(cfg, ALL_SCOPES)
+ self._build_client = android_build_client.AndroidBuildClient(
+ credentials)
+ self._storage_client = gstorage_client.StorageClient(credentials)
+ self._compute_client = android_compute_client.AndroidComputeClient(
+ cfg, credentials)
+
+ def _CreateGceImageWithBuildInfo(self, build_target, build_id):
+ """Creates a Gce image using build from Launch Control.
+
+ Clone avd-system.tar.gz of a build to a cache storage bucket
+ using launch control api. And then create a Gce image.
+
+ Args:
+ build_target: Target name, e.g. "gce_x86-userdebug"
+ build_id: Build id, a string, e.g. "2263051", "P2804227"
+
+ Returns:
+ String, name of the Gce image that has been created.
+ """
+ logger.info("Creating a new gce image using build: build_id %s, "
+ "build_target %s", build_id, build_target)
+ disk_image_id = utils.GenerateUniqueName(
+ suffix=self._cfg.disk_image_name)
+ self._build_client.CopyTo(
+ build_target,
+ build_id,
+ artifact_name=self._cfg.disk_image_name,
+ destination_bucket=self._cfg.storage_bucket_name,
+ destination_path=disk_image_id)
+ disk_image_url = self._storage_client.GetUrl(
+ self._cfg.storage_bucket_name, disk_image_id)
+ try:
+ image_name = self._compute_client.GenerateImageName(build_target,
+ build_id)
+ self._compute_client.CreateImage(image_name=image_name,
+ source_uri=disk_image_url)
+ finally:
+ self._storage_client.Delete(self._cfg.storage_bucket_name,
+ disk_image_id)
+ return image_name
+
+ def _CreateGceImageWithLocalFile(self, local_disk_image):
+ """Create a Gce image with a local image file.
+
+ The local disk image can be either a tar.gz file or a
+ raw vmlinux image.
+ e.g. /tmp/avd-system.tar.gz or /tmp/android_system_disk_syslinux.img
+ If a raw vmlinux image is provided, it will be archived into a tar.gz file.
+
+ The final tar.gz file will be uploaded to a cache bucket in storage.
+
+ Args:
+ local_disk_image: string, path to a local disk image,
+
+ Returns:
+ String, name of the Gce image that has been created.
+
+ Raises:
+ DriverError: if a file with an unexpected extension is given.
+ """
+ logger.info("Creating a new gce image from a local file %s",
+ local_disk_image)
+ with utils.TempDir() as tempdir:
+ if local_disk_image.endswith(self._cfg.disk_raw_image_extension):
+ dest_tar_file = os.path.join(tempdir,
+ self._cfg.disk_image_name)
+ utils.MakeTarFile(
+ src_dict={local_disk_image: self._cfg.disk_raw_image_name},
+ dest=dest_tar_file)
+ local_disk_image = dest_tar_file
+ elif not local_disk_image.endswith(self._cfg.disk_image_extension):
+ raise errors.DriverError(
+ "Wrong local_disk_image type, must be a *%s file or *%s file"
+ % (self._cfg.disk_raw_image_extension,
+ self._cfg.disk_image_extension))
+
+ disk_image_id = utils.GenerateUniqueName(
+ suffix=self._cfg.disk_image_name)
+ self._storage_client.Upload(
+ local_src=local_disk_image,
+ bucket_name=self._cfg.storage_bucket_name,
+ object_name=disk_image_id,
+ mime_type=self._cfg.disk_image_mime_type)
+ disk_image_url = self._storage_client.GetUrl(
+ self._cfg.storage_bucket_name, disk_image_id)
+ try:
+ image_name = self._compute_client.GenerateImageName()
+ self._compute_client.CreateImage(image_name=image_name,
+ source_uri=disk_image_url)
+ finally:
+ self._storage_client.Delete(self._cfg.storage_bucket_name,
+ disk_image_id)
+ return image_name
+
+ def CreateDevices(self,
+ num,
+ build_target=None,
+ build_id=None,
+ gce_image=None,
+ local_disk_image=None,
+ cleanup=True,
+ extra_data_disk_size_gb=None,
+ precreated_data_image=None):
+ """Creates |num| devices for given build_target and build_id.
+
+ - If gce_image is provided, will use it to create an instance.
+ - If local_disk_image is provided, will upload it to a temporary
+ caching storage bucket which is defined by user as |storage_bucket_name|
+ And then create an gce image with it; and then create an instance.
+ - If build_target and build_id are provided, will clone the disk image
+ via launch control to the temporary caching storage bucket.
+ And then create an gce image with it; and then create an instance.
+
+ Args:
+ num: Number of devices to create.
+ build_target: Target name, e.g. "gce_x86-userdebug"
+ build_id: Build id, a string, e.g. "2263051", "P2804227"
+ gce_image: string, if given, will use this image
+ instead of creating a new one.
+ implies cleanup=False.
+ local_disk_image: string, path to a local disk image, e.g.
+ /tmp/avd-system.tar.gz
+ cleanup: boolean, if True clean up compute engine image after creating
+ the instance.
+ extra_data_disk_size_gb: Integer, size of extra disk, or None.
+ precreated_data_image: A string, the image to use for the extra disk.
+
+ Raises:
+ errors.DriverError: If no source is specified for image creation.
+ """
+ if gce_image:
+ # GCE image is provided, we can directly move to instance creation.
+ logger.info("Using existing gce image %s", gce_image)
+ image_name = gce_image
+ cleanup = False
+ elif local_disk_image:
+ image_name = self._CreateGceImageWithLocalFile(local_disk_image)
+ elif build_target and build_id:
+ image_name = self._CreateGceImageWithBuildInfo(build_target,
+ build_id)
+ else:
+ raise errors.DriverError(
+ "Invalid image source, must specify one of the following: gce_image, "
+ "local_disk_image, or build_target and build id.")
+
+ # Create GCE instances.
+ try:
+ for _ in range(num):
+ instance = self._compute_client.GenerateInstanceName(
+ build_target, build_id)
+ extra_disk_name = None
+ if extra_data_disk_size_gb > 0:
+ extra_disk_name = self._compute_client.GetDataDiskName(
+ instance)
+ self._compute_client.CreateDisk(extra_disk_name,
+ precreated_data_image,
+ extra_data_disk_size_gb)
+ self._compute_client.CreateInstance(instance, image_name,
+ extra_disk_name)
+ ip = self._compute_client.GetInstanceIP(instance)
+ self.devices.append(avd.AndroidVirtualDevice(
+ ip=ip, instance_name=instance))
+ finally:
+ if cleanup:
+ self._compute_client.DeleteImage(image_name)
+
+ def DeleteDevices(self):
+ """Deletes devices.
+
+ Returns:
+ A tuple, (deleted, failed, error_msgs)
+ deleted: A list of names of instances that have been deleted.
+ faild: A list of names of instances that we fail to delete.
+ error_msgs: A list of failure messages.
+ """
+ instance_names = [device.instance_name for device in self._devices]
+ return self._compute_client.DeleteInstances(instance_names,
+ self._cfg.zone)
+
+ def WaitForBoot(self):
+ """Waits for all devices to boot up.
+
+ Returns:
+ A dictionary that contains all the failures.
+ The key is the name of the instance that fails to boot,
+ the value is an errors.DeviceBootTimeoutError object.
+ """
+ failures = {}
+ for device in self._devices:
+ try:
+ self._compute_client.WaitForBoot(device.instance_name)
+ except errors.DeviceBootTimeoutError as e:
+ failures[device.instance_name] = e
+ return failures
+
+ @property
+ def devices(self):
+ """Returns a list of devices in the pool.
+
+ Returns:
+ A list of devices in the pool.
+ """
+ return self._devices
+
+
+def _AddDeletionResultToReport(report_obj, deleted, failed, error_msgs,
+ resource_name):
+ """Adds deletion result to a Report object.
+
+ This function will add the following to report.data.
+ "deleted": [
+ {"name": "resource_name", "type": "resource_name"},
+ ],
+ "failed": [
+ {"name": "resource_name", "type": "resource_name"},
+ ],
+ This function will append error_msgs to report.errors.
+
+ Args:
+ report_obj: A Report object.
+ deleted: A list of names of the resources that have been deleted.
+ failed: A list of names of the resources that we fail to delete.
+ error_msgs: A list of error message strings to be added to the report.
+ resource_name: A string, representing the name of the resource.
+ """
+ for name in deleted:
+ report_obj.AddData(key="deleted",
+ value={"name": name,
+ "type": resource_name})
+ for name in failed:
+ report_obj.AddData(key="failed",
+ value={"name": name,
+ "type": resource_name})
+ report_obj.AddErrors(error_msgs)
+ if failed or error_msgs:
+ report_obj.SetStatus(report.Status.FAIL)
+
+
+def _FetchSerialLogsFromDevices(compute_client, instance_names, output_file,
+ port):
+ """Fetch serial logs from a port for a list of devices to a local file.
+
+ Args:
+ compute_client: An object of android_compute_client.AndroidComputeClient
+ instance_names: A list of instance names.
+ output_file: A path to a file ending with "tar.gz"
+ port: The number of serial port to read from, 0 for serial output, 1 for
+ logcat.
+ """
+ with utils.TempDir() as tempdir:
+ src_dict = {}
+ for instance_name in instance_names:
+ serial_log = compute_client.GetSerialPortOutput(
+ instance=instance_name, port=port)
+ file_name = "%s.log" % instance_name
+ file_path = os.path.join(tempdir, file_name)
+ src_dict[file_path] = file_name
+ with open(file_path, "w") as f:
+ f.write(serial_log.encode("utf-8"))
+ utils.MakeTarFile(src_dict, output_file)
+
+
+def CreateAndroidVirtualDevices(cfg,
+ build_target=None,
+ build_id=None,
+ num=1,
+ gce_image=None,
+ local_disk_image=None,
+ cleanup=True,
+ serial_log_file=None,
+ logcat_file=None):
+ """Creates one or multiple android devices.
+
+ Args:
+ cfg: An AcloudConfig instance.
+ build_target: Target name, e.g. "gce_x86-userdebug"
+ build_id: Build id, a string, e.g. "2263051", "P2804227"
+ num: Number of devices to create.
+ gce_image: string, if given, will use this gce image
+ instead of creating a new one.
+ implies cleanup=False.
+ local_disk_image: string, path to a local disk image, e.g.
+ /tmp/avd-system.tar.gz
+ cleanup: boolean, if True clean up compute engine image and
+ disk image in storage after creating the instance.
+ serial_log_file: A path to a file where serial output should
+ be saved to. Logs will be fetch only on boot failure.
+ logcat_file: A path to a file where logcat logs should be saved.
+ Logs will be fetch only on boot failure.
+
+ Returns:
+ A Report instance.
+ """
+ r = report.Report(command="create")
+ credentials = auth.CreateCredentials(cfg, ALL_SCOPES)
+ compute_client = android_compute_client.AndroidComputeClient(cfg,
+ credentials)
+ try:
+ device_pool = AndroidVirtualDevicePool(cfg)
+ device_pool.CreateDevices(
+ num,
+ build_target,
+ build_id,
+ gce_image,
+ local_disk_image,
+ cleanup,
+ extra_data_disk_size_gb=cfg.extra_data_disk_size_gb,
+ precreated_data_image=cfg.precreated_data_image_map.get(
+ cfg.extra_data_disk_size_gb))
+ failures = device_pool.WaitForBoot()
+ # Write result to report.
+ for device in device_pool.devices:
+ device_dict = {"ip": device.ip,
+ "instance_name": device.instance_name}
+ if device.instance_name in failures:
+ r.AddData(key="devices_failing_boot", value=device_dict)
+ r.AddError(str(failures[device.instance_name]))
+ else:
+ r.AddData(key="devices", value=device_dict)
+ if failures:
+ r.SetStatus(report.Status.BOOT_FAIL)
+ else:
+ r.SetStatus(report.Status.SUCCESS)
+
+ # Dump serial and logcat logs.
+ if serial_log_file:
+ _FetchSerialLogsFromDevices(compute_client,
+ instance_names=failures.keys(),
+ port=constants.DEFAULT_SERIAL_PORT,
+ output_file=serial_log_file)
+ if logcat_file:
+ _FetchSerialLogsFromDevices(compute_client,
+ instance_names=failures.keys(),
+ port=constants.LOGCAT_SERIAL_PORT,
+ output_file=logcat_file)
+ except errors.DriverError as e:
+ r.AddError(str(e))
+ r.SetStatus(report.Status.FAIL)
+ return r
+
+
+def DeleteAndroidVirtualDevices(cfg, instance_names):
+ """Deletes android devices.
+
+ Args:
+ cfg: An AcloudConfig instance.
+ instance_names: A list of names of the instances to delete.
+
+ Returns:
+ A Report instance.
+ """
+ r = report.Report(command="delete")
+ credentials = auth.CreateCredentials(cfg, ALL_SCOPES)
+ compute_client = android_compute_client.AndroidComputeClient(cfg,
+ credentials)
+ try:
+ deleted, failed, error_msgs = compute_client.DeleteInstances(
+ instance_names, cfg.zone)
+ _AddDeletionResultToReport(
+ r, deleted,
+ failed, error_msgs,
+ resource_name="instance")
+ if r.status == report.Status.UNKNOWN:
+ r.SetStatus(report.Status.SUCCESS)
+ except errors.DriverError as e:
+ r.AddError(str(e))
+ r.SetStatus(report.Status.FAIL)
+ return r
+
+
+def _FindOldItems(items, cut_time, time_key):
+ """Finds items from |items| whose timestamp is earlier than |cut_time|.
+
+ Args:
+ items: A list of items. Each item is a dictionary represent
+ the properties of the item. It should has a key as noted
+ by time_key.
+ cut_time: A datetime.datatime object.
+ time_key: String, key for the timestamp.
+
+ Returns:
+ A list of those from |items| whose timestamp is earlier than cut_time.
+ """
+ cleanup_list = []
+ for item in items:
+ t = dateutil.parser.parse(item[time_key])
+ if t < cut_time:
+ cleanup_list.append(item)
+ return cleanup_list
+
+
+def Cleanup(cfg, expiration_mins):
+ """Cleans up stale gce images, gce instances, and disk images in storage.
+
+ Args:
+ cfg: An AcloudConfig instance.
+ expiration_mins: Integer, resources older than |expiration_mins| will
+ be cleaned up.
+
+ Returns:
+ A Report instance.
+ """
+ r = report.Report(command="cleanup")
+ try:
+ cut_time = (datetime.datetime.now(dateutil.tz.tzlocal()) -
+ datetime.timedelta(minutes=expiration_mins))
+ logger.info(
+ "Cleaning up any gce images/instances and cached build artifacts."
+ "in google storage that are older than %s", cut_time)
+ credentials = auth.CreateCredentials(cfg, ALL_SCOPES)
+ compute_client = android_compute_client.AndroidComputeClient(
+ cfg, credentials)
+ storage_client = gstorage_client.StorageClient(credentials)
+
+ # Cleanup expired instances
+ items = compute_client.ListInstances(zone=cfg.zone)
+ cleanup_list = [
+ item["name"]
+ for item in _FindOldItems(items, cut_time, "creationTimestamp")
+ ]
+ logger.info("Found expired instances: %s", cleanup_list)
+ for i in range(0, len(cleanup_list), MAX_BATCH_CLEANUP_COUNT):
+ result = compute_client.DeleteInstances(
+ instances=cleanup_list[i:i + MAX_BATCH_CLEANUP_COUNT],
+ zone=cfg.zone)
+ _AddDeletionResultToReport(r, *result, resource_name="instance")
+
+ # Cleanup expired images
+ items = compute_client.ListImages()
+ skip_list = cfg.precreated_data_image_map.viewvalues()
+ cleanup_list = [
+ item["name"]
+ for item in _FindOldItems(items, cut_time, "creationTimestamp")
+ if item["name"] not in skip_list
+ ]
+ logger.info("Found expired images: %s", cleanup_list)
+ for i in range(0, len(cleanup_list), MAX_BATCH_CLEANUP_COUNT):
+ result = compute_client.DeleteImages(
+ image_names=cleanup_list[i:i + MAX_BATCH_CLEANUP_COUNT])
+ _AddDeletionResultToReport(r, *result, resource_name="image")
+
+ # Cleanup expired disks
+ # Disks should have been attached to instances with autoDelete=True.
+ # However, sometimes disks may not be auto deleted successfully.
+ items = compute_client.ListDisks(zone=cfg.zone)
+ cleanup_list = [
+ item["name"]
+ for item in _FindOldItems(items, cut_time, "creationTimestamp")
+ if not item.get("users")
+ ]
+ logger.info("Found expired disks: %s", cleanup_list)
+ for i in range(0, len(cleanup_list), MAX_BATCH_CLEANUP_COUNT):
+ result = compute_client.DeleteDisks(
+ disk_names=cleanup_list[i:i + MAX_BATCH_CLEANUP_COUNT],
+ zone=cfg.zone)
+ _AddDeletionResultToReport(r, *result, resource_name="disk")
+
+ # Cleanup expired google storage
+ items = storage_client.List(bucket_name=cfg.storage_bucket_name)
+ cleanup_list = [
+ item["name"]
+ for item in _FindOldItems(items, cut_time, "timeCreated")
+ ]
+ logger.info("Found expired cached artifacts: %s", cleanup_list)
+ for i in range(0, len(cleanup_list), MAX_BATCH_CLEANUP_COUNT):
+ result = storage_client.DeleteFiles(
+ bucket_name=cfg.storage_bucket_name,
+ object_names=cleanup_list[i:i + MAX_BATCH_CLEANUP_COUNT])
+ _AddDeletionResultToReport(
+ r, *result, resource_name="cached_build_artifact")
+
+ # Everything succeeded, write status to report.
+ if r.status == report.Status.UNKNOWN:
+ r.SetStatus(report.Status.SUCCESS)
+ except errors.DriverError as e:
+ r.AddError(str(e))
+ r.SetStatus(report.Status.FAIL)
+ return r
+
+
+def AddSshRsa(cfg, user, ssh_rsa_path):
+ """Add public ssh rsa key to the project.
+
+ Args:
+ cfg: An AcloudConfig instance.
+ user: the name of the user which the key belongs to.
+ ssh_rsa_path: The absolute path to public rsa key.
+
+ Returns:
+ A Report instance.
+ """
+ r = report.Report(command="sshkey")
+ try:
+ credentials = auth.CreateCredentials(cfg, ALL_SCOPES)
+ compute_client = android_compute_client.AndroidComputeClient(
+ cfg, credentials)
+ compute_client.AddSshRsa(user, ssh_rsa_path)
+ r.SetStatus(report.Status.SUCCESS)
+ except errors.DriverError as e:
+ r.AddError(str(e))
+ r.SetStatus(report.Status.FAIL)
+ return r
diff --git a/public/errors.py b/public/errors.py
new file mode 100755
index 0000000..c027e40
--- /dev/null
+++ b/public/errors.py
@@ -0,0 +1,84 @@
+#!/usr/bin/env python
+#
+# Copyright 2016 - The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Define errors that are raised by the driver."""
+
+import json
+
+HTTP_NOT_FOUND_CODE = 404
+
+
+class DriverError(Exception):
+ """Base Android Gce driver exception."""
+
+
+class ConfigError(DriverError):
+ """Error related to config."""
+
+
+class CommandArgError(DriverError):
+ """Error related to command line args."""
+
+
+class GceOperationTimeoutError(DriverError):
+ """Error raised when a GCE operation timedout."""
+
+
+class HttpError(DriverError):
+ """Error related to http requests."""
+
+ def __init__(self, code, message):
+ self.code = code
+ super(HttpError, self).__init__(message)
+
+ @staticmethod
+ def CreateFromHttpError(http_error):
+ """Create from an apiclient.errors.HttpError.
+
+ Parse the error code from apiclient.errors.HttpError
+ and create an instance of HttpError from this module
+ that has the error code.
+
+ Args:
+ http_error: An apiclient.errors.HttpError instance.
+
+ Returns:
+ An HttpError instance from this module.
+ """
+ return HttpError(http_error.resp.status, str(http_error))
+
+
+class ResourceNotFoundError(HttpError):
+ """Error raised when a resource is not found."""
+
+
+class InvalidVirtualDeviceIpError(DriverError):
+ """Invalid virtual device's IP is set.
+
+ Raise this when the virtual device's IP of an AVD instance is invalid.
+ """
+
+
+class DeviceBootTimeoutError(DriverError):
+ """Raised when an AVD defice failed to boot within timeout."""
+
+
+class HasRetriableRequestsError(DriverError):
+ """Raised when some retriable requests fail in a batch execution."""
+
+
+class AuthentcationError(DriverError):
+ """Raised when authentication fails."""
diff --git a/public/report.py b/public/report.py
new file mode 100755
index 0000000..2048329
--- /dev/null
+++ b/public/report.py
@@ -0,0 +1,170 @@
+#!/usr/bin/env python
+#
+# Copyright 2016 - The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Command report.
+
+Report class holds the results of a command execution.
+Each driver API call will generate a report instance.
+
+If running the CLI of the driver, a report will
+be printed as logs. And it will also be dumped to a json file
+if requested via command line option.
+
+The json format of a report dump looks like:
+
+ - A failed "delete" command:
+ {
+ "command": "delete",
+ "data": {},
+ "errors": [
+ "Can't find instances: ['104.197.110.255']"
+ ],
+ "status": "FAIL"
+ }
+
+ - A successful "create" command:
+ {
+ "command": "create",
+ "data": {
+ "devices": [
+ {
+ "instance_name": "instance_1",
+ "ip": "104.197.62.36"
+ },
+ {
+ "instance_name": "instance_2",
+ "ip": "104.197.62.37"
+ }
+ ]
+ },
+ "errors": [],
+ "status": "SUCCESS"
+ }
+"""
+
+import json
+import logging
+import os
+
+logger = logging.getLogger(__name__)
+
+
+class Status(object):
+ """Status of acloud command."""
+
+ SUCCESS = "SUCCESS"
+ FAIL = "FAIL"
+ BOOT_FAIL = "BOOT_FAIL"
+ UNKNOWN = "UNKNOWN"
+
+ SEVERITY_ORDER = {UNKNOWN: 0, SUCCESS: 1, FAIL: 2, BOOT_FAIL: 3}
+
+ @classmethod
+ def IsMoreSevere(cls, candidate, reference):
+ """Compare the severity of two statuses.
+
+ Args:
+ candidate: One of the statuses.
+ reference: One of the statuses.
+
+ Returns:
+ True if candidate is more severe than reference,
+ False otherwise.
+
+ Raises:
+ ValueError: if candidate or reference is not a known state.
+ """
+ if (candidate not in cls.SEVERITY_ORDER or
+ reference not in cls.SEVERITY_ORDER):
+ raise ValueError("%s or %s is not recognized." %
+ (candidate, reference))
+ return cls.SEVERITY_ORDER[candidate] > cls.SEVERITY_ORDER[reference]
+
+
+class Report(object):
+ """A class that stores and generates report."""
+
+ def __init__(self, command):
+ """Initialize.
+
+ Args:
+ command: A string, name of the command.
+ """
+ self.command = command
+ self.status = Status.UNKNOWN
+ self.errors = []
+ self.data = {}
+
+ def AddData(self, key, value):
+ """Add a key-val to the report.
+
+ Args:
+ key: A key of basic type.
+ value: A value of any json compatible type.
+ """
+ self.data.setdefault(key, []).append(value)
+
+ def AddError(self, error):
+ """Add error message.
+
+ Args:
+ error: A string.
+ """
+ self.errors.append(error)
+
+ def AddErrors(self, errors):
+ """Add a list of error messages.
+
+ Args:
+ errors: A list of string.
+ """
+ self.errors.extend(errors)
+
+ def SetStatus(self, status):
+ """Set status.
+
+ Args:
+ status: One of the status in Status.
+ """
+ if Status.IsMoreSevere(status, self.status):
+ self.status = status
+ else:
+ logger.debug(
+ "report: Current status is %s, "
+ "requested to update to a status with lower severity %s, ignored.",
+ self.status, status)
+
+ def Dump(self, report_file):
+ """Dump report content to a file.
+
+ Args:
+ report_file: A path to a file where result will be dumped to.
+ If None, will only output result as logs.
+ """
+ result = dict(command=self.command,
+ status=self.status,
+ errors=self.errors,
+ data=self.data)
+ logger.info("Report: %s", json.dumps(result, indent=2))
+ if not report_file:
+ return
+ try:
+ with open(report_file, "w") as f:
+ json.dump(result, f, indent=2)
+ logger.info("Report file generated at %s",
+ os.path.abspath(report_file))
+ except OSError as e:
+ logger.error("Failed to dump report to file: %s", str(e))