acloud: download remote image artifacts from Android Build.
Bug: 112879708
Test: atest acloud_test, m acloud && acloud create --local_instance or
acloud create --local_instance --branch master --build_target cf_x86_phone-userdebug --build_id 4972102 --log_file /tmp/acloud.log or
acloud create --local_instance --branch aosp_master --build_target aosp_cf_x86_phone-userdebug --build_id 5048089 --log_file /tmp/acloud.log
Change-Id: Ib3937807e533327ce288b546010c140d2e27228b
diff --git a/create/create.py b/create/create.py
index 7b9b23a..81710f4 100644
--- a/create/create.py
+++ b/create/create.py
@@ -25,6 +25,7 @@
from acloud.create import local_image_local_instance
from acloud.create import local_image_remote_instance
from acloud.create import remote_image_remote_instance
+from acloud.create import remote_image_local_instance
from acloud.internal import constants
@@ -50,6 +51,9 @@
if (instance_type == constants.INSTANCE_TYPE_REMOTE and
image_source == constants.IMAGE_SRC_REMOTE):
return remote_image_remote_instance.RemoteImageRemoteInstance
+ if (instance_type == constants.INSTANCE_TYPE_LOCAL and
+ image_source == constants.IMAGE_SRC_REMOTE):
+ return remote_image_local_instance.RemoteImageLocalInstance
raise errors.UnsupportedInstanceImageType(
"unsupported creation of instance type: %s, image source: %s" %
diff --git a/create/remote_image_local_instance.py b/create/remote_image_local_instance.py
new file mode 100644
index 0000000..6d85287
--- /dev/null
+++ b/create/remote_image_local_instance.py
@@ -0,0 +1,169 @@
+#!/usr/bin/env python
+#
+# Copyright 2018 - The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+r"""RemoteImageLocalInstance class.
+
+Create class that is responsible for creating a local instance AVD with a
+remote image.
+"""
+from __future__ import print_function
+import logging
+import os
+import subprocess
+import tempfile
+
+from acloud import errors
+from acloud.create import base_avd_create
+from acloud.internal import constants
+from acloud.internal.lib import android_build_client
+from acloud.internal.lib import auth
+from acloud.internal.lib import utils
+from acloud.setup import setup_common
+
+# Download remote image variables.
+_CVD_HOST_PACKAGE = "cvd-host_package.tar.gz"
+_CUTTLEFISH_COMMON_BIN_PATH = "/usr/lib/cuttlefish-common/bin/"
+_TEMP_IMAGE_FOLDER = os.path.join(tempfile.gettempdir(),
+ "acloud_image_artifacts", "cuttlefish")
+_CF_IMAGES = ["cache.img", "cmdline", "kernel", "ramdisk.img", "system.img",
+ "userdata.img", "vendor.img"]
+_BOOT_IMAGE = "boot.img"
+UNPACK_BOOTIMG_CMD = "%s -boot_img %s" % (
+ os.path.join(_CUTTLEFISH_COMMON_BIN_PATH, "unpack_boot_image.py"),
+ "%s -dest %s")
+ACL_CMD = "setfacl -m g:libvirt-qemu:rw %s"
+
+ALL_SCOPES = [android_build_client.AndroidBuildClient.SCOPE]
+
+logger = logging.getLogger(__name__)
+
+
+class RemoteImageLocalInstance(base_avd_create.BaseAVDCreate):
+ """Create class for a remote image local instance AVD."""
+
+ # pylint: disable=no-self-use
+ def Create(self, avd_spec):
+ """Create the AVD.
+
+ Args:
+ avd_spec: AVDSpec object that tells us what we're going to create.
+
+ Raises:
+ errors.NoCuttlefishCommonInstalled: cuttlefish-common doesn't install.
+ """
+ print("We will create a local instance AVD with a remote image: %s" %
+ avd_spec)
+ if not setup_common.PackageInstalled("cuttlefish-common"):
+ raise errors.NoCuttlefishCommonInstalled(
+ "Package [cuttlefish-common] is not installed!\n"
+ "Please run 'acloud setup --host' to install.")
+ self._DownloadAndProcessImageFiles(avd_spec)
+
+ def _DownloadAndProcessImageFiles(self, avd_spec):
+ """Download the CF image artifacts and process them.
+
+ Download from the Android Build system, unpack the boot img file,
+ and ACL the image files.
+
+ Args:
+ avd_spec: AVDSpec object that tells us what we're going to create.
+ """
+ cfg = avd_spec.cfg
+ build_id = avd_spec.remote_image[constants.BUILD_ID]
+ build_target = avd_spec.remote_image[constants.BUILD_TARGET]
+ extract_path = os.path.join(_TEMP_IMAGE_FOLDER, build_id)
+ logger.debug("Extract path: %s", extract_path)
+ if not os.path.exists(extract_path):
+ os.makedirs(extract_path)
+
+ # TODO(b/117189191): Check if the files are already downloaded and
+ # skip this step if they are.
+ self._DownloadRemoteImage(cfg, build_target, build_id, extract_path)
+ self._UnpackBootImage(extract_path)
+ self._AclCfImageFiles(extract_path)
+
+ @staticmethod
+ def _DownloadRemoteImage(cfg, build_target, build_id, extract_path):
+ """Download cuttlefish package and remote image then extract them.
+
+ Args:
+ cfg: An AcloudConfig instance.
+ build_target: String, the build target, e.g. cf_x86_phone-userdebug.
+ build_id: String, Build id, e.g. "2263051", "P2804227"
+ extract_path: String, a path include extracted files.
+ """
+ remote_image = "%s-img-%s.zip" % (build_target.split('-')[0],
+ build_id)
+ artifacts = [_CVD_HOST_PACKAGE, remote_image]
+
+ build_client = android_build_client.AndroidBuildClient(
+ auth.CreateCredentials(cfg, ALL_SCOPES))
+ for artifact in artifacts:
+ with utils.TempDir() as tempdir:
+ temp_filename = os.path.join(tempdir, artifact)
+ build_client.DownloadArtifact(
+ build_target,
+ build_id,
+ artifact,
+ temp_filename)
+ utils.Decompress(temp_filename, extract_path)
+
+ @staticmethod
+ def _UnpackBootImage(extract_path):
+ """Unpack Boot.img.
+
+ Args:
+ extract_path: String, a path include extracted files.
+
+ Raises:
+ errors.BootImgDoesNotExist: boot.img doesn't exist.
+ errors.UnpackBootImageError: Unpack boot.img fail.
+ """
+ bootimg_path = os.path.join(extract_path, _BOOT_IMAGE)
+ if not os.path.exists(bootimg_path):
+ raise errors.BootImgDoesNotExist(
+ "%s does not exist in %s" % (_BOOT_IMAGE, bootimg_path))
+
+ logger.info("Start to unpack boot.img.")
+ try:
+ subprocess.check_call(
+ UNPACK_BOOTIMG_CMD % (bootimg_path, extract_path),
+ shell=True)
+ except subprocess.CalledProcessError as e:
+ raise errors.UnpackBootImageError(
+ "Failed to unpack boot.img: %s" % str(e))
+ logger.info("Unpack boot.img complete!")
+
+ @staticmethod
+ def _AclCfImageFiles(extract_path):
+ """ACL related files.
+
+ Use setfacl so that libvirt does not lose access to this file if user
+ does anything to this file at any point.
+
+ Args:
+ extract_path: String, a path include extracted files.
+
+ Raises:
+ errors.CheckPathError: Path doesn't exist.
+ """
+ logger.info("Start to acl files: %s", ",".join(_CF_IMAGES))
+ for image in _CF_IMAGES:
+ image_path = os.path.join(extract_path, image)
+ if not os.path.exists(image_path):
+ raise errors.CheckPathError(
+ "Specified file doesn't exist: %s" % image_path)
+ subprocess.check_call(ACL_CMD % image_path, shell=True)
+ logger.info("ACL files completed!")
diff --git a/create/remote_image_local_instance_test.py b/create/remote_image_local_instance_test.py
new file mode 100644
index 0000000..c35e7b2
--- /dev/null
+++ b/create/remote_image_local_instance_test.py
@@ -0,0 +1,140 @@
+# Copyright 2018 - The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Tests for remote_image_local_instance."""
+
+import unittest
+import os
+import subprocess
+import mock
+
+from acloud import errors
+from acloud.create.remote_image_local_instance import RemoteImageLocalInstance
+from acloud.internal.lib import android_build_client
+from acloud.internal.lib import auth
+from acloud.internal.lib import driver_test_lib
+from acloud.internal.lib import utils
+from acloud.setup import setup_common
+
+
+# pylint: disable=invalid-name, protected-access
+class RemoteImageLocalInstanceTest(driver_test_lib.BaseDriverTest):
+ """Test remote_image_local_instance methods."""
+
+ def setUp(self):
+ """Initialize remote_image_local_instance."""
+ super(RemoteImageLocalInstanceTest, self).setUp()
+ self.build_client = mock.MagicMock()
+ self.Patch(
+ android_build_client,
+ "AndroidBuildClient",
+ return_value=self.build_client)
+ self.Patch(auth, "CreateCredentials", return_value=mock.MagicMock())
+ self.RemoteImageLocalInstance = RemoteImageLocalInstance()
+ self._fake_remote_image = {"build_target" : "aosp_cf_x86_phone-userdebug",
+ "build_id": "1234"}
+ self._extract_path = "/tmp/acloud_image_artifacts/cuttlefish/1234"
+
+ @mock.patch.object(RemoteImageLocalInstance, "_DownloadAndProcessImageFiles")
+ def testCreate(self, mock_proc):
+ """Test Create."""
+ avd_spec = mock.MagicMock()
+ # raise errors.NoCuttlefishCommonInstalled
+ self.Patch(setup_common, "PackageInstalled", return_value=False)
+ self.assertRaises(errors.NoCuttlefishCommonInstalled,
+ self.RemoteImageLocalInstance.Create,
+ avd_spec)
+
+ # Valid _DownloadAndProcessImageFiles run.
+ self.Patch(setup_common, "PackageInstalled", return_value=True)
+ self.RemoteImageLocalInstance.Create(avd_spec)
+ mock_proc.assert_called_once_with(avd_spec)
+
+ @mock.patch.object(RemoteImageLocalInstance, "_AclCfImageFiles")
+ @mock.patch.object(RemoteImageLocalInstance, "_UnpackBootImage")
+ @mock.patch.object(RemoteImageLocalInstance, "_DownloadRemoteImage")
+ def testDownloadAndProcessImageFiles(self, mock_download, mock_unpack, mock_acl):
+ """Test process remote cuttlefish image."""
+ avd_spec = mock.MagicMock()
+ avd_spec.cfg = mock.MagicMock()
+ avd_spec.remote_image = self._fake_remote_image
+ self.Patch(os.path, "exists", return_value=True)
+ self.RemoteImageLocalInstance._DownloadAndProcessImageFiles(avd_spec)
+
+ # To make sure each function execute once.
+ mock_download.assert_called_once_with(
+ avd_spec.cfg,
+ avd_spec.remote_image["build_target"],
+ avd_spec.remote_image["build_id"],
+ self._extract_path)
+ mock_unpack.assert_called_once_with(self._extract_path)
+ mock_acl.assert_called_once_with(self._extract_path)
+
+ @mock.patch.object(utils, "TempDir")
+ @mock.patch.object(utils, "Decompress")
+ def testDownloadRemoteImage(self, mock_decompress, mock_tmpdir):
+ """Test Download cuttlefish package."""
+ avd_spec = mock.MagicMock()
+ avd_spec.cfg = mock.MagicMock()
+ avd_spec.remote_image = self._fake_remote_image
+ mock_tmpdir.return_value = mock_tmpdir
+ mock_tmpdir.__exit__ = mock.MagicMock(return_value=None)
+ mock_tmpdir.__enter__ = mock.MagicMock(return_value="tmp")
+ build_id = "1234"
+ build_target = "aosp_cf_x86_phone-userdebug"
+ checkfile1 = "aosp_cf_x86_phone-img-1234.zip"
+ checkfile2 = "cvd-host_package.tar.gz"
+
+ self.RemoteImageLocalInstance._DownloadRemoteImage(
+ avd_spec.cfg,
+ avd_spec.remote_image["build_target"],
+ avd_spec.remote_image["build_id"],
+ self._extract_path)
+
+ # To validate DownloadArtifact runs twice.
+ self.assertEqual(self.build_client.DownloadArtifact.call_count, 2)
+ # To validate DownloadArtifact arguments correct.
+ self.build_client.DownloadArtifact.assert_has_calls([
+ mock.call(build_target, build_id, checkfile1,
+ "tmp/%s" % checkfile1),
+ mock.call(build_target, build_id, checkfile2,
+ "tmp/%s" % checkfile2)], any_order=True)
+ # To validate Decompress runs twice.
+ self.assertEqual(mock_decompress.call_count, 2)
+
+ @mock.patch.object(subprocess, "check_call")
+ def testUnpackBootImage(self, mock_call):
+ """Test Unpack boot image."""
+ self.Patch(os.path, "exists", side_effect=[True, False])
+ self.RemoteImageLocalInstance._UnpackBootImage(self._extract_path)
+ # check_call run once when boot.img exist.
+ self.assertEqual(mock_call.call_count, 1)
+ # raise errors.BootImgDoesNotExist when boot.img doesn't exist.
+ self.assertRaises(errors.BootImgDoesNotExist,
+ self.RemoteImageLocalInstance._UnpackBootImage,
+ self._extract_path)
+
+ @mock.patch.object(subprocess, "check_call")
+ def testAclCfImageFiles(self, mock_call):
+ """Test acl related files."""
+ self.Patch(os.path, "exists",
+ side_effect=[True, True, True, True, False, True, True])
+ # Raise error when acl required file does not exist at 5th run cehck_call.
+ self.assertRaises(errors.CheckPathError,
+ self.RemoteImageLocalInstance._AclCfImageFiles,
+ self._extract_path)
+ # it should be run check_call 4 times before raise error.
+ self.assertEqual(mock_call.call_count, 4)
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/errors.py b/errors.py
index ff992f2..600d92d 100644
--- a/errors.py
+++ b/errors.py
@@ -82,3 +82,19 @@
class GetCvdLocalHostPackageError(CreateError):
"""Can't find the lost host package."""
+
+
+class NoCuttlefishCommonInstalled(SetupError):
+ """Can't find cuttlefish_common lib."""
+
+
+class UnpackBootImageError(CreateError):
+ """Error related to unpack boot.img."""
+
+
+class BootImgDoesNotExist(CreateError):
+ """boot.img does not exist."""
+
+
+class UnsupportedCompressionFileType(SetupError):
+ """Don't support the compression file type."""
diff --git a/internal/lib/utils.py b/internal/lib/utils.py
index 2bdf8b0..ae334ac 100755
--- a/internal/lib/utils.py
+++ b/internal/lib/utils.py
@@ -30,7 +30,9 @@
import tempfile
import time
import uuid
+import zipfile
+from acloud import errors as root_errors
from acloud.internal import constants
from acloud.public import errors
@@ -363,6 +365,28 @@
raise errors.DriverError(
"rsa key is invalid: %s, error: %s" % (rsa, str(e)))
+def Decompress(sourcefile, dest=None):
+ """Decompress .zip or .tar.gz.
+
+ Args:
+ sourcefile: A string, a source file path to decompress.
+ dest: A string, a folder path as decompress destination.
+
+ Raises:
+ errors.UnsupportedCompressionFileType: Not supported extension.
+ """
+ logger.info("Start to decompress %s!", sourcefile)
+ dest_path = dest if dest else "."
+ if sourcefile.endswith(".tar.gz"):
+ with tarfile.open(sourcefile, "r:gz") as compressor:
+ compressor.extractall(dest_path)
+ elif sourcefile.endswith(".zip"):
+ with zipfile.ZipFile(sourcefile, 'r') as compressor:
+ compressor.extractall(dest_path)
+ else:
+ raise root_errors.UnsupportedCompressionFileType(
+ "Sorry, we could only support compression file type "
+ "for zip or tar.gz.")
# pylint: disable=old-style-class,no-init
class TextColors:
diff --git a/public/errors.py b/public/errors.py
index c1eeb8c..77ad0db 100755
--- a/public/errors.py
+++ b/public/errors.py
@@ -95,7 +95,3 @@
class NoGoogleSDKDetected(SetupError):
"""Can't find the SDK path."""
-
-
-class UnsupportedGoogleSDKFileType(SetupError):
- """Don't support the compression file type."""
diff --git a/setup/google_sdk.py b/setup/google_sdk.py
index 66bd082..2ff009b 100644
--- a/setup/google_sdk.py
+++ b/setup/google_sdk.py
@@ -33,11 +33,10 @@
import platform
import shutil
import sys
-import tarfile
import tempfile
import urllib2
-import zipfile
+from acloud.internal.lib import utils
from acloud.public import errors
SDK_BIN_PATH = os.path.join("google-cloud-sdk", "bin")
@@ -148,10 +147,6 @@
Download the google SDK from the GCP web.
Reference https://cloud.google.com/sdk/docs/downloads-versioned-archives.
-
- Raise:
- UnsupportedGoogleSDKFileType if we download a file type we can't
- extract.
"""
self._tmp_path = tempfile.mkdtemp(prefix="gcloud")
url = GetSdkUrl()
@@ -165,17 +160,7 @@
logger.info("Downloading google SDK: %s bytes.", file_size)
with open(file_path, 'wb') as output:
output.write(url_stream.read())
- google_sdk = None
- if filename.endswith(".tar.gz"):
- google_sdk = tarfile.open(file_path, "r:gz")
- elif filename.endswith(".zip"):
- google_sdk = zipfile.ZipFile(file_path, 'r')
- else:
- raise errors.UnsupportedGoogleSDKFileType(
- "Sorry, we could only support compression file type for zip or"
- "tar.gz.")
- google_sdk.extractall(self._tmp_path)
- google_sdk.close()
+ utils.Decompress(file_path, self._tmp_path)
self._tmp_sdk_path = os.path.join(self._tmp_path, SDK_BIN_PATH)
def CleanUp(self):