setup/gcp_setup_runner.py

#!/usr/bin/env python
#
# Copyright 2018 - The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Gcloud setup runner."""

from __future__ import print_function
import logging
import os
import re
import subprocess

from acloud import errors
from acloud.internal.lib import utils
from acloud.public import config
from acloud.setup import base_task_runner
from acloud.setup import google_sdk


logger = logging.getLogger(__name__)

# APIs that need to be enabled for GCP project.
_ANDROID_BUILD_SERVICE = "androidbuildinternal.googleapis.com"
_COMPUTE_ENGINE_SERVICE = "compute.googleapis.com"
_GOOGLE_CLOUD_STORAGE_SERVICE = "storage-component.googleapis.com"
_GOOGLE_APIS = [
    _GOOGLE_CLOUD_STORAGE_SERVICE, _ANDROID_BUILD_SERVICE,
    _COMPUTE_ENGINE_SERVICE
]
_BUILD_SERVICE_ACCOUNT = "android-build-prod@system.gserviceaccount.com"
_BILLING_ENABLE_MSG = "billingEnabled: true"
_DEFAULT_SSH_FOLDER = os.path.expanduser("~/.ssh")
_DEFAULT_SSH_KEY = "acloud_rsa"
_DEFAULT_SSH_PRIVATE_KEY = os.path.join(_DEFAULT_SSH_FOLDER,
                                        _DEFAULT_SSH_KEY)
_DEFAULT_SSH_PUBLIC_KEY = os.path.join(_DEFAULT_SSH_FOLDER,
                                       _DEFAULT_SSH_KEY + ".pub")
_GCLOUD_COMPONENT_ALPHA = "alpha"
# Bucket naming parameters
_BUCKET_HEADER = "gs://"
_BUCKET_LENGTH_LIMIT = 63
_DEFAULT_BUCKET_HEADER = "acloud"
_DEFAULT_BUCKET_REGION = "US"
_INVALID_BUCKET_NAME_END_CHARS = "_-"
_PROJECT_SEPARATOR = ":"
# Regular expression to get project/zone/bucket information.
_BUCKET_RE = re.compile(r"^gs://(?P<bucket>.+)/")
_BUCKET_REGION_RE = re.compile(r"^Location constraint:(?P<region>.+)")
_PROJECT_RE = re.compile(r"^project = (?P<project>.+)")
_ZONE_RE = re.compile(r"^zone = (?P<zone>.+)")


def UpdateConfigFile(config_path, item, value):
    """Update config data.

    Case A: config file contain this item.
        In config, "project = A_project". New value is B_project
        Set config "project = B_project".
    Case B: config file didn't contain this item.
        New value is B_project.
        Setup config as "project = B_project".

    Args:
        config_path: String, acloud config path.
        item: String, item name in config file. EX: project, zone
        value: String, value of item in config file.

    TODO(111574698): Refactor this to minimize writes to the config file.
    TODO(111574698): Use proto method to update config.
    """
    write_lines = []
    find_item = False
    write_line = item + ": \"" + value + "\"\n"
    if os.path.isfile(config_path):
        with open(config_path, "r") as cfg_file:
            for read_line in cfg_file.readlines():
                if read_line.startswith(item + ":"):
                    find_item = True
                    write_lines.append(write_line)
                else:
                    write_lines.append(read_line)
    if not find_item:
        write_lines.append(write_line)
    with open(config_path, "w") as cfg_file:
        cfg_file.writelines(write_lines)


def SetupSSHKeys(config_path, private_key_path, public_key_path):
    """Setup the pair of the ssh key for acloud.config.

    User can use the default path: "~/.ssh/acloud_rsa".

    Args:
        config_path: String, acloud config path.
        private_key_path: Path to the private key file.
                          e.g. ~/.ssh/acloud_rsa
        public_key_path: Path to the public key file.
                         e.g. ~/.ssh/acloud_rsa.pub
    """
    private_key_path = os.path.expanduser(private_key_path)
    if (private_key_path == "" or public_key_path == ""
            or private_key_path == _DEFAULT_SSH_PRIVATE_KEY):
        utils.CreateSshKeyPairIfNotExist(_DEFAULT_SSH_PRIVATE_KEY,
                                         _DEFAULT_SSH_PUBLIC_KEY)
        UpdateConfigFile(config_path, "ssh_private_key_path",
                         _DEFAULT_SSH_PRIVATE_KEY)
        UpdateConfigFile(config_path, "ssh_public_key_path",
                         _DEFAULT_SSH_PUBLIC_KEY)


def _InputIsEmpty(input_string):
    """Check input string is empty.

    Tool requests user to input client ID & client secret.
    This basic check can detect user input is empty.

    Args:
        input_string: String, user input string.

    Returns:
        Boolean: True if input is empty, False otherwise.
    """
    if input_string is None:
        return True
    if input_string == "":
        print("Please enter a non-empty value.")
        return True
    return False


class GoogleSDKBins(object):
    """Class to run tools in the Google SDK."""

    def __init__(self, google_sdk_folder):
        """GoogleSDKBins initialize.

        Args:
            google_sdk_folder: String, google sdk path.
        """
        self.gcloud_command_path = os.path.join(google_sdk_folder, "gcloud")
        self.gsutil_command_path = os.path.join(google_sdk_folder, "gsutil")

    def RunGcloud(self, cmd, **kwargs):
        """Run gcloud command.

        Args:
            cmd: String list, command strings.
                  Ex: [config], then this function call "gcloud config".
            **kwargs: dictionary of keyword based args to pass to func.

        Returns:
            String, return message after execute gcloud command.
        """
        return subprocess.check_output([self.gcloud_command_path] + cmd, **kwargs)

    def RunGsutil(self, cmd, **kwargs):
        """Run gsutil command.

        Args:
            cmd : String list, command strings.
                  Ex: [list], then this function call "gsutil list".
            **kwargs: dictionary of keyword based args to pass to func.

        Returns:
            String, return message after execute gsutil command.
        """
        return subprocess.check_output([self.gsutil_command_path] + cmd, **kwargs)


class GcpTaskRunner(base_task_runner.BaseTaskRunner):
    """Runner to setup google cloud user information."""

    WELCOME_MESSAGE_TITLE = "Setup google cloud user information"
    WELCOME_MESSAGE = (
        "This step will walk you through gcloud SDK installation."
        "Then configure gcloud user information."
        "Finally enable some gcloud API services.")

    def __init__(self, config_path):
        """Initialize parameters.

        Load config file to get current values.

        Args:
            config_path: String, acloud config path.
        """
        # pylint: disable=invalid-name
        config_mgr = config.AcloudConfigManager(config_path)
        cfg = config_mgr.Load()
        self.config_path = config_mgr.user_config_path
        self.project = cfg.project
        self.zone = cfg.zone
        self.storage_bucket_name = cfg.storage_bucket_name
        self.ssh_private_key_path = cfg.ssh_private_key_path
        self.ssh_public_key_path = cfg.ssh_public_key_path
        self.stable_host_image_name = cfg.stable_host_image_name
        self.client_id = cfg.client_id
        self.client_secret = cfg.client_secret
        self.service_account_name = cfg.service_account_name
        self.service_account_private_key_path = cfg.service_account_private_key_path
        self.service_account_json_private_key_path = cfg.service_account_json_private_key_path

    def ShouldRun(self):
        """Check if we actually need to run GCP setup.

        We'll only do the gcp setup if certain fields in the cfg are empty.

        Returns:
            True if reqired config fields are empty, False otherwise.
        """
        # We need to ensure the config has the proper auth-related fields set,
        # so config requires just 1 of the following:
        # 1. client id/secret
        # 2. service account name/private key path
        # 3. service account json private key path
        if ((not self.client_id or not self.client_secret)
                and (not self.service_account_name or not self.service_account_private_key_path)
                and not self.service_account_json_private_key_path):
            return True

        # If a project isn't set, then we need to run setup.
        return not self.project

    def _Run(self):
        """Run GCP setup task."""
        self._SetupGcloudInfo()
        SetupSSHKeys(self.config_path, self.ssh_private_key_path,
                     self.ssh_public_key_path)

    def _SetupGcloudInfo(self):
        """Setup Gcloud user information.
            1. Setup Gcloud SDK tools.
            2. Setup Gcloud project.
                a. Setup Gcloud project and zone.
                b. Setup Client ID and Client secret.
                c. Setup Google Cloud Storage bucket.
            3. Enable Gcloud API services.
        """
        google_sdk_init = google_sdk.GoogleSDK()
        try:
            google_sdk_runner = GoogleSDKBins(google_sdk_init.GetSDKBinPath())
            google_sdk_init.InstallGcloudComponent(google_sdk_runner,
                                                   _GCLOUD_COMPONENT_ALPHA)
            self._SetupProject(google_sdk_runner)
            self._EnableGcloudServices(google_sdk_runner)
            self._CreateStableHostImage()
        finally:
            google_sdk_init.CleanUp()

    def _CreateStableHostImage(self):
        """Create the stable host image."""
        # Write default stable_host_image_name with dummy value.
        # TODO(113091773): An additional step to create the host image.
        if not self.stable_host_image_name:
            UpdateConfigFile(self.config_path, "stable_host_image_name", "")


    def _NeedProjectSetup(self):
        """Confirm project setup should run or not.

        If the project settings (project name and zone) are blank (either one),
        we'll run the project setup flow. If they are set, we'll check with
        the user if they want to update them.

        Returns:
            Boolean: True if we need to setup the project, False otherwise.
        """
        user_question = (
            "Your default Project/Zone settings are:\n"
            "project:[%s]\n"
            "zone:[%s]\n"
            "Would you like to update them?[y/N]: \n") % (self.project, self.zone)

        if not self.project or not self.zone:
            logger.info("Project or zone is empty. Start to run setup process.")
            return True
        return utils.GetUserAnswerYes(user_question)

    def _NeedClientIDSetup(self, project_changed):
        """Confirm client setup should run or not.

        If project changed, client ID must also have to change.
        So tool will force to run setup function.
        If client ID or client secret is empty, tool force to run setup function.
        If project didn't change and config hold user client ID/secret, tool
        would skip client ID setup.

        Args:
            project_changed: Boolean, True for project changed.

        Returns:
            Boolean: True for run setup function.
        """
        if project_changed:
            logger.info("Your project changed. Start to run setup process.")
            return True
        elif not self.client_id or not self.client_secret:
            logger.info("Client ID or client secret is empty. Start to run setup process.")
            return True
        logger.info("Project was unchanged and client ID didn't need to changed.")
        return False

    def _SetupProject(self, gcloud_runner):
        """Setup gcloud project information.

        Setup project and zone.
        Setup client ID and client secret.
        Make sure billing account enabled in project.
        Setup Google Cloud Storage bucket.

        Args:
            gcloud_runner: A GcloudRunner class to run "gcloud" command.
        """
        project_changed = False
        if self._NeedProjectSetup():
            project_changed = self._UpdateProject(gcloud_runner)
        if self._NeedClientIDSetup(project_changed):
            self._SetupClientIDSecret()
        self._CheckBillingEnable(gcloud_runner)
        self._SetupStorageBucket(gcloud_runner)

    def _UpdateProject(self, gcloud_runner):
        """Setup gcloud project name and zone name and check project changed.

        Run "gcloud init" to handle gcloud project setup.
        Then "gcloud list" to get user settings information include "project" & "zone".
        Record project_changed for next setup steps.

        Args:
            gcloud_runner: A GcloudRunner class to run "gcloud" command.

        Returns:
            project_changed: True for project settings changed.
        """
        project_changed = False
        gcloud_runner.RunGcloud(["init"])
        gcp_config_list_out = gcloud_runner.RunGcloud(["config", "list"])
        for line in gcp_config_list_out.splitlines():
            project_match = _PROJECT_RE.match(line)
            if project_match:
                project = project_match.group("project")
                project_changed = (self.project != project)
                self.project = project
                continue
            zone_match = _ZONE_RE.match(line)
            if zone_match:
                self.zone = zone_match.group("zone")
                continue
        UpdateConfigFile(self.config_path, "project", self.project)
        UpdateConfigFile(self.config_path, "zone", self.zone)
        return project_changed

    def _SetupClientIDSecret(self):
        """Setup Client ID / Client Secret in config file.

        User can use input new values for Client ID and Client Secret.
        """
        print("Please generate a new client ID/secret by following the instructions here:")
        print("https://support.google.com/cloud/answer/6158849?hl=en")
        # TODO: Create markdown readme instructions since the link isn't too helpful.
        self.client_id = None
        self.client_secret = None
        while _InputIsEmpty(self.client_id):
            self.client_id = str(raw_input("Enter Client ID: ").strip())
        while _InputIsEmpty(self.client_secret):
            self.client_secret = str(raw_input("Enter Client Secret: ").strip())
        UpdateConfigFile(self.config_path, "client_id", self.client_id)
        UpdateConfigFile(self.config_path, "client_secret", self.client_secret)

    def _CheckBillingEnable(self, gcloud_runner):
        """Check billing enabled in gcp project.

        The billing info get by gcloud alpha command. Here is one example:
        $ gcloud alpha billing projects describe project_name
            billingAccountName: billingAccounts/011BXX-A30XXX-9XXXX
            billingEnabled: true
            name: projects/project_name/billingInfo
            projectId: project_name

        Args:
            gcloud_runner: A GcloudRunner class to run "gcloud" command.

        Raises:
            NoBillingError: gcp project doesn't enable billing account.
        """
        billing_info = gcloud_runner.RunGcloud(
            ["alpha", "billing", "projects", "describe", self.project])
        if _BILLING_ENABLE_MSG not in billing_info:
            raise errors.NoBillingError(
                "Please set billing account to project(%s) by following the "
                "instructions here: "
                "https://cloud.google.com/billing/docs/how-to/modify-project"
                % self.project)

    def _SetupStorageBucket(self, gcloud_runner):
        """Setup storage_bucket_name in config file.

        We handle the following cases:
            1. Bucket set in the config && bucket is valid.
                - Configure the bucket.
            2. Bucket set in the config && bucket is invalid.
                - Create a default acloud bucket and configure it
            3. Bucket is not set in the config.
                - Create a default acloud bucket and configure it.

        Args:
            gcloud_runner: A GcloudRunner class to run "gsutil" command.
        """
        if (not self.storage_bucket_name
                or not self._BucketIsValid(self.storage_bucket_name, gcloud_runner)):
            self.storage_bucket_name = self._CreateDefaultBucket(gcloud_runner)
        self._ConfigureBucket(gcloud_runner)
        UpdateConfigFile(self.config_path, "storage_bucket_name",
                         self.storage_bucket_name)
        logger.info("Storage bucket name set to [%s]", self.storage_bucket_name)

    def _ConfigureBucket(self, gcloud_runner):
        """Setup write access right for Android Build service account.

        To avoid confuse user, we don't show messages for processing messages.
        e.g. "No changes to gs://acloud-bucket/"

        Args:
            gcloud_runner: A GcloudRunner class to run "gsutil" command.
        """
        gcloud_runner.RunGsutil([
            "acl", "ch", "-u",
            "%s:W" % (_BUILD_SERVICE_ACCOUNT),
            "%s" % (_BUCKET_HEADER + self.storage_bucket_name)
        ], stderr=subprocess.STDOUT)

    def _BucketIsValid(self, bucket_name, gcloud_runner):
        """Check bucket is valid or not.

        If bucket exists and region is in default region,
        then this bucket is valid.

        Args:
            bucket_name: String, name of storage bucket.
            gcloud_runner: A GcloudRunner class to run "gsutil" command.

        Returns:
            Boolean: True if bucket is valid, otherwise False.
        """
        return (self._BucketExists(bucket_name, gcloud_runner) and
                self._BucketInDefaultRegion(bucket_name, gcloud_runner))

    def _CreateDefaultBucket(self, gcloud_runner):
        """Setup bucket to default bucket name.

        Default bucket name is "acloud-{project}".
        If default bucket exist and its region is not "US",
        then default bucket name is changed as "acloud-{project}-us"
        If default bucket didn't exist, tool will create it.

        Args:
            gcloud_runner: A GcloudRunner class to run "gsutil" command.

        Returns:
            String: string of bucket name.
        """
        bucket_name = self._GenerateBucketName(self.project)
        if (self._BucketExists(bucket_name, gcloud_runner) and
                not self._BucketInDefaultRegion(bucket_name, gcloud_runner)):
            bucket_name += ("-" + _DEFAULT_BUCKET_REGION.lower())
        if not self._BucketExists(bucket_name, gcloud_runner):
            self._CreateBucket(bucket_name, gcloud_runner)
        return bucket_name

    @staticmethod
    def _GenerateBucketName(project_name):
        """Generate GCS bucket name that meets the naming guidelines.

        Naming guidelines: https://cloud.google.com/storage/docs/naming
        1. Filter out organization name.
        2. Filter out illegal characters.
        3. Length limit.
        4. Name must end with a number or letter.

        Args:
            project_name: String, name of project.

        Returns:
            String: GCS bucket name compliant with naming guidelines.
        """
        # Sanitize the project name by filtering out the org name (e.g.
        # AOSP:fake_project -> fake_project)
        if _PROJECT_SEPARATOR in project_name:
            _, project_name = project_name.split(_PROJECT_SEPARATOR)

        bucket_name = "%s-%s" % (_DEFAULT_BUCKET_HEADER, project_name)

        # Rule 1: A bucket name can contain lowercase alphanumeric characters,
        # hyphens, and underscores.
        bucket_name = re.sub("[^a-zA-Z_/-]+", "", bucket_name).lower()

        # Rule 2: Bucket names must limit to 63 characters.
        if len(bucket_name) > _BUCKET_LENGTH_LIMIT:
            bucket_name = bucket_name[:_BUCKET_LENGTH_LIMIT]

        # Rule 3: Bucket names must end with a letter, strip out any ending
        # "-" or "_" at the end of the name.
        bucket_name = bucket_name.rstrip(_INVALID_BUCKET_NAME_END_CHARS)

        return bucket_name

    @staticmethod
    def _BucketExists(bucket_name, gcloud_runner):
        """Confirm bucket exist in project or not.

        Args:
            bucket_name: String, name of storage bucket.
            gcloud_runner: A GcloudRunner class to run "gsutil" command.

        Returns:
            Boolean: True for bucket exist in project.
        """
        output = gcloud_runner.RunGsutil(["list"])
        for output_line in output.splitlines():
            match = _BUCKET_RE.match(output_line)
            if match.group("bucket") == bucket_name:
                return True
        return False

    @staticmethod
    def _BucketInDefaultRegion(bucket_name, gcloud_runner):
        """Confirm bucket region settings is "US" or not.

        Args:
            bucket_name: String, name of storage bucket.
            gcloud_runner: A GcloudRunner class to run "gsutil" command.

        Returns:
            Boolean: True for bucket region is in default region.

        Raises:
            errors.SetupError: For parsing bucket region information error.
        """
        output = gcloud_runner.RunGsutil(
            ["ls", "-L", "-b", "%s" % (_BUCKET_HEADER + bucket_name)])
        for region_line in output.splitlines():
            region_match = _BUCKET_REGION_RE.match(region_line.strip())
            if region_match:
                region = region_match.group("region").strip()
                logger.info("Bucket[%s] is in %s (checking for %s)", bucket_name,
                            region, _DEFAULT_BUCKET_REGION)
                if region == _DEFAULT_BUCKET_REGION:
                    return True
                return False
        raise errors.ParseBucketRegionError("Could not determine bucket region.")

    @staticmethod
    def _CreateBucket(bucket_name, gcloud_runner):
        """Create new storage bucket in project.

        Args:
            bucket_name: String, name of storage bucket.
            gcloud_runner: A GcloudRunner class to run "gsutil" command.
        """
        gcloud_runner.RunGsutil(["mb", "%s" % (_BUCKET_HEADER + bucket_name)])
        logger.info("Create bucket [%s].", bucket_name)

    @staticmethod
    def _EnableGcloudServices(gcloud_runner):
        """Enable 3 Gcloud API services.

        1. Android build service
        2. Compute engine service
        3. Google cloud storage service
        To avoid confuse user, we don't show messages for services processing
        messages. e.g. "Waiting for async operation operations ...."

        Args:
            gcloud_runner: A GcloudRunner class to run "gcloud" command.
        """
        for service in _GOOGLE_APIS:
            gcloud_runner.RunGcloud(["services", "enable", service],
                                    stderr=subprocess.STDOUT)