Greg Hartman | c8d55bd | 2018-07-11 16:48:46 -0700 | [diff] [blame] | 1 | #!/bin/bash |
| 2 | |
| 3 | # Common code to build a host image on GCE |
| 4 | |
| 5 | # INTERNAL_extra_source may be set to a directory containing the source for |
| 6 | # extra package to build. |
| 7 | |
Greg Hartman | 9425801 | 2019-03-21 20:36:12 -0700 | [diff] [blame] | 8 | # INTERNAL_IP can be set to --internal-ip run on a GCE instance |
| 9 | # The instance will need --scope compute-rw |
| 10 | |
Greg Hartman | c8d55bd | 2018-07-11 16:48:46 -0700 | [diff] [blame] | 11 | source "${ANDROID_BUILD_TOP}/external/shflags/src/shflags" |
| 12 | |
| 13 | DEFINE_string build_instance \ |
| 14 | "${USER}-build" "Instance name to create for the build" "i" |
Greg Hartman | ce71dc1 | 2019-03-29 17:31:25 -0700 | [diff] [blame] | 15 | DEFINE_string build_project "$(gcloud config get-value project)" \ |
| 16 | "Project to use for scratch" |
| 17 | DEFINE_string build_zone "$(gcloud config get-value compute/zone)" \ |
| 18 | "Zone to use for scratch resources" |
Greg Hartman | c8d55bd | 2018-07-11 16:48:46 -0700 | [diff] [blame] | 19 | DEFINE_string dest_image "vsoc-host-scratch-${USER}" "Image to create" "o" |
| 20 | DEFINE_string dest_family "" "Image family to add the image to" "f" |
Greg Hartman | ce71dc1 | 2019-03-29 17:31:25 -0700 | [diff] [blame] | 21 | DEFINE_string dest_project "$(gcloud config get-value project)" \ |
| 22 | "Project to use for the new image" "p" |
Greg Hartman | c8d55bd | 2018-07-11 16:48:46 -0700 | [diff] [blame] | 23 | DEFINE_string launch_instance "" \ |
| 24 | "Name of the instance to launch with the new image" "l" |
Jason Macnak | 3cedcc5 | 2019-12-05 09:31:54 -0800 | [diff] [blame] | 25 | DEFINE_string source_image_family "debian-10" \ |
| 26 | "Image familty to use as the base" "s" |
Greg Hartman | c8d55bd | 2018-07-11 16:48:46 -0700 | [diff] [blame] | 27 | DEFINE_string source_image_project debian-cloud \ |
| 28 | "Project holding the base image" "m" |
| 29 | DEFINE_string repository_url \ |
Jason Macnak | 3cedcc5 | 2019-12-05 09:31:54 -0800 | [diff] [blame] | 30 | "https://github.com/google/android-cuttlefish.git" \ |
Greg Hartman | c8d55bd | 2018-07-11 16:48:46 -0700 | [diff] [blame] | 31 | "URL to the repository with host changes" "u" |
| 32 | DEFINE_string repository_branch master \ |
| 33 | "Branch to check out" "b" |
Greg Hartman | 20580ae | 2018-09-28 18:26:59 -0700 | [diff] [blame] | 34 | DEFINE_string variant master \ |
| 35 | "Variant to build: generally master or stable" |
Greg Hartman | c8d55bd | 2018-07-11 16:48:46 -0700 | [diff] [blame] | 36 | |
Greg Hartman | ce71dc1 | 2019-03-29 17:31:25 -0700 | [diff] [blame] | 37 | |
Greg Hartman | 9425801 | 2019-03-21 20:36:12 -0700 | [diff] [blame] | 38 | SSH_FLAGS=(${INTERNAL_IP}) |
| 39 | |
Jason Macnak | ed76aed | 2020-01-21 16:19:34 -0800 | [diff] [blame] | 40 | fatal_echo() { |
| 41 | echo "$1" |
| 42 | exit 1 |
| 43 | } |
| 44 | |
Greg Hartman | c8d55bd | 2018-07-11 16:48:46 -0700 | [diff] [blame] | 45 | wait_for_instance() { |
| 46 | alive="" |
| 47 | while [[ -z "${alive}" ]]; do |
| 48 | sleep 5 |
Greg Hartman | 9425801 | 2019-03-21 20:36:12 -0700 | [diff] [blame] | 49 | alive="$(gcloud compute ssh "${SSH_FLAGS[@]}" "$@" -- uptime || true)" |
Greg Hartman | c8d55bd | 2018-07-11 16:48:46 -0700 | [diff] [blame] | 50 | done |
| 51 | } |
| 52 | |
| 53 | package_source() { |
| 54 | local url="$1" |
| 55 | local branch="$2" |
| 56 | local repository_dir="${url/*\//}" |
| 57 | local debian_dir="$(basename "${repository_dir}" .git)" |
| 58 | if [[ $# -eq 4 ]]; then |
| 59 | debian_dir="${repository_dir}/$4" |
| 60 | fi |
| 61 | git clone "${url}" -b "${branch}" |
| 62 | dpkg-source -b "${debian_dir}" |
| 63 | rm -rf "${debian_dir}" |
| 64 | } |
| 65 | |
| 66 | main() { |
| 67 | set -o errexit |
| 68 | set -x |
Greg Hartman | ce71dc1 | 2019-03-29 17:31:25 -0700 | [diff] [blame] | 69 | PZ=(--project=${FLAGS_build_project} --zone=${FLAGS_build_zone}) |
Greg Hartman | c8d55bd | 2018-07-11 16:48:46 -0700 | [diff] [blame] | 70 | if [[ -n "${FLAGS_dest_family}" ]]; then |
| 71 | dest_family_flag=("--family=${FLAGS_dest_family}") |
| 72 | else |
| 73 | dest_family_flag=() |
| 74 | fi |
| 75 | scratch_dir="$(mktemp -d)" |
| 76 | pushd "${scratch_dir}" |
| 77 | package_source "${FLAGS_repository_url}" "${FLAGS_repository_branch}" \ |
| 78 | "cuttlefish-common_${FLAGS_version}" |
| 79 | popd |
Greg Hartman | 20580ae | 2018-09-28 18:26:59 -0700 | [diff] [blame] | 80 | source_files=( |
Ram Muthiah | 6fbd6c8 | 2019-12-10 15:58:27 -0800 | [diff] [blame] | 81 | "${ANDROID_BUILD_TOP}/device/google/cuttlefish/tools/create_base_image_gce.sh" |
Greg Hartman | 20580ae | 2018-09-28 18:26:59 -0700 | [diff] [blame] | 82 | ${scratch_dir}/* |
| 83 | ) |
Greg Hartman | c8d55bd | 2018-07-11 16:48:46 -0700 | [diff] [blame] | 84 | if [[ -n "${INTERNAL_extra_source}" ]]; then |
Greg Hartman | 20580ae | 2018-09-28 18:26:59 -0700 | [diff] [blame] | 85 | source_files+=("${INTERNAL_extra_source}"/*) |
Greg Hartman | c8d55bd | 2018-07-11 16:48:46 -0700 | [diff] [blame] | 86 | fi |
| 87 | |
| 88 | delete_instances=("${FLAGS_build_instance}" "${FLAGS_dest_image}") |
| 89 | if [[ -n "${FLAGS_launch_instance}" ]]; then |
| 90 | delete_instances+=("${FLAGS_launch_instance}") |
| 91 | fi |
| 92 | gcloud compute instances delete -q \ |
Greg Hartman | ce71dc1 | 2019-03-29 17:31:25 -0700 | [diff] [blame] | 93 | "${PZ[@]}" "${delete_instances[@]}" || \ |
Greg Hartman | c8d55bd | 2018-07-11 16:48:46 -0700 | [diff] [blame] | 94 | echo Not running |
| 95 | gcloud compute disks delete -q \ |
Greg Hartman | ce71dc1 | 2019-03-29 17:31:25 -0700 | [diff] [blame] | 96 | "${PZ[@]}" "${FLAGS_dest_image}" || echo No scratch disk |
Greg Hartman | c8d55bd | 2018-07-11 16:48:46 -0700 | [diff] [blame] | 97 | gcloud compute images delete -q \ |
Greg Hartman | ce71dc1 | 2019-03-29 17:31:25 -0700 | [diff] [blame] | 98 | --project="${FLAGS_build_project}" "${FLAGS_dest_image}" || echo Not respinning |
Greg Hartman | c8d55bd | 2018-07-11 16:48:46 -0700 | [diff] [blame] | 99 | gcloud compute disks create \ |
Greg Hartman | ce71dc1 | 2019-03-29 17:31:25 -0700 | [diff] [blame] | 100 | "${PZ[@]}" \ |
Greg Hartman | c8d55bd | 2018-07-11 16:48:46 -0700 | [diff] [blame] | 101 | --image-family="${FLAGS_source_image_family}" \ |
| 102 | --image-project="${FLAGS_source_image_project}" \ |
| 103 | "${FLAGS_dest_image}" |
Jason Macnak | ed76aed | 2020-01-21 16:19:34 -0800 | [diff] [blame] | 104 | local gpu_type="nvidia-tesla-p100-vws" |
| 105 | gcloud compute accelerator-types describe "${gpu_type}" "${PZ[@]}" || \ |
| 106 | fatal_echo "Please use a zone with ${gpu_type} GPUs available." |
Greg Hartman | c8d55bd | 2018-07-11 16:48:46 -0700 | [diff] [blame] | 107 | gcloud compute instances create \ |
Greg Hartman | ce71dc1 | 2019-03-29 17:31:25 -0700 | [diff] [blame] | 108 | "${PZ[@]}" \ |
Greg Hartman | 20580ae | 2018-09-28 18:26:59 -0700 | [diff] [blame] | 109 | --machine-type=n1-standard-16 \ |
Greg Hartman | c8d55bd | 2018-07-11 16:48:46 -0700 | [diff] [blame] | 110 | --image-family="${FLAGS_source_image_family}" \ |
| 111 | --image-project="${FLAGS_source_image_project}" \ |
Greg Hartman | 20580ae | 2018-09-28 18:26:59 -0700 | [diff] [blame] | 112 | --boot-disk-size=200GiB \ |
Jason Macnak | ed76aed | 2020-01-21 16:19:34 -0800 | [diff] [blame] | 113 | --accelerator="type=${gpu_type},count=1" \ |
Jason Macnak | 3cedcc5 | 2019-12-05 09:31:54 -0800 | [diff] [blame] | 114 | --maintenance-policy=TERMINATE \ |
Greg Hartman | c8d55bd | 2018-07-11 16:48:46 -0700 | [diff] [blame] | 115 | "${FLAGS_build_instance}" |
Greg Hartman | ce71dc1 | 2019-03-29 17:31:25 -0700 | [diff] [blame] | 116 | wait_for_instance "${PZ[@]}" "${FLAGS_build_instance}" |
Greg Hartman | c8d55bd | 2018-07-11 16:48:46 -0700 | [diff] [blame] | 117 | # Ubuntu tends to mount the wrong disk as root, so help it by waiting until |
| 118 | # it has booted before giving it access to the clean image disk |
| 119 | gcloud compute instances attach-disk \ |
Greg Hartman | ce71dc1 | 2019-03-29 17:31:25 -0700 | [diff] [blame] | 120 | "${PZ[@]}" \ |
Greg Hartman | c8d55bd | 2018-07-11 16:48:46 -0700 | [diff] [blame] | 121 | "${FLAGS_build_instance}" --disk="${FLAGS_dest_image}" |
Greg Hartman | 9425801 | 2019-03-21 20:36:12 -0700 | [diff] [blame] | 122 | # beta for the --internal-ip flag that may be passed via SSH_FLAGS |
Greg Hartman | ce71dc1 | 2019-03-29 17:31:25 -0700 | [diff] [blame] | 123 | gcloud beta compute scp "${SSH_FLAGS[@]}" "${PZ[@]}" \ |
Greg Hartman | c8d55bd | 2018-07-11 16:48:46 -0700 | [diff] [blame] | 124 | "${source_files[@]}" \ |
Greg Hartman | c8d55bd | 2018-07-11 16:48:46 -0700 | [diff] [blame] | 125 | "${FLAGS_build_instance}:" |
Greg Hartman | 9425801 | 2019-03-21 20:36:12 -0700 | [diff] [blame] | 126 | gcloud compute ssh "${SSH_FLAGS[@]}" \ |
Greg Hartman | ce71dc1 | 2019-03-29 17:31:25 -0700 | [diff] [blame] | 127 | "${PZ[@]}" "${FLAGS_build_instance}" -- \ |
Greg Hartman | c8d55bd | 2018-07-11 16:48:46 -0700 | [diff] [blame] | 128 | ./create_base_image_gce.sh |
| 129 | gcloud compute instances delete -q \ |
Greg Hartman | ce71dc1 | 2019-03-29 17:31:25 -0700 | [diff] [blame] | 130 | "${PZ[@]}" "${FLAGS_build_instance}" |
| 131 | gcloud compute images create \ |
| 132 | --project="${FLAGS_build_project}" \ |
Greg Hartman | c8d55bd | 2018-07-11 16:48:46 -0700 | [diff] [blame] | 133 | --source-disk="${FLAGS_dest_image}" \ |
Greg Hartman | ce71dc1 | 2019-03-29 17:31:25 -0700 | [diff] [blame] | 134 | --source-disk-zone="${FLAGS_build_zone}" \ |
Greg Hartman | c8d55bd | 2018-07-11 16:48:46 -0700 | [diff] [blame] | 135 | --licenses=https://www.googleapis.com/compute/v1/projects/vm-options/global/licenses/enable-vmx \ |
| 136 | "${dest_family_flag[@]}" \ |
| 137 | "${FLAGS_dest_image}" |
Greg Hartman | ce71dc1 | 2019-03-29 17:31:25 -0700 | [diff] [blame] | 138 | gcloud compute disks delete -q "${PZ[@]}" \ |
Greg Hartman | c8d55bd | 2018-07-11 16:48:46 -0700 | [diff] [blame] | 139 | "${FLAGS_dest_image}" |
| 140 | if [[ -n "${FLAGS_launch_instance}" ]]; then |
Greg Hartman | ce71dc1 | 2019-03-29 17:31:25 -0700 | [diff] [blame] | 141 | gcloud compute instances create "${PZ[@]}" \ |
| 142 | --image-project="${FLAGS_build_project}" \ |
Greg Hartman | c8d55bd | 2018-07-11 16:48:46 -0700 | [diff] [blame] | 143 | --image="${FLAGS_dest_image}" \ |
Greg Hartman | 20580ae | 2018-09-28 18:26:59 -0700 | [diff] [blame] | 144 | --machine-type=n1-standard-4 \ |
Greg Hartman | c8d55bd | 2018-07-11 16:48:46 -0700 | [diff] [blame] | 145 | --scopes storage-ro \ |
Jason Macnak | ed76aed | 2020-01-21 16:19:34 -0800 | [diff] [blame] | 146 | --accelerator="type=${gpu_type},count=1" \ |
Jason Macnak | 3cedcc5 | 2019-12-05 09:31:54 -0800 | [diff] [blame] | 147 | --maintenance-policy=TERMINATE \ |
Greg Hartman | c8d55bd | 2018-07-11 16:48:46 -0700 | [diff] [blame] | 148 | "${FLAGS_launch_instance}" |
| 149 | fi |
Greg Hartman | ce71dc1 | 2019-03-29 17:31:25 -0700 | [diff] [blame] | 150 | cat <<EOF |
| 151 | echo Test and if this looks good, consider releasing it via: |
| 152 | |
| 153 | gcloud compute images create \ |
| 154 | --project="${FLAGS_dest_project}" \ |
| 155 | --source-image="${FLAGS_dest_image}" \ |
| 156 | --source-image-project="${FLAGS_build_project}" \ |
| 157 | "${dest_family_flag[@]}" \ |
| 158 | "${FLAGS_dest_image}" |
| 159 | EOF |
Greg Hartman | c8d55bd | 2018-07-11 16:48:46 -0700 | [diff] [blame] | 160 | } |