Igor Murashkin | 2553887 | 2017-02-28 16:38:58 -0800 | [diff] [blame] | 1 | #!/bin/bash |
| 2 | # |
| 3 | # Copyright (C) 2017 The Android Open Source Project |
| 4 | # |
| 5 | # Licensed under the Apache License, Version 2.0 (the "License"); |
| 6 | # you may not use this file except in compliance with the License. |
| 7 | # You may obtain a copy of the License at |
| 8 | # |
| 9 | # http://www.apache.org/licenses/LICENSE-2.0 |
| 10 | # |
| 11 | # Unless required by applicable law or agreed to in writing, software |
| 12 | # distributed under the License is distributed on an "AS IS" BASIS, |
| 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 14 | # See the License for the specific language governing permissions and |
| 15 | # limitations under the License. |
| 16 | |
| 17 | if [[ ! -d art ]]; then |
| 18 | echo "Script needs to be run at the root of the android tree" |
| 19 | exit 1 |
| 20 | fi |
| 21 | |
| 22 | ALL_CONFIGS=(linux-ia32 linux-x64 linux-armv8 linux-armv7 android-armv8 android-armv7) |
| 23 | |
| 24 | usage() { |
| 25 | local config |
| 26 | local golem_target |
| 27 | |
| 28 | (cat << EOF |
| 29 | Usage: $(basename "${BASH_SOURCE[0]}") [--golem=<target>] --machine-type=MACHINE_TYPE |
| 30 | [--tarball[=<target>.tar.gz]] |
| 31 | |
| 32 | Build minimal art binaries required to run golem benchmarks either |
| 33 | locally or on the golem servers. |
| 34 | |
| 35 | Creates the \$MACHINE_TYPE binaries in your \$OUT_DIR, and if --tarball was specified, |
| 36 | it also tars the results of the build together into your <target.tar.gz> file. |
| 37 | -------------------------------------------------------- |
| 38 | Required Flags: |
| 39 | --machine-type=MT Specify the machine type that will be built. |
| 40 | |
| 41 | Optional Flags": |
| 42 | --golem=<target> Builds with identical commands that Golem servers use. |
| 43 | --tarball[=o.tgz] Tar/gz the results. File name defaults to <machine_type>.tar.gz |
| 44 | -j<num> Specify how many jobs to use for parallelism. |
| 45 | --help Print this help listing. |
| 46 | --showcommands Show commands as they are being executed. |
| 47 | --simulate Print commands only, don't execute commands. |
| 48 | EOF |
| 49 | ) | sed -e 's/^[[:space:]][[:space:]]//g' >&2 # Strip leading whitespace from heredoc. |
| 50 | |
| 51 | echo >&2 "Available machine types:" |
| 52 | for config in "${ALL_CONFIGS[@]}"; do |
| 53 | echo >&2 " $config" |
| 54 | done |
| 55 | |
| 56 | echo >&2 |
| 57 | echo >&2 "Available Golem targets:" |
| 58 | while IFS='' read -r golem_target; do |
| 59 | echo >&2 " $golem_target" |
| 60 | done < <("$(thisdir)/env" --list-targets) |
| 61 | } |
| 62 | |
| 63 | # Check if $1 element is in array $2 |
| 64 | contains_element() { |
| 65 | local e |
| 66 | for e in "${@:2}"; do [[ "$e" == "$1" ]] && return 0; done |
| 67 | return 1 |
| 68 | } |
| 69 | |
| 70 | # Display a command, but don't execute it, if --showcommands was set. |
| 71 | show_command() { |
| 72 | if [[ $showcommands == "showcommands" ]]; then |
| 73 | echo "$@" |
| 74 | fi |
| 75 | } |
| 76 | |
| 77 | # Execute a command, displaying it if --showcommands was set. |
| 78 | # If --simulate is used, command is not executed. |
| 79 | execute() { |
| 80 | show_command "$@" |
| 81 | execute_noshow "$@" |
| 82 | } |
| 83 | |
| 84 | # Execute a command unless --simulate was used. |
| 85 | execute_noshow() { |
| 86 | if [[ $simulate == "simulate" ]]; then |
| 87 | return 0 |
| 88 | fi |
| 89 | |
| 90 | local prog="$1" |
| 91 | shift |
| 92 | "$prog" "$@" |
| 93 | } |
| 94 | |
| 95 | # Export environment variable, echoing it to screen. |
| 96 | setenv() { |
| 97 | local name="$1" |
| 98 | local value="$2" |
| 99 | |
| 100 | export $name="$value" |
| 101 | echo export $name="$value" |
| 102 | } |
| 103 | |
| 104 | # Export environment variable, echoing $3 to screen ($3 is meant to be unevaluated). |
| 105 | setenv_escape() { |
| 106 | local name="$1" |
| 107 | local value="$2" |
| 108 | local escaped_value="$3" |
| 109 | |
| 110 | export $name="$value" |
| 111 | echo export $name="$escaped_value" |
| 112 | } |
| 113 | |
| 114 | log_usage_error() { |
| 115 | echo >&2 "ERROR: " "$@" |
| 116 | echo >&2 " See --help for the correct usage information." |
| 117 | exit 1 |
| 118 | } |
| 119 | |
| 120 | log_fatal() { |
| 121 | echo >&2 "FATAL: " "$@" |
| 122 | exit 2 |
| 123 | } |
| 124 | |
| 125 | # Get the directory of this script. |
| 126 | thisdir() { |
| 127 | (\cd "$(dirname "${BASH_SOURCE[0]}")" && pwd ) |
| 128 | } |
| 129 | |
| 130 | # Get the path to the top of the Android source tree. |
| 131 | gettop() { |
| 132 | if [[ "x$ANDROID_BUILD_TOP" != "x" ]]; then |
| 133 | echo "$ANDROID_BUILD_TOP"; |
| 134 | else |
| 135 | echo "$(thisdir)/../../.." |
| 136 | fi |
| 137 | } |
| 138 | |
| 139 | # Get a build variable from the Android build system. |
| 140 | get_build_var() { |
| 141 | local varname="$1" |
| 142 | |
| 143 | # include the desired target product/build-variant |
| 144 | # which won't be set in our env if neither we nor the user first executed |
| 145 | # source build/envsetup.sh (e.g. if simulating from a fresh shell). |
| 146 | local extras |
| 147 | [[ -n $target_product ]] && extras+=" TARGET_PRODUCT=$target_product" |
| 148 | [[ -n $target_build_variant ]] && extras+=" TARGET_BUILD_VARIANT=$target_build_variant" |
| 149 | |
Dan Willemsen | 7b44ac0 | 2017-10-17 18:00:57 -0700 | [diff] [blame] | 150 | # call dumpvar from the build system. |
| 151 | (\cd "$(gettop)"; env $extras build/soong/soong_ui.bash --dumpvar-mode $varname) |
Igor Murashkin | 2553887 | 2017-02-28 16:38:58 -0800 | [diff] [blame] | 152 | } |
| 153 | |
| 154 | # Defaults from command-line. |
| 155 | |
| 156 | mode="" # blank or 'golem' if --golem was specified. |
| 157 | golem_target="" # --golem=$golem_target |
| 158 | config="" # --machine-type=$config |
Dan Willemsen | 7b44ac0 | 2017-10-17 18:00:57 -0700 | [diff] [blame] | 159 | j_arg="" |
Igor Murashkin | 2553887 | 2017-02-28 16:38:58 -0800 | [diff] [blame] | 160 | showcommands="" |
| 161 | simulate="" |
| 162 | make_tarball="" |
| 163 | tarball="" |
| 164 | |
| 165 | # Parse command line arguments |
| 166 | |
| 167 | while [[ "$1" != "" ]]; do |
| 168 | case "$1" in |
| 169 | --help) |
| 170 | usage |
| 171 | exit 1 |
| 172 | ;; |
| 173 | --golem=*) |
| 174 | mode="golem" |
| 175 | golem_target="${1##--golem=}" |
| 176 | |
| 177 | if [[ "x$golem_target" == x ]]; then |
| 178 | log_usage_error "Missing --golem target type." |
| 179 | fi |
| 180 | |
| 181 | shift |
| 182 | ;; |
| 183 | --machine-type=*) |
| 184 | config="${1##--machine-type=}" |
| 185 | if ! contains_element "$config" "${ALL_CONFIGS[@]}"; then |
| 186 | log_usage_error "Invalid --machine-type value '$config'" |
| 187 | fi |
| 188 | shift |
| 189 | ;; |
| 190 | --tarball) |
| 191 | tarball="" # reuse the machine type name. |
| 192 | make_tarball="make_tarball" |
| 193 | shift |
| 194 | ;; |
| 195 | --tarball=*) |
| 196 | tarball="${1##--tarball=}" |
| 197 | make_tarball="make_tarball" |
| 198 | shift |
| 199 | ;; |
| 200 | -j*) |
| 201 | j_arg="$1" |
| 202 | shift |
| 203 | ;; |
| 204 | --showcommands) |
| 205 | showcommands="showcommands" |
| 206 | shift |
| 207 | ;; |
| 208 | --simulate) |
| 209 | simulate="simulate" |
| 210 | shift |
| 211 | ;; |
| 212 | *) |
| 213 | log_usage_error "Unknown options $1" |
| 214 | ;; |
| 215 | esac |
| 216 | done |
| 217 | |
| 218 | ################################### |
| 219 | ################################### |
| 220 | ################################### |
| 221 | |
| 222 | if [[ -z $config ]]; then |
| 223 | log_usage_error "--machine-type option is required." |
| 224 | fi |
| 225 | |
| 226 | # --tarball defaults to the --machine-type value with .tar.gz. |
| 227 | tarball="${tarball:-$config.tar.gz}" |
| 228 | |
| 229 | target_product="$TARGET_PRODUCT" |
| 230 | target_build_variant="$TARGET_BUILD_VARIANT" |
| 231 | |
| 232 | # If not using --golem, use whatever the user had lunch'd prior to this script. |
| 233 | if [[ $mode == "golem" ]]; then |
| 234 | # This section is intended solely to be executed by a golem build server. |
| 235 | |
| 236 | target_build_variant=eng |
| 237 | case "$config" in |
| 238 | *-armv7) |
| 239 | target_product="arm_krait" |
| 240 | ;; |
| 241 | *-armv8) |
| 242 | target_product="armv8" |
| 243 | ;; |
| 244 | *) |
| 245 | target_product="sdk" |
| 246 | ;; |
| 247 | esac |
| 248 | |
| 249 | if [[ $target_product = arm* ]]; then |
| 250 | # If using the regular manifest, e.g. 'master' |
| 251 | # The lunch command for arm will assuredly fail because we don't have device/generic/art. |
| 252 | # |
| 253 | # Print a human-readable error message instead of trying to lunch and failing there. |
| 254 | if ! [[ -d "$(gettop)/device/generic/art" ]]; then |
| 255 | log_fatal "Missing device/generic/art directory. Perhaps try master-art repo manifest?\n" \ |
| 256 | " Cannot build ARM targets (arm_krait, armv8) for Golem." >&2 |
| 257 | fi |
| 258 | # We could try to keep on simulating but it seems brittle because we won't have the proper |
| 259 | # build variables to output the right strings. |
| 260 | fi |
| 261 | |
| 262 | # Get this particular target's environment variables (e.g. ART read barrier on/off). |
| 263 | source "$(thisdir)"/env "$golem_target" || exit 1 |
| 264 | |
| 265 | lunch_target="$target_product-$target_build_variant" |
| 266 | |
| 267 | execute 'source' build/envsetup.sh |
| 268 | # Build generic targets (as opposed to something specific like aosp_angler-eng). |
| 269 | execute lunch "$lunch_target" |
Igor Murashkin | 2553887 | 2017-02-28 16:38:58 -0800 | [diff] [blame] | 270 | # Golem uses master-art repository which is missing a lot of other libraries. |
| 271 | setenv SOONG_ALLOW_MISSING_DEPENDENCIES true |
| 272 | # Golem may be missing tools such as javac from its path. |
| 273 | setenv_escape PATH "/usr/lib/jvm/java-8-openjdk-amd64/bin/:$PATH" '/usr/lib/jvm/java-8-openjdk-amd64/bin/:$PATH' |
| 274 | else |
| 275 | # Look up the default variables from the build system if they weren't set already. |
| 276 | [[ -z $target_product ]] && target_product="$(get_build_var TARGET_PRODUCT)" |
| 277 | [[ -z $target_build_variant ]] && target_build_variant="$(get_build_var TARGET_BUILD_VARIANT)" |
| 278 | fi |
| 279 | |
| 280 | # Defaults for all machine types. |
| 281 | make_target="build-art-target-golem" |
| 282 | out_dir="out/x86_64" |
| 283 | root_dir_var="PRODUCT_OUT" |
| 284 | strip_symbols=false |
| 285 | bit64_suffix="" |
| 286 | tar_directories=(system data/art-test) |
| 287 | |
| 288 | # Per-machine type overrides |
| 289 | if [[ $config == linux-arm* ]]; then |
| 290 | setenv ART_TARGET_LINUX true |
| 291 | fi |
| 292 | |
| 293 | case "$config" in |
| 294 | linux-ia32|linux-x64) |
| 295 | root_dir_var="HOST_OUT" |
| 296 | # Android strips target builds automatically, but not host builds. |
| 297 | strip_symbols=true |
| 298 | make_target="build-art-host-golem" |
| 299 | |
| 300 | if [[ $config == linux-ia32 ]]; then |
| 301 | out_dir="out/x86" |
| 302 | setenv HOST_PREFER_32_BIT true |
| 303 | else |
| 304 | bit64_suffix="64" |
| 305 | fi |
| 306 | |
| 307 | tar_directories=(bin framework usr lib${bit64_suffix}) |
| 308 | ;; |
| 309 | *-armv8) |
| 310 | bit64_suffix="64" |
| 311 | ;; |
| 312 | *-armv7) |
| 313 | ;; |
| 314 | *) |
| 315 | log_fatal "Unsupported machine-type '$config'" |
| 316 | esac |
| 317 | |
| 318 | # Golem benchmark run commands expect a certain $OUT_DIR to be set, |
| 319 | # so specify it here. |
| 320 | # |
| 321 | # Note: It is questionable if we want to customize this since users |
| 322 | # could alternatively probably use their own build directly (and forgo this script). |
| 323 | setenv OUT_DIR "$out_dir" |
| 324 | root_dir="$(get_build_var "$root_dir_var")" |
| 325 | |
| 326 | if [[ $mode == "golem" ]]; then |
| 327 | # For golem-style running only. |
| 328 | # Sets the DT_INTERP to this path in every .so we can run the |
| 329 | # non-system version of dalvikvm with our own copies of the dependencies (e.g. our own libc++). |
| 330 | if [[ $config == android-* ]]; then |
| 331 | # TODO: the linker can be relative to the binaries |
| 332 | # (which is what we do for linux-armv8 and linux-armv7) |
| 333 | golem_run_path="/data/local/tmp/runner/" |
| 334 | else |
| 335 | golem_run_path="" |
| 336 | fi |
| 337 | |
| 338 | # Only do this for target builds. Host doesn't need this. |
| 339 | if [[ $config == *-arm* ]]; then |
| 340 | setenv CUSTOM_TARGET_LINKER "${golem_run_path}${root_dir}/system/bin/linker${bit64_suffix}" |
| 341 | fi |
| 342 | fi |
| 343 | |
| 344 | # |
| 345 | # Main command execution below here. |
| 346 | # (everything prior to this just sets up environment variables, |
| 347 | # and maybe calls lunch). |
| 348 | # |
| 349 | |
Dan Willemsen | 7b44ac0 | 2017-10-17 18:00:57 -0700 | [diff] [blame] | 350 | execute build/soong/soong_ui.bash --make-mode "${j_arg}" "${make_target}" |
Igor Murashkin | 2553887 | 2017-02-28 16:38:58 -0800 | [diff] [blame] | 351 | |
| 352 | if $strip_symbols; then |
| 353 | # Further reduce size by stripping symbols. |
| 354 | execute_noshow strip $root_dir/bin/* || true |
| 355 | show_command strip $root_dir/bin/'*' '|| true' |
| 356 | execute_noshow strip $root_dir/lib${bit64_suffix}/'*' |
| 357 | show_command strip $root_dir/lib${bit64_suffix}/'*' |
| 358 | fi |
| 359 | |
| 360 | if [[ "$make_tarball" == "make_tarball" ]]; then |
| 361 | # Create a tarball which is required for the golem build resource. |
| 362 | # (In particular, each golem benchmark's run commands depend on a list of resource files |
| 363 | # in order to have all the files it needs to actually execute, |
| 364 | # and this tarball would satisfy that particular target+machine-type's requirements). |
| 365 | dirs_rooted=() |
| 366 | for tar_dir in "${tar_directories[@]}"; do |
| 367 | dirs_rooted+=("$root_dir/$tar_dir") |
| 368 | done |
| 369 | |
Orion Hodson | e477412 | 2018-08-24 17:02:08 +0100 | [diff] [blame] | 370 | execute tar -czf "${tarball}" --exclude ".git" --exclude ".gitignore" "${dirs_rooted[@]}" |
Igor Murashkin | 2553887 | 2017-02-28 16:38:58 -0800 | [diff] [blame] | 371 | tar_result=$? |
| 372 | if [[ $tar_result -ne 0 ]]; then |
| 373 | [[ -f $tarball ]] && rm $tarball |
| 374 | fi |
| 375 | |
| 376 | show_command '[[ $? -ne 0 ]] && rm' "$tarball" |
| 377 | fi |
| 378 | |