bootstat: introduce sys.boot.reason

Adding functionality to bootstat --record_boot_complete and
--record_boot_reason to initialize sys.boot.reason, the canonical
system boot reason.

Filter out ro.boot.bootreason oem noise into sys.boot.reason.  Add
heuristics to determine what the boot reason is, when otherwise would
be defaulting to the blunt and relatively devoid of detail catch-all
reboot reasons ("reboot", "shutdown", "cold", "warm", "hard").

boot_reason_test.sh is also a compliance test.

Test: boot_reason_test.sh all
Bug: 63736262
Change-Id: Ic9a42cccbcfc89a5c0e081ba66d577a97c8c8c76
diff --git a/bootstat/boot_reason_test.sh b/bootstat/boot_reason_test.sh
new file mode 100755
index 0000000..7b0b0da
--- /dev/null
+++ b/bootstat/boot_reason_test.sh
@@ -0,0 +1,689 @@
+#! /bin/bash
+#
+# Bootstat boot reason tests
+#
+# throughout testing:
+# - manual tests can only run on eng/userdebug builds
+# - watch adb logcat -b all -d -s bootstat
+# - watch adb logcat -b all -d | audit2allow
+# - wait until screen is up, boot has completed, can mean wait for
+#   sys.boot_completed=1 and sys.logbootcomplete=1 to be true
+#
+# All test frames, and nothing else, must be function names prefixed and
+# specifiged with the pattern 'test_<test>() {' as this is also how the
+# script discovers the full list of tests by inspecting its own code.
+#
+
+# Helper variables
+
+SPACE=" "
+ESCAPE=""
+TAB="	"
+GREEN="${ESCAPE}[38;5;40m"
+RED="${ESCAPE}[38;5;196m"
+NORMAL="${ESCAPE}[0m"
+
+# Helper functions
+
+[ "USAGE: inFastboot
+
+Returns: true if device is in fastboot mode" ]
+inFastboot() {
+  fastboot devices | grep "^${ANDROID_SERIAL}[${SPACE}${TAB}]" > /dev/null
+}
+
+[ "USAGE: format_duration <seconds>
+
+human readable output whole seconds, whole minutes or mm:ss" ]
+format_duration() {
+  if [ -z "${1}" ]; then
+    echo unknown
+    return
+  fi
+  seconds=`expr ${1} % 60`
+  minutes=`expr ${1} / 60`
+  if [ 0 -eq ${minutes} ]; then
+    if [ 1 -eq ${1} ]; then
+      echo 1 second
+      return
+    fi
+    echo ${1} seconds
+    return
+  elif [ 60 -eq ${1} ]; then
+    echo 1 minute
+    return
+  elif [ 0 -eq ${seconds} ]; then
+    echo ${minutes} minutes
+    return
+  fi
+  echo ${minutes}:`expr ${seconds} / 10``expr ${seconds} % 10`
+}
+
+wait_for_screen_timeout=900
+[ "USAGE: wait_for_screen [-n] [TIMEOUT]
+
+-n - echo newline at exit
+TIMEOUT - default `format_duration ${wait_for_screen_timeout}`" ]
+wait_for_screen() {
+  exit_function=true
+  if [ X"-n" = X"${1}" ]; then
+    exit_function=echo
+    shift
+  fi
+  timeout=${wait_for_screen_timeout}
+  if [ ${#} -gt 0 ]; then
+    timeout=${1}
+    shift
+  fi
+  counter=0
+  while true; do
+    [ 0 = ${counter} ] ||
+      adb wait-for-device </dev/null >/dev/null 2>/dev/null
+    vals=`adb shell getprop </dev/null 2>/dev/null |
+          sed -n 's/[[]sys[.]\(boot_completed\|logbootcomplete\)[]]: [[]\([01]\)[]]$/\1=\2/p'`
+    [ 0 = ${counter} ] ||
+      sleep 1
+    if [ "${vals}" = "`echo boot_completed=1 ; echo logbootcomplete=1`" ]; then
+      break
+    fi
+    if [ "${vals}" = "`echo logbootcomplete=1 ; echo boot_completed=1`" ]; then
+      break
+    fi
+    counter=`expr ${counter} + 1`
+    if [ ${counter} -gt ${timeout} ]; then
+      ${exit_function}
+      echo "ERROR: wait_for_screen() timed out (`format_duration ${timeout}`)" >&2
+      return 1
+    fi
+  done
+  ${exit_function}
+}
+
+[ "USAGE: EXPECT_EQ <lval> <rval> [message]
+
+Returns true if (regex) lval matches rval" ]
+EXPECT_EQ() {
+  lval="${1}"
+  rval="${2}"
+  shift 2
+  if ! ( echo X"${rval}" | grep '^X'"${lval}"'$' >/dev/null 2>/dev/null ); then
+    echo "ERROR: expected \"${lval}\" got \"${rval}\"" >&2
+    if [ -n "${*}" ] ; then
+      echo "       ${*}" >&2
+    fi
+    return 1
+  fi
+  if [ -n "${*}" ] ; then
+    if [ X"${lval}" != X"${rval}" ]; then
+      echo "INFO: ok \"${lval}\"(=\"${rval}\") ${*}" >&2
+    else
+      echo "INFO: ok \"${lval}\" ${*}" >&2
+    fi
+  fi
+  return 0
+}
+
+[ "USAGE: EXPECT_PROPERTY <prop> <value>
+
+Returns true if current return (regex) value is true and the result matches" ]
+EXPECT_PROPERTY() {
+  save_ret=${?}
+  property="${1}"
+  value="${2}"
+  shift 2
+  val=`adb shell getprop ${property} 2>&1`
+  EXPECT_EQ "${value}" "${val}" for Android property ${property} ||
+    save_ret=${?}
+  return ${save_ret}
+}
+
+[ "USAGE: report_bootstat_logs <expected> ...
+
+if not prefixed with a minus (-), <expected> will become a series of expected
+matches:
+
+    bootstat: Canonical boot reason: <expected_property_value>
+
+If prefixed with a minus, <expected> will look for an exact match after
+removing the minux prefix.  All expected content is _dropped_ from the output
+and in essence forms a known blacklist, unexpected content will show.
+
+Report any logs, minus a known blacklist, preserve the current exit status" ]
+report_bootstat_logs() {
+  save_ret=${?}
+  match=
+  for i in ${*}; do
+    if [ X"${i}" != X"${i#-}" ] ; then
+      match="${match}
+${i#-}"
+    else
+      match="${match}
+bootstat: Canonical boot reason: ${i}"
+    fi
+  done
+  adb logcat -b all -d |
+  grep bootstat |
+  grep -v -F "bootstat: Service started: /system/bin/bootstat --record_boot_complete${match}
+bootstat: Failed to read /data/misc/bootstat/post_decrypt_time_elapsed: No such file or directory
+bootstat: Failed to parse boot time record: /data/misc/bootstat/post_decrypt_time_elapsed
+bootstat: Service started: /system/bin/bootstat --record_boot_reason
+bootstat: Service started: /system/bin/bootstat --record_time_since_factory_reset
+bootstat: Service started: /system/bin/bootstat -l
+bootstat: Battery level at shutdown 100%
+bootstat: Battery level at startup 100%
+init    : Parsing file /system/etc/init/bootstat.rc...
+init    : processing action (post-fs-data) from (/system/etc/init/bootstat.rc
+init    : processing action (boot) from (/system/etc/init/bootstat.rc
+init    : processing action (ro.boot.bootreason=*) from (/system/etc/init/bootstat.rc
+init    : processing action (sys.boot_completed=1 && sys.logbootcomplete=1) from (/system/etc/init/bootstat.rc
+init    : Command 'exec - system log -- /system/bin/bootstat --record_boot_complete' action=sys.boot_completed=1 && sys.logbootcomplete=1 (/system/etc/init/bootstat.rc:
+init    : Command 'exec - system log -- /system/bin/bootstat --record_boot_reason' action=sys.boot_completed=1 && sys.logbootcomplete=1 (/system/etc/init/bootstat.rc:
+init    : Command 'exec - system log -- /system/bin/bootstat --record_time_since_factory_reset' action=sys.boot_completed=1 && sys.logbootcomplete=1 (/system/etc/init/bootstat.rc:
+ (/system/bin/bootstat --record_boot_complete)'...
+ (/system/bin/bootstat --record_boot_complete)' (pid${SPACE}
+ (/system/bin/bootstat --record_boot_reason)'...
+ (/system/bin/bootstat --record_boot_reason)' (pid${SPACE}
+ (/system/bin/bootstat --record_time_since_factory_reset)'...
+ (/system/bin/bootstat --record_time_since_factory_reset)' (pid${SPACE}
+ (/system/bin/bootstat -l)'...
+ (/system/bin/bootstat -l)' (pid " |
+  grep -v 'bootstat: Unknown boot reason: $' # Hikey Special
+  return ${save_ret}
+}
+
+[ "USAGE: start_test [message]
+
+Record start of test, preserve exit status" ]
+start_test() {
+  save_ret=${?}
+  START=`date +%s`
+  echo "${GREEN}[ RUN      ]${NORMAL} ${TEST} ${*}"
+  return ${save_ret}
+}
+
+[ "USAGE: end_test [message]
+
+Document duration and success of test, preserve exit status" ]
+end_test() {
+  save_ret=${?}
+  END=`date +%s`
+  duration=`expr ${END} - ${START} 2>/dev/null`
+  [ 0 = ${duration} ] ||
+    echo INFO: ${TEST} test duration `format_duration ${duration}` >&2
+  if [ ${save_ret} = 0 ]; then
+    echo "${GREEN}[       OK ]${NORMAL} ${TEST} ${*}"
+  else
+    echo "${RED}[  FAILED  ]${NORMAL} ${TEST} ${*}"
+  fi
+  return ${save_ret}
+}
+
+[ "USAGE: wrap_test <test> [message]
+
+All tests below are wrapped with this helper" ]
+wrap_test() {
+  if [ -z "${1}" -o X"nothing" = X"${1}" ]; then
+    return
+  fi
+  TEST=${1}
+  shift
+  start_test ${1}
+  eval test_${TEST}
+  end_test ${2}
+}
+
+[ "USAGE: validate_property <property>
+
+Check property for CTS compliance with our expectations. Return a cleansed
+string representing what is acceptable.
+
+NB: must roughly match heuristics in system/core/bootstat/bootstat.cpp" ]
+validate_property() {
+  var=`adb shell getprop ${1} 2>&1`
+  var=`echo -n ${var} |
+       tr '[A-Z]' '[a-z]' |
+       tr ' \f\t\r\n' '_____'`
+  case ${var} in
+    watchdog) ;;
+    watchdog,?*) ;;
+    kernel_panic) ;;
+    kernel_panic,?*) ;;
+    recovery) ;;
+    recovery,?*) ;;
+    bootloader) ;;
+    bootloader,?*) ;;
+    cold) ;;
+    cold,?*) ;;
+    hard) ;;
+    hard,?*) ;;
+    warm) ;;
+    warm,?*) ;;
+    shutdown) ;;
+    shutdown,?*) ;;
+    reboot) ;;
+    reboot,?*) ;;
+    # Aliases
+    *wdog* | *watchdog* )     var="watchdog" ;;
+    *powerkey* )              var="cold,powerkey" ;;
+    *panic* | *kernel_panic*) var="kernel_panic" ;;
+    *thermal*)                var="shutdown,thermal" ;;
+    *s3_wakeup*)              var="warm,s3_wakeup" ;;
+    *hw_reset*)               var="hard,hw_reset" ;;
+    *bootloader*)             var="bootloader" ;;
+    ?*)                       var="reboot,${var}" ;;
+    *)                        var="reboot" ;;
+  esac
+  echo ${var}
+}
+
+#
+# Actual test frames
+#
+
+[ "USAGE: test_properties
+
+properties test
+- (wait until screen is up, boot has completed)
+- adb shell getprop ro.boot.bootreason (bootloader reason)
+- adb shell getprop persist.sys.boot.reason (last reason)
+- adb shell getprop sys.boot.reason (system reason)
+- NB: all should have a value that is compliant with our known set." ]
+test_properties() {
+  wait_for_screen
+  retval=0
+  check_set="ro.boot.bootreason persist.sys.boot.reason sys.boot.reason"
+  bootloader=""
+  # NB: this test could fail if performed _after_ optional_factory_reset test
+  # and will report
+  #  ERROR: expected "reboot" got ""
+  #        for Android property persist.sys.boot.reason
+  # following is mitigation for the persist.sys.boot.reason, skip it
+  if [ "reboot,factory_reset" = `validate_property ro.boot_bootreason` ]; then
+    check_set="ro.boot.bootreason sys.boot.reason"
+    bootloader="bootloader"
+  fi
+  for prop in ${check_set}; do
+    reason=`validate_property ${prop}`
+    EXPECT_PROPERTY ${prop} ${reason} || retval=${?}
+  done
+  # sys.boot.reason is last for a reason
+  report_bootstat_logs ${reason} ${bootloader}
+  return ${retval}
+}
+
+[ "USAGE: test_ota
+
+ota test
+- rm out/.kati_stamp-* out/build_date.txt out/build_number.txt
+- rm out/target/product/*/*/*.prop
+- rm -r out/target/product/*/obj/ETC/system_build_prop_intermediates
+- m
+- NB: ro.build.date.utc should update
+- fastboot flashall
+- (wait until screen is up, boot has completed)
+- adb shell getprop sys.boot.reason
+- NB: should report ota
+
+Decision to change the build itself rather than trick bootstat by
+rummaging through its data files was made." ]
+test_ota() {
+  echo "INFO: expected duration of ${TEST} test about 5 minutes or more" >&2
+  echo "      extended by build and flashing times" >&2
+  if [ -z "${TARGET_PRODUCT}" -o \
+       -z "${ANDROID_PRODUCT_OUT}" -o \
+       -z "${ANDROID_BUILD_TOP}" -o \
+       -z "${TARGET_BUILD_VARIANT}" ]; then
+    echo "ERROR: Missing envsetup.sh and lunch" >&2
+    return 1
+  fi
+  rm ${ANDROID_PRODUCT_OUT%/out/*}/out/.kati_stamp-* ||
+    true
+  rm ${ANDROID_PRODUCT_OUT%/out/*}/out/build_date.txt ||
+    true
+  rm ${ANDROID_PRODUCT_OUT%/out/*}/out/build_number.txt ||
+    true
+  rm ${ANDROID_PRODUCT_OUT}/*/*.prop ||
+    true
+  rm -r ${ANDROID_PRODUCT_OUT}/obj/ETC/system_build_prop_intermediates ||
+    true
+  pushd ${ANDROID_BUILD_TOP} >&2
+  make -j50 >&2
+  if [ ${?} != 0 ]; then
+    popd >&2
+    return 1
+  fi
+  if ! inFastboot; then
+    adb reboot-bootloader >&2
+  fi
+  fastboot flashall >&2
+  popd >&2
+  wait_for_screen
+  EXPECT_PROPERTY sys.boot.reason "\(reboot,ota\|bootloader\)"
+  EXPECT_PROPERTY persist.sys.boot.reason reboot,bootloader
+  report_bootstat_logs reboot,ota bootloader
+}
+
+[ "USAGE: test_optional_ota
+
+fast and fake (touch build_date on device to make it different)" ]
+test_optional_ota() {
+  echo "INFO: expected duration of ${TEST} test about 45 seconds" >&2
+  adb shell su root touch /data/misc/bootstat/build_date >&2
+  adb reboot ota
+  wait_for_screen
+  EXPECT_PROPERTY sys.boot.reason reboot,ota
+  EXPECT_PROPERTY persist.sys.boot.reason reboot,ota
+  report_bootstat_logs reboot,ota
+}
+
+[ "USAGE: [TEST=<test>] blind_reboot_test [<match>]
+
+Simple tests helper
+- adb reboot <test>
+- (wait until screen is up, boot has completed)
+- adb shell getprop sys.boot.reason
+- NB: should report <test>, or overriden <match>
+
+We interleave the simple reboot tests between the hard/complex ones
+as a means of checking sanity and any persistent side effect of the
+other tests." ]
+blind_reboot_test() {
+  if [ -z "${1}" ]; then
+    set ${TEST}
+  fi
+  echo "INFO: expected duration of ${TEST} test roughly 45 seconds" >&2
+  adb reboot ${TEST}
+  wait_for_screen
+  EXPECT_PROPERTY sys.boot.reason ${1}
+  EXPECT_PROPERTY persist.sys.boot.reason reboot,${TEST}
+  report_bootstat_logs ${1}
+}
+
+[ "USAGE: test_cold
+
+cold test
+- adb reboot cold
+- (wait until screen is up, boot has completed)
+- adb shell getprop sys.boot.reason
+- NB: should report cold" ]
+test_cold() {
+  blind_reboot_test
+}
+
+[ "USAGE: test_factory_reset
+
+factory_reset test
+- adb shell su root rm /data/misc/bootstat/build_date
+- adb reboot
+- (wait until screen is up, boot has completed)
+- adb shell getprop sys.boot.reason
+- NB: should report factory_reset
+
+Decision to rummage through bootstat data files was made as
+a _real_ factory_reset is too destructive to the device." ]
+test_factory_reset() {
+  echo "INFO: expected duration of ${TEST} test roughly 45 seconds" >&2
+  adb shell su root rm /data/misc/bootstat/build_date >&2
+  adb reboot >&2
+  wait_for_screen
+  EXPECT_PROPERTY sys.boot.reason reboot,factory_reset
+  EXPECT_PROPERTY persist.sys.boot.reason reboot,
+  report_bootstat_logs reboot,factory_reset reboot, \
+    "-bootstat: Failed to read /data/misc/bootstat/build_date: No such file or directory" \
+    "-bootstat: Failed to parse boot time record: /data/misc/bootstat/build_date"
+}
+
+[ "USAGE: test_optional_factory_reset
+
+factory_reset test
+- adb reboot-bootloader
+- fastboot format userdata
+- fastboot reboot
+- (wait until screen is up, boot has completed)
+- adb shell getprop sys.boot.reason
+- NB: should report factory_reset
+
+For realz, and disruptive" ]
+test_optional_factory_reset() {
+  echo "INFO: expected duration of ${TEST} test roughly a minute" >&2
+  if ! inFastboot; then
+    adb reboot-bootloader
+  fi
+  fastboot format userdata >&2
+  fastboot reboot >&2
+  wait_for_screen
+  EXPECT_PROPERTY sys.boot.reason reboot,factory_reset
+  EXPECT_PROPERTY persist.sys.boot.reason ""
+  report_bootstat_logs reboot,factory_reset bootloader \
+    "-bootstat: Failed to read /data/misc/bootstat/last_boot_time_utc: No such file or directory" \
+    "-bootstat: Failed to parse boot time record: /data/misc/bootstat/last_boot_time_utc" \
+    "-bootstat: Failed to read /data/misc/bootstat/build_date: No such file or directory" \
+    "-bootstat: Failed to parse boot time record: /data/misc/bootstat/build_date" \
+    "-bootstat: Failed to read /data/misc/bootstat/factory_reset: No such file or directory" \
+    "-bootstat: Failed to parse boot time record: /data/misc/bootstat/factory_reset"
+}
+
+[ "USAGE: test_hard
+
+hard test:
+- adb reboot hard
+- (wait until screen is up, boot has completed)
+- adb shell getprop sys.boot.reason
+- NB: should report hard" ]
+test_hard() {
+  blind_reboot_test
+}
+
+[ "USAGE: test_battery
+
+battery test (trick):
+- echo healthd: battery l=2 | adb shell su root tee /dev/kmsg ; adb reboot warm
+- (wait until screen is up, boot has completed)
+- adb shell getprop sys.boot.reason
+- NB: should report reboot,battery, unless healthd managed to log
+  before reboot in above trick.
+
+- Bonus points (manual extras)
+- Make sure the following is added to the /init.rc file in post-fs
+  section before logd is started:
+    +    setprop logd.kernel false
+    +    rm /sys/fs/pstore/console-ramoops
+    +    rm /sys/fs/pstore/console-ramoops-0
+    +    write /dev/kmsg \"healthd: battery l=2
+    +\"
+- adb reboot fs
+- (wait until screen is up, boot has completed)
+- adb shell getprop sys.boot.reason
+- NB: should report reboot,battery
+- (replace set logd.kernel true to the above, and retry test)" ]
+test_battery() {
+  echo "INFO: expected duration of ${TEST} test roughly two minutes" >&2
+  echo healthd: battery l=2 | adb shell su root tee /dev/kmsg >/dev/null
+  adb reboot warm >&2
+  wait_for_screen
+  adb shell su root \
+    cat /proc/fs/pstore/console-ramoops \
+        /proc/fs/pstore/console-ramoops-0 2>/dev/null |
+    grep 'healthd: battery l=' |
+    tail -1 |
+    grep 'healthd: battery l=2' >/dev/null || (
+      if ! EXPECT_PROPERTY sys.boot.reason reboot,battery >/dev/null 2>/dev/null; then
+        # retry
+        echo healthd: battery l=2 | adb shell su root tee /dev/kmsg >/dev/null
+        adb reboot warm >&2
+        wait_for_screen
+      fi
+    )
+
+  EXPECT_PROPERTY sys.boot.reason shutdown,battery
+  EXPECT_PROPERTY persist.sys.boot.reason reboot,warm
+  report_bootstat_logs shutdown,battery "-bootstat: Battery level at shutdown 2%"
+}
+
+[ "USAGE: test_unknown
+
+unknown test
+- adb reboot unknown
+- (wait until screen is up, boot has completed)
+- adb shell getprop sys.boot.reason
+- NB: should report reboot,unknown
+- NB: expect log \"... I bootstat: Unknown boot reason: reboot,unknown\"" ]
+test_unknown() {
+  blind_reboot_test reboot,unknown
+}
+
+[ "USAGE: test_kernel_panic
+
+kernel_panic test:
+- echo c | adb shell su root tee /proc/sysrq-trigger
+- (wait until screen is up, boot has completed)
+- adb shell getprop sys.boot.reason
+- NB: should report kernel_panic,sysrq" ]
+test_kernel_panic() {
+  echo "INFO: expected duration of ${TEST} test > 2 minutes" >&2
+  echo c | adb shell su root tee /proc/sysrq-trigger >/dev/null
+  wait_for_screen
+  EXPECT_PROPERTY sys.boot.reason kernel_panic,sysrq
+  EXPECT_PROPERTY persist.sys.boot.reason kernel_panic,sysrq
+  report_bootstat_logs kernel_panic,sysrq
+}
+
+[ "USAGE: test_warm
+
+warm test
+- adb reboot warm
+- (wait until screen is up, boot has completed)
+- adb shell getprop sys.boot.reason
+- NB: should report warm" ]
+test_warm() {
+  blind_reboot_test
+}
+
+[ "USAGE: test_thermal_shutdown
+
+thermal shutdown test:
+- adb shell setprop sys.powerctl shutdown,thermal
+- (power up the device)
+- (wait until screen is up, boot has completed)
+- adb shell getprop sys.boot.reason
+- NB: should report shutdown,thermal" ]
+test_thermal_shutdown() {
+  echo "INFO: expected duration of ${TEST} test roughly a minute plus" >&2
+  echo "      power on request" >&2
+  adb shell setprop sys.powerctl shutdown,thermal
+  sleep 5
+  echo -n "WARNING: Please power device back up, waiting ... " >&2
+  wait_for_screen -n >&2
+  EXPECT_PROPERTY sys.boot.reason shutdown,thermal
+  EXPECT_PROPERTY persist.sys.boot.reason shutdown,thermal
+  report_bootstat_logs shutdown,thermal
+}
+
+[ "USAGE: test_userrequested_shutdown
+
+userrequested shutdown test:
+- adb shell setprop sys.powerctl shutdown,userrequested
+- (power up the device)
+- (wait until screen is up, boot has completed)
+- adb shell getprop sys.boot.reason
+- NB: should report shutdown,userrequested" ]
+test_userrequested_shutdown() {
+  echo "INFO: expected duration of ${TEST} test roughly a minute plus" >&2
+  echo "      power on request" >&2
+  adb shell setprop sys.powerctl shutdown,userrequested
+  sleep 5
+  echo -n "WARNING: Please power device back up, waiting ... " >&2
+  wait_for_screen -n >&2
+  EXPECT_PROPERTY sys.boot.reason shutdown,userrequested
+  EXPECT_PROPERTY persist.sys.boot.reason shutdown,userrequested
+  report_bootstat_logs shutdown,userrequested
+}
+
+[ "USAGE: ${0##*/} [-s SERIAL] [tests]
+
+Mainline executive to run the above tests" ]
+
+# Rudimentary argument parsing
+
+if [ ${#} -ge 2 -a X"-s" = X"${1}" ]; then
+  export ANDROID_SERIAL="${2}"
+  shift 2
+fi
+
+if [ X"--help" = X"${1}" -o X"-h" = X"${1}" -o X"-?" = X"${1}" ]; then
+  echo "USAGE: ${0##*/} [-s SERIAL] [tests]"
+  echo tests - `sed -n 's/^test_\([^ ()]*\)() {/\1/p' $0 </dev/null`
+  exit 0
+fi
+
+# Check if all conditions for the script are sane
+
+if [ -z "${ANDROID_SERIAL}" ]; then
+  ndev=`(
+      adb devices | grep -v 'List of devices attached'
+      fastboot devices
+    ) |
+    grep -v "^[${SPACE}${TAB}]*\$" |
+    wc -l`
+  if [ ${ndev} -gt 1 ]; then
+    echo "ERROR: no target device specified, ${ndev} connected" >&2
+    echo "${RED}[  FAILED  ]${NORMAL}"
+    exit 1
+  fi
+  echo "WARNING: no target device specified" >&2
+fi
+
+ret=0
+
+# Test Series
+if [ X"all" = X"${*}" ]; then
+  # automagically pick up all test_<function>s.
+  eval set nothing `sed -n 's/^test_\([^ ()]*\)() {/\1/p' $0 </dev/null`
+  if [ X"nothing" = X"${1}" ]; then
+    shift 1
+  fi
+fi
+if [ -z "$*" ]; then
+  # automagically pick up all test_<function>, except test_optional_<function>.
+  eval set nothing `sed -n 's/^test_\([^ ()]*\)() {/\1/p' $0 </dev/null |
+                            grep -v '^optional_'`
+  if [ -z "${2}" ]; then
+    # Hard coded should shell fail to find them above (search/permission issues)
+    eval set ota cold factory_reset hard battery unknown kernel_panic warm \
+             thermal_shutdown userrequested_shutdown
+  fi
+  if [ X"nothing" = X"${1}" ]; then
+    shift 1
+  fi
+fi
+echo "INFO: selected test(s): ${@}" >&2
+echo
+failures=
+successes=
+for t in "${@}"; do
+  wrap_test ${t}
+  retval=${?}
+  if [ 0 = ${retval} ]; then
+    if [ -z "${successes}" ]; then
+      successes=${t}
+    else
+      successes="${successes} ${t}"
+    fi
+  else
+    ret=${retval}
+    if [ -z "${failures}" ]; then
+      failures=${t}
+    else
+      failures="${failures} ${t}"
+    fi
+  fi
+  echo
+done
+
+if [ -n "${successes}" ]; then
+  echo "${GREEN}[  PASSED  ]${NORMAL} ${successes}"
+fi
+if [ -n "${failures}" ]; then
+  echo "${RED}[  FAILED  ]${NORMAL} ${failures}"
+fi
+exit ${ret}