Add framework for testing builtin futility functions.
This tweaks the Makefile and adds a couple of placeholder tests to prepare
for testing the builtin futility operations. There aren't any useful builtin
functions yet, but this lets us start adding them along with the tests.
BUG=chromium:224734
BRANCH=none
TEST=none
This doesn't actually do anything yet.
Change-Id: Iff0ca514f7d26346f072bd80a3bcd04621284843
Signed-off-by: Bill Richardson <wfrichar@chromium.org>
Reviewed-on: https://gerrit.chromium.org/gerrit/47432
Reviewed-by: Randall Spangler <rspangler@chromium.org>
diff --git a/Makefile b/Makefile
index 9ac96d3..ec22984 100644
--- a/Makefile
+++ b/Makefile
@@ -401,35 +401,35 @@
# These utilities should be linked statically.
UTIL_NAMES_STATIC = \
- crossystem \
- dump_fmap \
- gbb_utility
+ utility/crossystem \
+ utility/dump_fmap \
+ utility/gbb_utility
UTIL_NAMES = ${UTIL_NAMES_STATIC} \
- dev_sign_file \
- dump_kernel_config \
- dumpRSAPublicKey \
- tpm_init_temp_fix \
- tpmc \
- vbutil_firmware \
- vbutil_kernel \
- vbutil_key \
- vbutil_keyblock \
+ utility/dev_sign_file \
+ utility/dump_kernel_config \
+ utility/dumpRSAPublicKey \
+ utility/tpm_init_temp_fix \
+ utility/tpmc \
+ utility/vbutil_firmware \
+ utility/vbutil_kernel \
+ utility/vbutil_key \
+ utility/vbutil_keyblock \
ifeq (${MINIMAL},)
UTIL_NAMES += \
- bmpblk_font \
- bmpblk_utility \
- eficompress \
- efidecompress \
- load_kernel_test \
- pad_digest_utility \
- signature_digest_utility \
- verify_data
+ utility/bmpblk_font \
+ utility/bmpblk_utility \
+ utility/eficompress \
+ utility/efidecompress \
+ utility/load_kernel_test \
+ utility/pad_digest_utility \
+ utility/signature_digest_utility \
+ utility/verify_data
endif
-UTIL_BINS_STATIC := $(addprefix ${BUILD}/utility/,${UTIL_NAMES_STATIC})
-UTIL_BINS = $(addprefix ${BUILD}/utility/,${UTIL_NAMES})
+UTIL_BINS_STATIC := $(addprefix ${BUILD}/,${UTIL_NAMES_STATIC})
+UTIL_BINS = $(addprefix ${BUILD}/,${UTIL_NAMES})
ALL_OBJS += $(addsuffix .o,${UTIL_BINS} ${UTIL_BINS_STATIC})
@@ -481,34 +481,35 @@
# And some compiled tests.
TEST_NAMES = \
- cgptlib_test \
- rollback_index2_tests \
- rollback_index3_tests \
- rsa_padding_test \
- rsa_utility_tests \
- rsa_verify_benchmark \
- sha_benchmark \
- sha_tests \
- stateful_util_tests \
- tlcl_tests \
- tpm_bootmode_tests \
- utility_string_tests \
- utility_tests \
- vboot_api_init_tests \
- vboot_api_devmode_tests \
- vboot_api_firmware_tests \
- vboot_api_kernel_tests \
- vboot_api_kernel2_tests \
- vboot_api_kernel3_tests \
- vboot_api_kernel4_tests \
- vboot_audio_tests \
- vboot_common_tests \
- vboot_common2_tests \
- vboot_common3_tests \
- vboot_display_tests \
- vboot_firmware_tests \
- vboot_kernel_tests \
- vboot_nvstorage_test
+ tests/cgptlib_test \
+ tests/rollback_index2_tests \
+ tests/rollback_index3_tests \
+ tests/rsa_padding_test \
+ tests/rsa_utility_tests \
+ tests/rsa_verify_benchmark \
+ tests/sha_benchmark \
+ tests/sha_tests \
+ tests/stateful_util_tests \
+ tests/tlcl_tests \
+ tests/tpm_bootmode_tests \
+ tests/utility_string_tests \
+ tests/utility_tests \
+ tests/vboot_api_init_tests \
+ tests/vboot_api_devmode_tests \
+ tests/vboot_api_firmware_tests \
+ tests/vboot_api_kernel_tests \
+ tests/vboot_api_kernel2_tests \
+ tests/vboot_api_kernel3_tests \
+ tests/vboot_api_kernel4_tests \
+ tests/vboot_audio_tests \
+ tests/vboot_common_tests \
+ tests/vboot_common2_tests \
+ tests/vboot_common3_tests \
+ tests/vboot_display_tests \
+ tests/vboot_firmware_tests \
+ tests/vboot_kernel_tests \
+ tests/vboot_nvstorage_test \
+ tests/futility/test_not_really
# TODO: port these tests to new API, if not already eqivalent
# functionality in other tests. These don't even compile at present.
@@ -529,24 +530,23 @@
# utility/load_firmware_test
# And a few more...
-TLCL_TESTS = \
- tpmtest_earlyextend \
- tpmtest_earlynvram \
- tpmtest_earlynvram2 \
- tpmtest_enable \
- tpmtest_fastenable \
- tpmtest_globallock \
- tpmtest_redefine_unowned \
- tpmtest_spaceperm \
- tpmtest_testsetup \
- tpmtest_timing \
- tpmtest_writelimit
-TLCL_TEST_NAMES = $(addprefix tpm_lite/,${TLCL_TESTS})
-TLCL_TEST_BINS = $(addprefix ${BUILD}/tests/,${TLCL_TEST_NAMES})
+TLCL_TEST_NAMES = \
+ tests/tpm_lite/tpmtest_earlyextend \
+ tests/tpm_lite/tpmtest_earlynvram \
+ tests/tpm_lite/tpmtest_earlynvram2 \
+ tests/tpm_lite/tpmtest_enable \
+ tests/tpm_lite/tpmtest_fastenable \
+ tests/tpm_lite/tpmtest_globallock \
+ tests/tpm_lite/tpmtest_redefine_unowned \
+ tests/tpm_lite/tpmtest_spaceperm \
+ tests/tpm_lite/tpmtest_testsetup \
+ tests/tpm_lite/tpmtest_timing \
+ tests/tpm_lite/tpmtest_writelimit
TEST_NAMES += ${TLCL_TEST_NAMES}
-TEST_BINS = $(addprefix ${BUILD}/tests/,${TEST_NAMES})
+# Finally
+TEST_BINS = $(addprefix ${BUILD}/,${TEST_NAMES})
ALL_OBJS += $(addsuffix .o,${TEST_BINS})
# Directory containing test keys
@@ -791,6 +791,7 @@
tests: ${TEST_BINS}
${TEST_BINS}: ${HOSTLIB} ${TESTLIB}
+${TEST_BINS}: INCLUDES += -Itests
${TEST_BINS}: LIBS = ${HOSTLIB} ${TESTLIB}
${TESTLIB}: ${TESTLIB_OBJS}
@@ -897,6 +898,7 @@
${BUILD}/tests/rollback_index_test: INCLUDES += -I/usr/include
${BUILD}/tests/rollback_index_test: LIBS += -ltlcl
+TLCL_TEST_BINS = $(addprefix ${BUILD}/,${TLCL_TEST_NAMES})
${TLCL_TEST_BINS}: OBJS += ${BUILD}/tests/tpm_lite/tlcl_tests.o
${TLCL_TEST_BINS}: ${BUILD}/tests/tpm_lite/tlcl_tests.o
ALL_OBJS += ${BUILD}/tests/tpm_lite/tlcl_tests.o
@@ -995,7 +997,8 @@
.PHONY: runfutiltests
runfutiltests: override DESTDIR = ${TEST_INSTALL_DIR}
runfutiltests: test_setup install
- futility/tests/run_futility_tests.sh ${DESTDIR}
+ tests/futility/run_test_scripts.sh ${DESTDIR}
+ ${RUNTEST} ${BUILD_RUN}/tests/futility/test_not_really
# Run long tests, including all permutations of encryption keys (instead of
# just the ones we use) and tests of currently-unused code.
diff --git a/futility/tests/run_futility_tests.sh b/futility/tests/run_futility_tests.sh
deleted file mode 100755
index ff2de4b..0000000
--- a/futility/tests/run_futility_tests.sh
+++ /dev/null
@@ -1,78 +0,0 @@
-#!/bin/bash
-# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-# Load common constants and variables.
-. "$(dirname "$0")/common.sh"
-
-# Where are the programs I'm testing against?
-[ -z "${1:-}" ] && error "Directory argument is required"
-BINDIR="$1"
-shift
-
-FUTILITY="$BINDIR/futility"
-OLDDIR="$BINDIR/old_bins"
-
-BUILD=$(dirname "${BINDIR}")
-
-# Here are the old programs to be wrapped
-# FIXME(chromium-os:37062): There are others besides these.
-# FIXME: dev_debug_vboot isn't tested right now.
-PROGS=${*:-cgpt crossystem dev_sign_file dumpRSAPublicKey
- dump_fmap dump_kernel_config enable_dev_usb_boot gbb_utility
- tpm_init_temp_fix tpmc vbutil_firmware vbutil_kernel vbutil_key
- vbutil_keyblock vbutil_what_keys}
-
-# Get ready
-pass=0
-progs=0
-pwd
-OUTDIR="${BUILD}/tests/futility_test_dir"
-[ -d "$OUTDIR" ] || mkdir -p "$OUTDIR"
-
-# For now just compare results of invoking each program with no args.
-# FIXME(chromium-os:37062): Create true rigorous tests for every program.
-for i in $PROGS; do
- : $(( progs++ ))
-
- # Try the real thing first
- echo -n "$i ... "
- rc=$("${OLDDIR}/$i" \
- 1>"${OUTDIR}/$i.stdout.orig" 2>"${OUTDIR}/$i.stderr.orig" \
- || echo "$?")
- echo "${rc:-0}" > "${OUTDIR}/$i.return.orig"
-
- # Then try the symlink
- rc=$("$BINDIR/$i" 1>"${OUTDIR}/$i.stdout.link" \
- 2>"${OUTDIR}/$i.stderr.link" || echo "$?")
- echo "${rc:-0}" > "${OUTDIR}/$i.return.link"
-
- # And finally try the explicit wrapper
- rc=$("$FUTILITY" "$i" 1>"${OUTDIR}/$i.stdout.futil" \
- 2>"${OUTDIR}/$i.stderr.futil" || echo "$?")
- echo "${rc:-0}" > "${OUTDIR}/$i.return.futil"
-
- # Different?
- if cmp -s "${OUTDIR}/$i.return.orig" "${OUTDIR}/$i.return.link" &&
- cmp -s "${OUTDIR}/$i.stdout.orig" "${OUTDIR}/$i.stdout.link" &&
- cmp -s "${OUTDIR}/$i.stderr.orig" "${OUTDIR}/$i.stderr.link" &&
- cmp -s "${OUTDIR}/$i.return.orig" "${OUTDIR}/$i.return.futil" &&
- cmp -s "${OUTDIR}/$i.stdout.orig" "${OUTDIR}/$i.stdout.futil" &&
- cmp -s "${OUTDIR}/$i.stderr.orig" "${OUTDIR}/$i.stderr.futil" ; then
- green "passed"
- : $(( pass++ ))
- rm -f ${OUTDIR}/$i.{stdout,stderr,return}.{orig,link,futil}
- else
- red "failed"
- fi
-done
-
-# done
-if [ "$pass" -eq "$progs" ]; then
- green "Success: $pass / $progs passed"
- exit 0
-fi
-
-red "FAIL: $pass / $progs passed"
-exit 1
diff --git a/futility/tests/common.sh b/tests/futility/common.sh
similarity index 100%
rename from futility/tests/common.sh
rename to tests/futility/common.sh
diff --git a/tests/futility/run_test_scripts.sh b/tests/futility/run_test_scripts.sh
new file mode 100755
index 0000000..d886ea9
--- /dev/null
+++ b/tests/futility/run_test_scripts.sh
@@ -0,0 +1,125 @@
+#!/bin/bash -eu
+# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# Load common constants and variables.
+SCRIPTDIR=$(dirname $(readlink -f "$0"))
+. "$SCRIPTDIR/common.sh"
+
+# Mandatory arg is the path to the futility executable to test.
+[ -z "${1:-}" ] && error "Directory argument is required"
+BINDIR="$1"
+shift
+
+FUTILITY="$BINDIR/futility"
+OLDDIR="$BINDIR/old_bins"
+
+
+# The Makefile should export the $BUILD directory, but if it's not just warn
+# and guess (mostly so we can run the script manually).
+if [ -z "${BUILD:-}" ]; then
+ BUILD=$(dirname "${BINDIR}")
+ yellow "Assuming \$BUILD=$BUILD"
+fi
+OUTDIR="${BUILD}/tests/futility_test_results"
+[ -d "$OUTDIR" ] || mkdir -p "$OUTDIR"
+
+
+# Let each test know where to find things...
+export FUTILITY
+export SCRIPTDIR
+export OUTDIR
+
+# These are the scripts to run. Binaries are invoked directly by the Makefile.
+TESTS="${SCRIPTDIR}/test_not_really.sh"
+
+
+# Get ready...
+pass=0
+progs=0
+
+##############################################################################
+# But first, we'll just test the wrapped functions. This will go away when
+# everything is built in (chromium:196079).
+
+# Here are the old programs to be wrapped
+# FIXME: dev_debug_vboot isn't tested right now.
+PROGS=${*:-cgpt crossystem dev_sign_file dumpRSAPublicKey
+ dump_fmap dump_kernel_config enable_dev_usb_boot gbb_utility
+ tpm_init_temp_fix tpmc vbutil_firmware vbutil_kernel vbutil_key
+ vbutil_keyblock vbutil_what_keys}
+
+# For now just compare results of invoking each program with no args.
+# FIXME(chromium-os:37062): Create true rigorous tests for every program.
+echo "-- old_bins --"
+for i in $PROGS; do
+ : $(( progs++ ))
+
+ # Try the real thing first
+ echo -n "$i ... "
+ rc=$("${OLDDIR}/$i" \
+ 1>"${OUTDIR}/$i.stdout.orig" 2>"${OUTDIR}/$i.stderr.orig" \
+ || echo "$?")
+ echo "${rc:-0}" > "${OUTDIR}/$i.return.orig"
+
+ # Then try the symlink
+ rc=$("$BINDIR/$i" 1>"${OUTDIR}/$i.stdout.link" \
+ 2>"${OUTDIR}/$i.stderr.link" || echo "$?")
+ echo "${rc:-0}" > "${OUTDIR}/$i.return.link"
+
+ # And finally try the explicit wrapper
+ rc=$("$FUTILITY" "$i" 1>"${OUTDIR}/$i.stdout.futil" \
+ 2>"${OUTDIR}/$i.stderr.futil" || echo "$?")
+ echo "${rc:-0}" > "${OUTDIR}/$i.return.futil"
+
+ # Different?
+ if cmp -s "${OUTDIR}/$i.return.orig" "${OUTDIR}/$i.return.link" &&
+ cmp -s "${OUTDIR}/$i.stdout.orig" "${OUTDIR}/$i.stdout.link" &&
+ cmp -s "${OUTDIR}/$i.stderr.orig" "${OUTDIR}/$i.stderr.link" &&
+ cmp -s "${OUTDIR}/$i.return.orig" "${OUTDIR}/$i.return.futil" &&
+ cmp -s "${OUTDIR}/$i.stdout.orig" "${OUTDIR}/$i.stdout.futil" &&
+ cmp -s "${OUTDIR}/$i.stderr.orig" "${OUTDIR}/$i.stderr.futil" ; then
+ green "passed"
+ : $(( pass++ ))
+ rm -f ${OUTDIR}/$i.{stdout,stderr,return}.{orig,link,futil}
+ else
+ red "failed"
+ fi
+done
+
+
+##############################################################################
+# Invoke the scripts that test the builtin functions.
+
+echo "-- builtin --"
+for i in $TESTS; do
+ j=${i##*/}
+
+ : $(( progs++ ))
+
+ echo -n "$j ... "
+ rm -f "${OUTDIR}/$j."*
+ rc=$("$i" "$FUTILITY" 1>"${OUTDIR}/$j.stdout" \
+ 2>"${OUTDIR}/$j.stderr" || echo "$?")
+ echo "${rc:-0}" > "${OUTDIR}/$j.return"
+ if [ ! "$rc" ]; then
+ green "passed"
+ : $(( pass++ ))
+ rm -f ${OUTDIR}/$i.{stdout,stderr,return}
+ else
+ red "failed"
+ fi
+
+done
+
+##############################################################################
+# How'd we do?
+
+if [ "$pass" -eq "$progs" ]; then
+ green "Success: $pass / $progs passed"
+ exit 0
+fi
+
+red "FAIL: $pass / $progs passed"
+exit 1
diff --git a/tests/futility/test_not_really.c b/tests/futility/test_not_really.c
new file mode 100644
index 0000000..6a6555c
--- /dev/null
+++ b/tests/futility/test_not_really.c
@@ -0,0 +1,15 @@
+/*
+ * Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
+ * Use of this source code is governed by a BSD-style license that can be
+ * found in the LICENSE file.
+ */
+#include <stdio.h>
+#include "test_common.h"
+
+int main(int argc, char *argv[])
+{
+ TEST_EQ(0, 0, "Not Really A");
+
+ return !gTestSuccess;
+}
+
diff --git a/tests/futility/test_not_really.sh b/tests/futility/test_not_really.sh
new file mode 100755
index 0000000..8ea0c9a
--- /dev/null
+++ b/tests/futility/test_not_really.sh
@@ -0,0 +1,13 @@
+#!/bin/bash -eu
+# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+me=${0##*/}
+
+TMP="$OUTDIR/$me.tmp"
+
+echo "FUTILITY=$FUTILITY" > "$TMP"
+echo "SCRIPTDIR=$SCRIPTDIR" >> "$TMP"
+
+exit 0