Merge "Ensure we don't go into the auto-retry if the runner doesn't support it"
diff --git a/atest/atest.py b/atest/atest.py
index 9271a32..2ee41c2 100755
--- a/atest/atest.py
+++ b/atest/atest.py
@@ -599,6 +599,10 @@
atest_execution_info.print_test_result(constants.ATEST_RESULT_ROOT,
args.history)
return constants.EXIT_CODE_SUCCESS
+ if args.latest_result:
+ atest_execution_info.print_test_result_by_path(
+ constants.LATEST_RESULT_FILE)
+ return constants.EXIT_CODE_SUCCESS
mod_info = module_info.ModuleInfo(force_build=args.rebuild_module_info)
if args.rebuild_module_info:
_run_extra_tasks(join=True)
diff --git a/atest/atest_arg_parser.py b/atest/atest_arg_parser.py
index 6e5079c..c497de4 100644
--- a/atest/atest_arg_parser.py
+++ b/atest/atest_arg_parser.py
@@ -53,6 +53,7 @@
'Note: Nothing\'s going to run if it\'s not an Instant App test and '
'"--instant" is passed.')
ITERATION = 'Loop-run tests until the max iteration is reached. (10 by default)'
+LATEST_RESULT = 'Print latest test result.'
LIST_MODULES = 'List testable modules for the given suite.'
REBUILD_MODULE_INFO = ('Forces a rebuild of the module-info.json file. '
'This may be necessary following a repo sync or '
@@ -205,10 +206,14 @@
type=_positive_int, const=10, default=0,
metavar='MAX_ITERATIONS', help=RETRY_ANY_FAILURE)
- # Option for test result history.
- group.add_argument('--history', nargs='?',
- type=_positive_int, const=1000, default=0,
- help=HISTORY)
+ # A group of options for history. They are mutually exclusive
+ # in a command line.
+ history_group = self.add_mutually_exclusive_group()
+ # History related options.
+ history_group.add_argument('--latest-result', action='store_true',
+ help=LATEST_RESULT)
+ history_group.add_argument('--history', nargs='?', const='99999',
+ help=HISTORY)
# This arg actually doesn't consume anything, it's primarily used for
# the help description and creating custom_args in the NameSpace object.
@@ -252,6 +257,7 @@
INSTALL=INSTALL,
INSTANT=INSTANT,
ITERATION=ITERATION,
+ LATEST_RESULT=LATEST_RESULT,
LIST_MODULES=LIST_MODULES,
REBUILD_MODULE_INFO=REBUILD_MODULE_INFO,
RERUN_UNTIL_FAILURE=RERUN_UNTIL_FAILURE,
@@ -346,6 +352,9 @@
-L, --list-modules
{LIST_MODULES}
+ --latest-result
+ {LATEST_RESULT}
+
-v, --verbose
{VERBOSE}
diff --git a/atest/atest_execution_info.py b/atest/atest_execution_info.py
index 39cd3c1..0c67e19 100644
--- a/atest/atest_execution_info.py
+++ b/atest/atest_execution_info.py
@@ -23,6 +23,7 @@
import os
import sys
+import atest_utils as au
import constants
from metrics import metrics_utils
@@ -40,6 +41,10 @@
_TEST_RESULT_NAME = 'test_result'
_EXIT_CODE_ATTR = 'EXIT_CODE'
_MAIN_MODULE_KEY = '__main__'
+_UUID_LEN = 30
+_RESULT_LEN = 35
+_COMMAND_LEN = 50
+_LOGCAT_FMT = '{}/log/invocation_*/{}*logcat-on-failure*'
_SUMMARY_MAP_TEMPLATE = {_STATUS_PASSED_KEY : 0,
_STATUS_FAILED_KEY : 0,
@@ -72,18 +77,30 @@
os.symlink(test_result_dir, symlink)
-def print_test_result(root, num):
+def print_test_result(root, history_arg):
"""Make a list of latest n test result.
Args:
root: A string of the test result root path.
- num: An integer, the number of latest results.
+ history_arg: A string of an integer or uuid. If it's an integer string,
+ the number of lines of test result will be given; else it
+ will be treated a uuid and print test result accordingly
+ in detail.
"""
+ if not history_arg.isdigit():
+ path = os.path.join(constants.ATEST_RESULT_ROOT, history_arg,
+ 'test_result')
+ print_test_result_by_path(path)
+ return
target = '%s/20*_*_*' % root
paths = glob.glob(target)
paths.sort(reverse=True)
- print('{:-^22} {:-^35} {:-^50}'.format('uuid', 'result', 'command'))
- for path in paths[0: num+1]:
+ print('{:-^{uuid_len}} {:-^{result_len}} {:-^{command_len}}'
+ .format('uuid', 'result', 'command',
+ uuid_len=_UUID_LEN,
+ result_len=_RESULT_LEN,
+ command_len=_COMMAND_LEN))
+ for path in paths[0: int(history_arg)+1]:
result_path = os.path.join(path, 'test_result')
if os.path.isfile(result_path):
try:
@@ -92,14 +109,58 @@
total_summary = result.get(_TOTAL_SUMMARY_KEY, {})
summary_str = ', '.join([k+':'+str(v)
for k, v in total_summary.items()])
- print('{:<22} {:<35} {:<50}'
+ print('{:<{uuid_len}} {:<{result_len}} {:<{command_len}}'
.format(os.path.basename(path),
summary_str,
- 'atest '+result.get(_ARGS_KEY, '')))
+ 'atest '+result.get(_ARGS_KEY, ''),
+ uuid_len=_UUID_LEN,
+ result_len=_RESULT_LEN,
+ command_len=_COMMAND_LEN))
except ValueError:
pass
+def print_test_result_by_path(path):
+ """Print latest test result.
+
+ Args:
+ path: A string of test result path.
+ """
+ if os.path.isfile(path):
+ with open(path) as json_file:
+ result = json.load(json_file)
+ print("\natest {}".format(result.get(_ARGS_KEY, '')))
+ print('\nTotal Summary:\n--------------')
+ total_summary = result.get(_TOTAL_SUMMARY_KEY, {})
+ print(', '.join([(k+':'+str(v))
+ for k, v in total_summary.items()]))
+ fail_num = total_summary.get(_STATUS_FAILED_KEY)
+ if fail_num > 0:
+ message = '%d test failed' % fail_num
+ print('\n')
+ print(au.colorize(message, constants.RED))
+ print('-' * len(message))
+ test_runner = result.get(_TEST_RUNNER_KEY, {})
+ for runner_name in test_runner.keys():
+ test_dict = test_runner.get(runner_name, {})
+ for test_name in test_dict:
+ test_details = test_dict.get(test_name, {})
+ for fail in test_details.get(_STATUS_FAILED_KEY):
+ print(au.colorize('{}'.format(
+ fail.get(_TEST_NAME_KEY)), constants.RED))
+ failure_files = glob.glob(_LOGCAT_FMT.format(
+ os.path.dirname(path), fail.get(_TEST_NAME_KEY)
+ ))
+ if failure_files:
+ print('{} {}'.format(
+ au.colorize('LOGCAT-ON-FAILURES:',
+ constants.CYAN),
+ failure_files[0]))
+ print('{} {}'.format(
+ au.colorize('STACKTRACE:\n', constants.CYAN),
+ fail.get(_TEST_DETAILS_KEY)))
+
+
def has_non_test_options(args):
"""
check whether non-test option in the args.
@@ -116,7 +177,8 @@
or args.help
or args.history
or args.info
- or args.version)
+ or args.version
+ or args.latest_result)
class AtestExecutionInfo(object):
diff --git a/atest/constants_default.py b/atest/constants_default.py
index 1df9e2d..f730b60 100644
--- a/atest/constants_default.py
+++ b/atest/constants_default.py
@@ -233,3 +233,4 @@
r'(?P<package>[^(;|\s)]+)\s*')
ATEST_RESULT_ROOT = '/tmp/atest_result'
+LATEST_RESULT_FILE = os.path.join(ATEST_RESULT_ROOT, 'LATEST', 'test_result')
diff --git a/device_build_interfaces/com/android/tradefed/build/IBuildInfo.java b/device_build_interfaces/com/android/tradefed/build/IBuildInfo.java
index 8fd2026..85b4a30 100644
--- a/device_build_interfaces/com/android/tradefed/build/IBuildInfo.java
+++ b/device_build_interfaces/com/android/tradefed/build/IBuildInfo.java
@@ -298,9 +298,6 @@
return false;
}
- /** Set the build as test resource build. */
- public default void setTestResourceBuild(boolean testResourceBuild) {}
-
/** Get the paths for build artifacts that are delayed download. */
public default Set<File> getRemoteFiles() {
return null;
diff --git a/device_build_interfaces/com/android/tradefed/device/INativeDevice.java b/device_build_interfaces/com/android/tradefed/device/INativeDevice.java
index ee6e900..3ceb944 100644
--- a/device_build_interfaces/com/android/tradefed/device/INativeDevice.java
+++ b/device_build_interfaces/com/android/tradefed/device/INativeDevice.java
@@ -1402,22 +1402,12 @@
/**
* Extra steps for device specific required setup that will be executed on the device prior to
* the invocation flow.
- */
- public default void preInvocationSetup(IBuildInfo info)
- throws TargetSetupError, DeviceNotAvailableException {
- preInvocationSetup(info, null);
- }
-
- /**
- * Extra steps for device specific required setup that will be executed on the device prior to
- * the invocation flow.
*
* @param info The {@link IBuildInfo} of the device.
- * @param testResourceBuildInfos The list of test resources.
* @throws TargetSetupError
* @throws DeviceNotAvailableException
*/
- public default void preInvocationSetup(IBuildInfo info, List<IBuildInfo> testResourceBuildInfos)
+ public default void preInvocationSetup(IBuildInfo info)
throws TargetSetupError, DeviceNotAvailableException {
// Empty default implementation.
}
diff --git a/src/com/android/tradefed/build/BuildInfo.java b/src/com/android/tradefed/build/BuildInfo.java
index 2069b2a..04b2135 100644
--- a/src/com/android/tradefed/build/BuildInfo.java
+++ b/src/com/android/tradefed/build/BuildInfo.java
@@ -134,12 +134,6 @@
return mTestResourceBuild;
}
- /** {@inheritDoc} */
- @Override
- public void setTestResourceBuild(boolean testResourceBuild) {
- mTestResourceBuild = testResourceBuild;
- }
-
/**
* {@inheritDoc}
*/
diff --git a/src/com/android/tradefed/device/DeviceManager.java b/src/com/android/tradefed/device/DeviceManager.java
index a653c1e..485af19 100644
--- a/src/com/android/tradefed/device/DeviceManager.java
+++ b/src/com/android/tradefed/device/DeviceManager.java
@@ -615,7 +615,7 @@
@Override
public ITestDevice forceAllocateDevice(String serial) {
checkInit();
- IManagedTestDevice d = mManagedDeviceList.findOrCreate(new StubDevice(serial, false));
+ IManagedTestDevice d = mManagedDeviceList.forceAllocate(serial);
if (d != null) {
DeviceEventResponse r = d.handleAllocationEvent(DeviceEvent.FORCE_ALLOCATE_REQUEST);
if (r.stateChanged && r.allocationState == DeviceAllocationState.Allocated) {
@@ -836,10 +836,17 @@
*/
@Override
public ITestDevice connectToTcpDevice(String ipAndPort) {
- ITestDevice tcpDevice = forceAllocateDevice(ipAndPort);
+ IManagedTestDevice tcpDevice = mManagedDeviceList.findOrCreate(new StubDevice(ipAndPort));
if (tcpDevice == null) {
return null;
}
+ DeviceEventResponse r = tcpDevice.handleAllocationEvent(DeviceEvent.FORCE_ALLOCATE_REQUEST);
+ if (r.stateChanged && r.allocationState == DeviceAllocationState.Allocated) {
+ // Wait for the fastboot state to be updated once to update the IDevice.
+ tcpDevice.getMonitor().waitForDeviceBootloaderStateUpdate();
+ } else {
+ return null;
+ }
if (doAdbConnect(ipAndPort)) {
try {
tcpDevice.setRecovery(new WaitDeviceRecovery());
diff --git a/src/com/android/tradefed/device/DeviceSelectionOptions.java b/src/com/android/tradefed/device/DeviceSelectionOptions.java
index eae78ea..6dbae6d 100644
--- a/src/com/android/tradefed/device/DeviceSelectionOptions.java
+++ b/src/com/android/tradefed/device/DeviceSelectionOptions.java
@@ -499,12 +499,12 @@
}
// If battery check is required and we have a min/max battery requested
if (mRequireBatteryCheck) {
- if (((mMinBattery != null) || (mMaxBattery != null))
- && (!(device instanceof StubDevice) || (device instanceof FastbootDevice))) {
+ if ((mMinBattery != null || mMaxBattery != null)) {
// Only check battery on physical device. (FastbootDevice placeholder is always for
// a physical device
- if (device instanceof FastbootDevice) {
- // Ready battery of fastboot device does not work and could lead to weird log.
+ if (device instanceof StubDevice || device instanceof FastbootDevice) {
+ // Reading battery of fastboot and StubDevice device does not work and could
+ // lead to weird log.
return false;
}
Integer deviceBattery = getBatteryLevel(device);
diff --git a/src/com/android/tradefed/device/LocalAndroidVirtualDevice.java b/src/com/android/tradefed/device/LocalAndroidVirtualDevice.java
index 66a92fb..ea45061 100644
--- a/src/com/android/tradefed/device/LocalAndroidVirtualDevice.java
+++ b/src/com/android/tradefed/device/LocalAndroidVirtualDevice.java
@@ -35,9 +35,11 @@
import com.android.tradefed.util.RunUtil;
import com.android.tradefed.util.TarUtil;
import com.android.tradefed.util.ZipUtil;
+
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Strings;
import com.google.common.net.HostAndPort;
+
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
@@ -76,10 +78,10 @@
/** Execute common setup procedure and launch the virtual device. */
@Override
- public void preInvocationSetup(IBuildInfo info, List<IBuildInfo> testResourceBuildInfos)
+ public void preInvocationSetup(IBuildInfo info)
throws TargetSetupError, DeviceNotAvailableException {
// The setup method in super class does not require the device to be online.
- super.preInvocationSetup(info, testResourceBuildInfos);
+ super.preInvocationSetup(info);
createTempDirs((IDeviceBuildInfo) info);
diff --git a/src/com/android/tradefed/device/ManagedDeviceList.java b/src/com/android/tradefed/device/ManagedDeviceList.java
index d33c769..d296ab7 100644
--- a/src/com/android/tradefed/device/ManagedDeviceList.java
+++ b/src/com/android/tradefed/device/ManagedDeviceList.java
@@ -237,6 +237,23 @@
}
/**
+ * Force allocate a device based on its serial, if the device isn't tracked yet we won't create
+ * its tracking. This is meant for force-allocate known devices, usually for automated recovery.
+ */
+ public IManagedTestDevice forceAllocate(String serial) {
+ if (!isValidDeviceSerial(serial)) {
+ return null;
+ }
+ mListLock.lock();
+ try {
+ // Unlike findOrCreate we don't attempt to create in this case.
+ return find(serial);
+ } finally {
+ mListLock.unlock();
+ }
+ }
+
+ /**
* Directly add a device to the list. Should not be used outside of unit testing.
* @param device
*/
diff --git a/src/com/android/tradefed/device/NativeDevice.java b/src/com/android/tradefed/device/NativeDevice.java
index cc5b8d6..e730a83 100644
--- a/src/com/android/tradefed/device/NativeDevice.java
+++ b/src/com/android/tradefed/device/NativeDevice.java
@@ -4392,7 +4392,7 @@
/** {@inheritDoc} */
@Override
- public void preInvocationSetup(IBuildInfo info, List<IBuildInfo> testResourceBuildInfos)
+ public void preInvocationSetup(IBuildInfo info)
throws TargetSetupError, DeviceNotAvailableException {
// Default implementation
mContentProvider = null;
diff --git a/src/com/android/tradefed/device/cloud/ManagedRemoteDevice.java b/src/com/android/tradefed/device/cloud/ManagedRemoteDevice.java
index 282c9bd..cabd21f 100644
--- a/src/com/android/tradefed/device/cloud/ManagedRemoteDevice.java
+++ b/src/com/android/tradefed/device/cloud/ManagedRemoteDevice.java
@@ -44,7 +44,6 @@
import java.io.File;
import java.io.IOException;
-import java.util.List;
/**
* A device running inside a virtual machine that we manage remotely via a Tradefed instance inside
@@ -72,9 +71,9 @@
}
@Override
- public void preInvocationSetup(IBuildInfo info, List<IBuildInfo> testResourceBuildInfos)
+ public void preInvocationSetup(IBuildInfo info)
throws TargetSetupError, DeviceNotAvailableException {
- super.preInvocationSetup(info, testResourceBuildInfos);
+ super.preInvocationSetup(info);
mGceAvd = null;
// First get the options
TestDeviceOptions options = getOptions();
diff --git a/src/com/android/tradefed/device/cloud/NestedRemoteDevice.java b/src/com/android/tradefed/device/cloud/NestedRemoteDevice.java
index 9ae0988..b362b34 100644
--- a/src/com/android/tradefed/device/cloud/NestedRemoteDevice.java
+++ b/src/com/android/tradefed/device/cloud/NestedRemoteDevice.java
@@ -161,7 +161,7 @@
// Reset recovery since it's a new device
setRecoveryMode(RecoveryMode.AVAILABLE);
try {
- preInvocationSetup(info, null);
+ preInvocationSetup(info);
} catch (TargetSetupError e) {
CLog.e("Failed to re-init the device %s", getSerialNumber());
CLog.e(e);
diff --git a/src/com/android/tradefed/device/cloud/RemoteAndroidVirtualDevice.java b/src/com/android/tradefed/device/cloud/RemoteAndroidVirtualDevice.java
index 641fe31..5042966 100644
--- a/src/com/android/tradefed/device/cloud/RemoteAndroidVirtualDevice.java
+++ b/src/com/android/tradefed/device/cloud/RemoteAndroidVirtualDevice.java
@@ -84,9 +84,9 @@
/** {@inheritDoc} */
@Override
- public void preInvocationSetup(IBuildInfo info, List<IBuildInfo> testResourceBuildInfos)
+ public void preInvocationSetup(IBuildInfo info)
throws TargetSetupError, DeviceNotAvailableException {
- super.preInvocationSetup(info, testResourceBuildInfos);
+ super.preInvocationSetup(info);
try {
mGceAvd = null;
mGceSshMonitor = null;
diff --git a/src/com/android/tradefed/invoker/InvocationExecution.java b/src/com/android/tradefed/invoker/InvocationExecution.java
index bd35c72..f2a31ff 100644
--- a/src/com/android/tradefed/invoker/InvocationExecution.java
+++ b/src/com/android/tradefed/invoker/InvocationExecution.java
@@ -91,7 +91,6 @@
import java.util.concurrent.Callable;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit;
-import java.util.stream.Collectors;
/**
* Class that describes all the invocation steps: build download, target_prep, run tests, clean up.
@@ -339,12 +338,7 @@
if (device instanceof ITestLoggerReceiver) {
((ITestLoggerReceiver) context.getDevice(deviceName)).setTestLogger(logger);
}
- device.preInvocationSetup(
- context.getBuildInfo(deviceName),
- context.getBuildInfos()
- .stream()
- .filter(buildInfo -> buildInfo.isTestResourceBuild())
- .collect(Collectors.toList()));
+ device.preInvocationSetup(context.getBuildInfo(deviceName));
}
}
diff --git a/src/com/android/tradefed/testtype/UsbResetTest.java b/src/com/android/tradefed/testtype/UsbResetTest.java
index 1be0792..40b7047 100644
--- a/src/com/android/tradefed/testtype/UsbResetTest.java
+++ b/src/com/android/tradefed/testtype/UsbResetTest.java
@@ -47,6 +47,7 @@
} else {
CLog.d("Resetting USB port for device '%s'", serial);
usbDevice.reset();
+ device.waitForDeviceOnline();
// If device fails to reboot it will throw an exception and be left unavailable
// again.
device.reboot();
diff --git a/test_framework/com/android/tradefed/targetprep/RunHostScriptTargetPreparer.java b/test_framework/com/android/tradefed/targetprep/RunHostScriptTargetPreparer.java
index 346c86e..0245523 100644
--- a/test_framework/com/android/tradefed/targetprep/RunHostScriptTargetPreparer.java
+++ b/test_framework/com/android/tradefed/targetprep/RunHostScriptTargetPreparer.java
@@ -42,7 +42,7 @@
/**
* Target preparer which executes a script before running a test. The script can reference the
- * device's serial number using the SERIAL environment variable.
+ * device's serial number using the ANDROID_SERIAL environment variable.
*/
@OptionClass(alias = "run-host-script")
public class RunHostScriptTargetPreparer extends BaseTargetPreparer {
@@ -79,7 +79,7 @@
// Set working directory and environment variables
getRunUtil().setWorkingDir(mWorkDir);
- getRunUtil().setEnvVariable("SERIAL", device.getSerialNumber());
+ getRunUtil().setEnvVariable("ANDROID_SERIAL", device.getSerialNumber());
setPathVariable(testInfo);
// Execute script and handle result
diff --git a/test_framework/com/android/tradefed/testtype/GTestListener.java b/test_framework/com/android/tradefed/testtype/GTestListener.java
index c967d06..38c6498 100644
--- a/test_framework/com/android/tradefed/testtype/GTestListener.java
+++ b/test_framework/com/android/tradefed/testtype/GTestListener.java
@@ -22,8 +22,10 @@
import com.android.tradefed.result.TestDescription;
import com.android.tradefed.result.proto.TestRecordProto.FailureStatus;
+import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.List;
import java.util.Set;
/**
@@ -32,6 +34,7 @@
*/
final class GTestListener extends ResultForwarder {
+ private static final int MAX_PARTIAL_SET_SIZE = 20;
private Set<TestDescription> mTests = new HashSet<>();
private Set<TestDescription> mDuplicateTests = new HashSet<>();
@@ -50,11 +53,19 @@
@Override
public void testRunEnded(long elapsedTime, HashMap<String, Metric> runMetrics) {
if (!mDuplicateTests.isEmpty()) {
- FailureDescription error =
- FailureDescription.create(
- String.format(
- "The following tests ran more than once: %s.",
- mDuplicateTests));
+ StringBuilder errorMessage = new StringBuilder();
+ errorMessage.append(
+ String.format("%s tests ran more than once.", mDuplicateTests.size()));
+ if (mDuplicateTests.size() > MAX_PARTIAL_SET_SIZE) {
+ List<TestDescription> partialDuplicateSet = new ArrayList<>(mDuplicateTests);
+ while (partialDuplicateSet.size() > MAX_PARTIAL_SET_SIZE) {
+ partialDuplicateSet.remove(0);
+ }
+ errorMessage.append(String.format(" Partial list: %s", partialDuplicateSet));
+ } else {
+ errorMessage.append(String.format(" Full list: %s", mDuplicateTests));
+ }
+ FailureDescription error = FailureDescription.create(errorMessage.toString());
error.setFailureStatus(FailureStatus.TEST_FAILURE);
super.testRunFailed(error);
}
diff --git a/test_framework/com/android/tradefed/testtype/GoogleBenchmarkTest.java b/test_framework/com/android/tradefed/testtype/GoogleBenchmarkTest.java
index 163b4fb..373844a 100644
--- a/test_framework/com/android/tradefed/testtype/GoogleBenchmarkTest.java
+++ b/test_framework/com/android/tradefed/testtype/GoogleBenchmarkTest.java
@@ -16,6 +16,7 @@
package com.android.tradefed.testtype;
import com.android.ddmlib.FileListingService;
+import com.android.ddmlib.IShellOutputReceiver;
import com.android.tradefed.config.Option;
import com.android.tradefed.config.OptionClass;
import com.android.tradefed.device.CollectingOutputReceiver;
@@ -26,24 +27,28 @@
import com.android.tradefed.result.ITestInvocationListener;
import com.android.tradefed.util.proto.TfMetricProtoUtil;
+import com.google.common.annotations.VisibleForTesting;
+
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import java.util.concurrent.TimeUnit;
-/**
- * A Test that runs a Google benchmark test package on given device.
- */
+/** A Test that runs a Google benchmark test package on given device. */
@OptionClass(alias = "gbenchmark")
-public class GoogleBenchmarkTest implements IDeviceTest, IRemoteTest {
+public class GoogleBenchmarkTest implements IDeviceTest, IRemoteTest, ITestFilterReceiver {
static final String DEFAULT_TEST_PATH = "/data/benchmarktest";
+ static final String GBENCHMARK_FILTER_OPTION = "--benchmark_filter";
+ static final int ADB_CMD_CHAR_LIMIT = 4000; // May apply to some old device models.
private static final String GBENCHMARK_JSON_OUTPUT_FORMAT = "--benchmark_format=json";
private static final String GBENCHMARK_LIST_TESTS_OPTION = "--benchmark_list_tests=true";
-
private static final List<String> DEFAULT_FILE_EXCLUDE_FILTERS = new ArrayList<>();
static {
@@ -75,6 +80,16 @@
"The maximum time to allow for each benchmark run in ms.", isTimeVal=true)
private long mMaxRunTime = 15 * 60 * 1000;
+ @Option(
+ name = "include-filter",
+ description = "The benchmark (regex) filters used to match benchmarks to include.")
+ private Set<String> mIncludeFilters = new LinkedHashSet<>();
+
+ @Option(
+ name = "exclude-filter",
+ description = "The benchmark (regex) filters used to match benchmarks to exclude.")
+ private Set<String> mExcludeFilters = new LinkedHashSet<>();
+
private ITestDevice mDevice = null;
/**
@@ -168,8 +183,11 @@
}
long startTime = System.currentTimeMillis();
+ Set<String> filteredTests = getFilteredTests(testDevice, root);
+ CLog.d("List that will be used: %s", Arrays.asList(filteredTests));
+
// Count expected number of tests
- int numTests = countExpectedTests(testDevice, root);
+ int numTests = filteredTests.size();
if (numTests == 0) {
CLog.d("No tests to run.");
return;
@@ -180,11 +198,15 @@
GoogleBenchmarkResultParser resultParser = createResultParser(runName, listener);
listener.testRunStarted(runName, numTests);
try {
- String cmd = String.format("%s %s", root, GBENCHMARK_JSON_OUTPUT_FORMAT);
+ String cmd =
+ String.format(
+ "%s%s %s",
+ root,
+ getFilterFlagForTests(filteredTests),
+ GBENCHMARK_JSON_OUTPUT_FORMAT);
CLog.i(String.format("Running google benchmark test on %s: %s",
mDevice.getSerialNumber(), cmd));
- testDevice.executeShellCommand(cmd, outputCollector,
- mMaxRunTime, TimeUnit.MILLISECONDS, 0);
+ executeCommand(testDevice, cmd, outputCollector);
metricMap = resultParser.parse(outputCollector);
} catch (DeviceNotAvailableException e) {
listener.testRunFailed(e.getMessage());
@@ -196,17 +218,60 @@
}
}
- private int countExpectedTests(ITestDevice testDevice, String fullBinaryPath)
+ /**
+ * Returns benchmark tests matching current filters.
+ *
+ * @param testDevice the device on which to run the command
+ * @param fullBinaryPath the full binary path
+ * @return matching benchmark tests.
+ * @throws DeviceNotAvailableException
+ */
+ private Set<String> getFilteredTests(ITestDevice testDevice, String fullBinaryPath)
throws DeviceNotAvailableException {
- if (!testDevice.isExecutable(fullBinaryPath)) {
- CLog.d("%s does not look like an executable", fullBinaryPath);
- return 0;
+ Set<String> filteredTests = getTestsForFilters(testDevice, fullBinaryPath, mIncludeFilters);
+ if (!mExcludeFilters.isEmpty()) {
+ filteredTests.removeAll(
+ getTestsForFilters(testDevice, fullBinaryPath, mExcludeFilters));
}
- String cmd = String.format("%s %s", fullBinaryPath, GBENCHMARK_LIST_TESTS_OPTION);
- String list_output = testDevice.executeShellCommand(cmd);
+ return filteredTests;
+ }
+
+ /**
+ * Returns benchmark tests matching the filters.
+ *
+ * @param testDevice the device on which to run the command
+ * @param fullBinaryPath the full binary path
+ * @param filters filters for matching benchmark tests
+ * @return matching benchmark tests.
+ * @throws DeviceNotAvailableException
+ */
+ private Set<String> getTestsForFilters(
+ ITestDevice testDevice, String fullBinaryPath, Set<String> filters)
+ throws DeviceNotAvailableException {
+ String cmd =
+ String.format(
+ "%s%s %s",
+ fullBinaryPath,
+ getFilterFlagForFilters(filters),
+ GBENCHMARK_LIST_TESTS_OPTION);
+ String list_output = executeCommand(testDevice, cmd, null /* outputReceiver */);
String[] list = list_output.trim().split("\n");
- CLog.d("List that will be used: %s", Arrays.asList(list));
- return list.length;
+ if (noMatchesFound(list)) {
+ list = new String[0];
+ }
+ return new LinkedHashSet<String>(Arrays.asList(list));
+ }
+
+ // ** Returns true if no matches found. */
+ private boolean noMatchesFound(String[] list) {
+ if (list.length == 0) {
+ return true;
+ }
+
+ // A benchmark name is a single word.
+ // A no-match output is "Failed to match any benchmarks ..."
+ // Consider no matches found if the output line has multiple-words.
+ return (list[0].indexOf(' ') >= 0);
}
/**
@@ -262,4 +327,162 @@
}
doRunAllTestsInSubdirectory(testPath, mDevice, listener);
}
+
+ /*
+ * Conforms filters using a {@link TestDescription} format to be recognized by the
+ * GoogleBenchmarkTest executable.
+ */
+ public String cleanFilter(String filter) {
+ Integer index = filter.indexOf('#');
+ if (index >= 0) {
+ // For "class#method", takes the "method" part as the benchmark name.
+ String benchmark = filter.substring(index + 1);
+ if (benchmark.isEmpty()) {
+ CLog.e("Invalid filter %s. Result could be unexpected.", filter);
+ } else {
+ return benchmark;
+ }
+ }
+ return filter;
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public void addIncludeFilter(String filter) {
+ mIncludeFilters.add(cleanFilter(filter));
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public void addAllIncludeFilters(Set<String> filters) {
+ for (String filter : filters) {
+ mIncludeFilters.add(cleanFilter(filter));
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public void addExcludeFilter(String filter) {
+ mExcludeFilters.add(cleanFilter(filter));
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public void addAllExcludeFilters(Set<String> filters) {
+ for (String filter : filters) {
+ mExcludeFilters.add(cleanFilter(filter));
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public Set<String> getIncludeFilters() {
+ return mIncludeFilters;
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public Set<String> getExcludeFilters() {
+ return mExcludeFilters;
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public void clearIncludeFilters() {
+ mIncludeFilters.clear();
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public void clearExcludeFilters() {
+ mExcludeFilters.clear();
+ }
+
+ @VisibleForTesting
+ protected String getFilterFlagForFilters(Set<String> filters) {
+ // Use single filter as only the last "--benchmark_filter" is recognized.
+ StringBuilder filterFlag = new StringBuilder();
+ Iterator<String> iterator = filters.iterator();
+ if (iterator.hasNext()) {
+ filterFlag.append(String.format(" %s=%s", GBENCHMARK_FILTER_OPTION, iterator.next()));
+ while (iterator.hasNext()) {
+ filterFlag.append(String.format("\\|%s", iterator.next()));
+ }
+ }
+ return filterFlag.toString();
+ }
+
+ @VisibleForTesting
+ protected String getFilterFlagForTests(Set<String> fitlererTests) {
+ // Pass the list of tests as filter since BenchmarkTest can't handle negative filters.
+ StringBuilder filterFlag = new StringBuilder();
+ Iterator<String> iterator = fitlererTests.iterator();
+ if (iterator.hasNext()) {
+ // Format benchmark as "^benchmark$" to avoid unintended regex partial matching.
+ filterFlag.append(String.format(" %s=^%s$", GBENCHMARK_FILTER_OPTION, iterator.next()));
+ while (iterator.hasNext()) {
+ filterFlag.append(String.format("\\|^%s$", iterator.next()));
+ }
+ }
+ return filterFlag.toString();
+ }
+
+ /**
+ * Helper method to run a benchmarktest command. If the command is too long to be run directly
+ * by adb, it runs from a temporary script.
+ *
+ * @param testDevice the device on which to run the command
+ * @param cmd the command string to run
+ * @param outputReceiver the output receiver for reading test results
+ * @return shell output if outputReceiver is null
+ */
+ protected String executeCommand(
+ final ITestDevice testDevice,
+ final String cmd,
+ final IShellOutputReceiver outputReceiver)
+ throws DeviceNotAvailableException {
+ // Ensure that command is not too long for adb
+ if (cmd.length() < ADB_CMD_CHAR_LIMIT) {
+ if (outputReceiver == null) {
+ return testDevice.executeShellCommand(cmd);
+ }
+
+ testDevice.executeShellCommand(
+ cmd,
+ outputReceiver,
+ mMaxRunTime /* maxTimeToShellOutputResponse */,
+ TimeUnit.MILLISECONDS,
+ 0 /* retryAttempts */);
+ return null;
+ }
+
+ // Wrap adb shell command in script if command is too long for direct execution
+ return executeCommandByScript(testDevice, cmd, outputReceiver);
+ }
+
+ /** Runs a command from a temporary script. */
+ private String executeCommandByScript(
+ final ITestDevice testDevice,
+ final String cmd,
+ final IShellOutputReceiver outputReceiver)
+ throws DeviceNotAvailableException {
+ String tmpFileDevice = "/data/local/tmp/gbenchmarktest_script.sh";
+ testDevice.pushString(String.format("#!/bin/bash\n%s", cmd), tmpFileDevice);
+ String shellOutput = null;
+ try {
+ if (outputReceiver == null) {
+ shellOutput = testDevice.executeShellCommand(String.format("sh %s", tmpFileDevice));
+ } else {
+ testDevice.executeShellCommand(
+ String.format("sh %s", tmpFileDevice),
+ outputReceiver,
+ mMaxRunTime /* maxTimeToShellOutputResponse */,
+ TimeUnit.MILLISECONDS,
+ 0 /* retry attempts */);
+ }
+ } finally {
+ testDevice.deleteFile(tmpFileDevice);
+ }
+ return shellOutput;
+ }
}
diff --git a/tests/src/com/android/tradefed/device/DeviceManagerTest.java b/tests/src/com/android/tradefed/device/DeviceManagerTest.java
index 3d1d564..d3c4a4e 100644
--- a/tests/src/com/android/tradefed/device/DeviceManagerTest.java
+++ b/tests/src/com/android/tradefed/device/DeviceManagerTest.java
@@ -502,11 +502,9 @@
/** Test {@link DeviceManager#forceAllocateDevice(String)} when device is unknown */
@Test
public void testForceAllocateDevice() {
- EasyMock.expect(mMockTestDevice.handleAllocationEvent(DeviceEvent.FORCE_ALLOCATE_REQUEST))
- .andReturn(new DeviceEventResponse(DeviceAllocationState.Allocated, true));
replayMocks();
DeviceManager manager = createDeviceManager(null);
- assertNotNull(manager.forceAllocateDevice("unknownserial"));
+ assertNull(manager.forceAllocateDevice("unknownserial"));
verifyMocks();
}
@@ -514,8 +512,6 @@
@Test
public void testForceAllocateDevice_available() {
setCheckAvailableDeviceExpectations();
- EasyMock.expect(mMockTestDevice.getAllocationState())
- .andReturn(DeviceAllocationState.Available);
EasyMock.expect(mMockTestDevice.handleAllocationEvent(DeviceEvent.FORCE_ALLOCATE_REQUEST))
.andReturn(new DeviceEventResponse(DeviceAllocationState.Allocated, true));
replayMocks();
@@ -528,8 +524,6 @@
@Test
public void testForceAllocateDevice_alreadyAllocated() {
setCheckAvailableDeviceExpectations();
- EasyMock.expect(mMockTestDevice.getAllocationState())
- .andReturn(DeviceAllocationState.Allocated);
EasyMock.expect(mMockTestDevice.handleAllocationEvent(DeviceEvent.ALLOCATE_REQUEST))
.andReturn(new DeviceEventResponse(DeviceAllocationState.Allocated, true));
EasyMock.expect(mMockTestDevice.handleAllocationEvent(DeviceEvent.FORCE_ALLOCATE_REQUEST))
diff --git a/tests/src/com/android/tradefed/device/DeviceSelectionOptionsTest.java b/tests/src/com/android/tradefed/device/DeviceSelectionOptionsTest.java
index c263704..12d849a 100644
--- a/tests/src/com/android/tradefed/device/DeviceSelectionOptionsTest.java
+++ b/tests/src/com/android/tradefed/device/DeviceSelectionOptionsTest.java
@@ -538,15 +538,22 @@
assertFalse(mDeviceSelection.matches(mMockDevice));
}
- /**
- * Test that min-battery is not used to check non physical devices otherwise they will never
- * match.
- */
+ /** Test that if min-battery is used for a StubDevice it will not match. */
@Test
public void testStubDevice_minBattery() throws Exception {
OptionSetter setter = new OptionSetter(mDeviceSelection);
+ setter.setOptionValue("require-battery-check", "true");
setter.setOptionValue("min-battery", "20");
setter.setOptionValue("null-device", "true");
+ assertFalse(mDeviceSelection.matches(new NullDevice("test")));
+ }
+
+ /** Test that if we require a battery check but no minimal or max, StubDevice can be matched. */
+ @Test
+ public void testStubDevice_requireBatteryCheck() throws Exception {
+ OptionSetter setter = new OptionSetter(mDeviceSelection);
+ setter.setOptionValue("require-battery-check", "true");
+ setter.setOptionValue("null-device", "true");
assertTrue(mDeviceSelection.matches(new NullDevice("test")));
}
diff --git a/tests/src/com/android/tradefed/device/LocalAndroidVirtualDeviceTest.java b/tests/src/com/android/tradefed/device/LocalAndroidVirtualDeviceTest.java
index c123833..70a20f2 100644
--- a/tests/src/com/android/tradefed/device/LocalAndroidVirtualDeviceTest.java
+++ b/tests/src/com/android/tradefed/device/LocalAndroidVirtualDeviceTest.java
@@ -18,7 +18,6 @@
import com.android.ddmlib.IDevice;
import com.android.ddmlib.Log.LogLevel;
-import com.android.tradefed.build.IBuildInfo;
import com.android.tradefed.build.IDeviceBuildInfo;
import com.android.tradefed.log.ITestLogger;
import com.android.tradefed.result.LogDataType;
@@ -29,14 +28,7 @@
import com.android.tradefed.util.IRunUtil;
import com.android.tradefed.util.StreamUtil;
import com.android.tradefed.util.ZipUtil;
-import java.io.BufferedOutputStream;
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.OutputStream;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.zip.GZIPOutputStream;
+
import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
import org.easymock.Capture;
@@ -49,6 +41,14 @@
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
+import java.io.BufferedOutputStream;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.ArrayList;
+import java.util.zip.GZIPOutputStream;
+
/** Unit tests for {@link LocalAndroidVirtualDevice}. */
@RunWith(JUnit4.class)
public class LocalAndroidVirtualDeviceTest {
@@ -351,7 +351,7 @@
mLocalAvd.setTestLogger(testLogger);
mLocalAvd.currentRunUtil = acloudCreateRunUtil;
mLocalAvd.expectToConnect = true;
- mLocalAvd.preInvocationSetup(mMockDeviceBuildInfo, null);
+ mLocalAvd.preInvocationSetup(mMockDeviceBuildInfo);
Assert.assertEquals(ONLINE_SERIAL_NUMBER, mLocalAvd.getIDevice().getSerialNumber());
@@ -407,7 +407,7 @@
mLocalAvd.setTestLogger(testLogger);
mLocalAvd.currentRunUtil = acloudCreateRunUtil;
try {
- mLocalAvd.preInvocationSetup(mMockDeviceBuildInfo, null);
+ mLocalAvd.preInvocationSetup(mMockDeviceBuildInfo);
Assert.fail("TargetSetupError is not thrown");
} catch (TargetSetupError e) {
expectedException = e;
@@ -449,7 +449,7 @@
mLocalAvd.setTestLogger(testLogger);
mLocalAvd.currentRunUtil = acloudCreateRunUtil;
try {
- mLocalAvd.preInvocationSetup(mMockDeviceBuildInfo, null);
+ mLocalAvd.preInvocationSetup(mMockDeviceBuildInfo);
Assert.fail("TargetSetupError is not thrown");
} catch (TargetSetupError e) {
expectedException = e;
diff --git a/tests/src/com/android/tradefed/invoker/SandboxedInvocationExecutionTest.java b/tests/src/com/android/tradefed/invoker/SandboxedInvocationExecutionTest.java
index 654055e..a29e7ae 100644
--- a/tests/src/com/android/tradefed/invoker/SandboxedInvocationExecutionTest.java
+++ b/tests/src/com/android/tradefed/invoker/SandboxedInvocationExecutionTest.java
@@ -322,7 +322,7 @@
mContext.addInvocationAttribute("test", "test");
// Device early preInvocationSetup was called and even if no tests run we still call tear
// down
- Mockito.verify(mMockDevice).preInvocationSetup(any(), any());
+ Mockito.verify(mMockDevice).preInvocationSetup(any());
Mockito.verify(mMockDevice).postInvocationTearDown(null);
}
@@ -379,7 +379,7 @@
doReturn(info).when(mMockProvider).getBuild();
DeviceNotAvailableException exception = new DeviceNotAvailableException("reason", "serial");
- doThrow(exception).when(mMockDevice).preInvocationSetup(eq(info), any());
+ doThrow(exception).when(mMockDevice).preInvocationSetup(eq(info));
mInvocation.invoke(mContext, mConfig, mMockRescheduler, mMockListener);
// No tests to run but we still call start/end
@@ -390,7 +390,7 @@
mContext.addInvocationAttribute("test", "test");
// Device early preInvocationSetup was called and even if no tests run we still call tear
// down
- Mockito.verify(mMockDevice).preInvocationSetup(any(), any());
+ Mockito.verify(mMockDevice).preInvocationSetup(any());
Mockito.verify(mMockDevice).postInvocationTearDown(exception);
}
diff --git a/tests/src/com/android/tradefed/invoker/TestInvocationTest.java b/tests/src/com/android/tradefed/invoker/TestInvocationTest.java
index 1925909..3d56501 100644
--- a/tests/src/com/android/tradefed/invoker/TestInvocationTest.java
+++ b/tests/src/com/android/tradefed/invoker/TestInvocationTest.java
@@ -221,8 +221,7 @@
EasyMock.expect(mMockDevice.getDeviceState()).andStubReturn(TestDeviceState.NOT_AVAILABLE);
mMockDevice.setRecoveryMode(RecoveryMode.AVAILABLE);
mMockDevice.setRecovery(mMockRecovery);
- mMockDevice.preInvocationSetup(
- (IBuildInfo) EasyMock.anyObject(), EasyMock.<List<IBuildInfo>>anyObject());
+ mMockDevice.preInvocationSetup((IBuildInfo) EasyMock.anyObject());
EasyMock.expectLastCall().anyTimes();
mFakeDescriptor =
new DeviceDescriptor(
@@ -1582,8 +1581,6 @@
mMockBuildInfo = EasyMock.createMock(IDeviceBuildInfo.class);
EasyMock.expect(mMockBuildInfo.getProperties()).andStubReturn(new HashSet<>());
EasyMock.expect(mMockBuildInfo.isTestResourceBuild()).andStubReturn(false);
- mMockBuildInfo.setTestResourceBuild(EasyMock.anyBoolean());
- EasyMock.expectLastCall().anyTimes();
IRemoteTest test = EasyMock.createNiceMock(IRemoteTest.class);
mMockPreparer.tearDown(EasyMock.anyObject(), EasyMock.isNull());
@@ -1683,8 +1680,6 @@
.andReturn(tmpTestsDir);
EasyMock.expect(mMockBuildInfo.getProperties()).andStubReturn(new HashSet<>());
EasyMock.expect(mMockBuildInfo.isTestResourceBuild()).andStubReturn(false);
- mMockBuildInfo.setTestResourceBuild(EasyMock.anyBoolean());
- EasyMock.expectLastCall().anyTimes();
setupMockSuccessListeners();
setupNormalInvoke(test);
@@ -1777,8 +1772,6 @@
prop.add(BuildInfoProperties.DO_NOT_LINK_TESTS_DIR);
EasyMock.expect(mMockBuildInfo.getProperties()).andStubReturn(prop);
EasyMock.expect(mMockBuildInfo.isTestResourceBuild()).andStubReturn(false);
- mMockBuildInfo.setTestResourceBuild(EasyMock.anyBoolean());
- EasyMock.expectLastCall().anyTimes();
setupMockSuccessListeners();
setupNormalInvoke(test);
diff --git a/tests/src/com/android/tradefed/targetprep/RunHostScriptTargetPreparerTest.java b/tests/src/com/android/tradefed/targetprep/RunHostScriptTargetPreparerTest.java
index ee43e22..4125fa1 100644
--- a/tests/src/com/android/tradefed/targetprep/RunHostScriptTargetPreparerTest.java
+++ b/tests/src/com/android/tradefed/targetprep/RunHostScriptTargetPreparerTest.java
@@ -112,7 +112,7 @@
mOptionSetter.setOptionValue("script-timeout", "10");
// Verify environment, timeout, and script path
mPreparer.setUp(mTestInfo);
- verify(mRunUtil).setEnvVariable("SERIAL", DEVICE_SERIAL);
+ verify(mRunUtil).setEnvVariable("ANDROID_SERIAL", DEVICE_SERIAL);
verify(mRunUtil, never()).setEnvVariable(eq("PATH"), any()); // uses default PATH
verify(mRunUtil).runTimedCmd(10L, mScriptFile.getAbsolutePath());
// Verify that script is executable
diff --git a/tests/src/com/android/tradefed/testtype/GTestListenerTest.java b/tests/src/com/android/tradefed/testtype/GTestListenerTest.java
index 392ee75..d05f3fb 100644
--- a/tests/src/com/android/tradefed/testtype/GTestListenerTest.java
+++ b/tests/src/com/android/tradefed/testtype/GTestListenerTest.java
@@ -96,7 +96,7 @@
String moduleName = "testWithDuplicateTests";
String testClass = "testClass";
String testName1 = "testName1";
- String duplicateTestsMessage = "The following tests ran more than once: ";
+ String duplicateTestsMessage = "1 tests ran more than once. Full list:";
TestDescription testId1 = new TestDescription(testClass, testName1);
mMockListener.testRunStarted(EasyMock.eq(moduleName), EasyMock.eq(2));
diff --git a/tests/src/com/android/tradefed/testtype/GoogleBenchmarkTestTest.java b/tests/src/com/android/tradefed/testtype/GoogleBenchmarkTestTest.java
index 9fbfa13..51877ce 100644
--- a/tests/src/com/android/tradefed/testtype/GoogleBenchmarkTestTest.java
+++ b/tests/src/com/android/tradefed/testtype/GoogleBenchmarkTestTest.java
@@ -22,14 +22,18 @@
import com.android.tradefed.invoker.TestInformation;
import com.android.tradefed.metrics.proto.MetricMeasurement.Metric;
import com.android.tradefed.result.ITestInvocationListener;
+import com.android.tradefed.result.TestDescription;
import junit.framework.TestCase;
import org.easymock.EasyMock;
+import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
+import java.util.LinkedHashSet;
import java.util.Map;
+import java.util.Set;
import java.util.concurrent.TimeUnit;
/**
@@ -42,6 +46,7 @@
private ITestDevice mMockITestDevice = null;
private GoogleBenchmarkTest mGoogleBenchmarkTest;
private TestInformation mTestInfo;
+ private TestDescription mDummyTest;
/**
* Helper to initialize the various EasyMocks we'll need.
@@ -52,23 +57,28 @@
mMockInvocationListener = EasyMock.createMock(ITestInvocationListener.class);
mMockReceiver = new CollectingOutputReceiver();
mMockITestDevice = EasyMock.createMock(ITestDevice.class);
+ mDummyTest = new TestDescription("Class", "method");
EasyMock.expect(mMockITestDevice.getSerialNumber()).andStubReturn("serial");
- mGoogleBenchmarkTest = new GoogleBenchmarkTest() {
- @Override
- CollectingOutputReceiver createOutputCollector() {
- return mMockReceiver;
- }
- @Override
- GoogleBenchmarkResultParser createResultParser(String runName,
- ITestInvocationListener listener) {
- return new GoogleBenchmarkResultParser(runName, listener) {
+ mGoogleBenchmarkTest =
+ new GoogleBenchmarkTest() {
@Override
- public Map<String, String> parse(CollectingOutputReceiver output) {
- return Collections.emptyMap();
+ CollectingOutputReceiver createOutputCollector() {
+ return mMockReceiver;
+ }
+
+ @Override
+ GoogleBenchmarkResultParser createResultParser(
+ String runName, ITestInvocationListener listener) {
+ return new GoogleBenchmarkResultParser(runName, listener) {
+ @Override
+ public Map<String, String> parse(CollectingOutputReceiver output) {
+ listener.testStarted(mDummyTest);
+ listener.testEnded(mDummyTest, Collections.emptyMap());
+ return Collections.emptyMap();
+ }
+ };
}
};
- }
- };
mGoogleBenchmarkTest.setDevice(mMockITestDevice);
mTestInfo = TestInformation.newBuilder().build();
}
@@ -108,13 +118,11 @@
mMockITestDevice.executeShellCommand(EasyMock.contains(test2), EasyMock.same(mMockReceiver),
EasyMock.anyLong(), (TimeUnit)EasyMock.anyObject(), EasyMock.anyInt());
- EasyMock.expect(mMockITestDevice.isExecutable(nativeTestPath + "/test1")).andReturn(true);
EasyMock.expect(
mMockITestDevice.executeShellCommand(
String.format(
"%s/test1 --benchmark_list_tests=true", nativeTestPath)))
.andReturn("method1\nmethod2\nmethod3");
- EasyMock.expect(mMockITestDevice.isExecutable(nativeTestPath + "/test2")).andReturn(true);
EasyMock.expect(
mMockITestDevice.executeShellCommand(
@@ -122,7 +130,13 @@
"%s/test2 --benchmark_list_tests=true", nativeTestPath)))
.andReturn("method1\nmethod2\n");
mMockInvocationListener.testRunStarted(test1, 3);
+ mMockInvocationListener.testStarted(mDummyTest);
+ mMockInvocationListener.testEnded(
+ EasyMock.eq(mDummyTest), EasyMock.<HashMap<String, String>>anyObject());
mMockInvocationListener.testRunStarted(test2, 2);
+ mMockInvocationListener.testStarted(mDummyTest);
+ mMockInvocationListener.testEnded(
+ EasyMock.eq(mDummyTest), EasyMock.<HashMap<String, String>>anyObject());
mMockInvocationListener.testRunEnded(
EasyMock.anyLong(), EasyMock.<HashMap<String, Metric>>anyObject());
EasyMock.expectLastCall().times(2);
@@ -185,20 +199,24 @@
EasyMock.anyLong(), (TimeUnit)EasyMock.anyObject(), EasyMock.anyInt());
mMockITestDevice.executeShellCommand(EasyMock.contains(test2), EasyMock.same(mMockReceiver),
EasyMock.anyLong(), (TimeUnit)EasyMock.anyObject(), EasyMock.anyInt());
- EasyMock.expect(mMockITestDevice.isExecutable(nativeTestPath + "/test1")).andReturn(true);
EasyMock.expect(
mMockITestDevice.executeShellCommand(
String.format(
"%s/test1 --benchmark_list_tests=true", nativeTestPath)))
.andReturn("\nmethod1\nmethod2\nmethod3\n\n");
- EasyMock.expect(mMockITestDevice.isExecutable(nativeTestPath + "/test2")).andReturn(true);
EasyMock.expect(
mMockITestDevice.executeShellCommand(
String.format(
"%s/test2 --benchmark_list_tests=true", nativeTestPath)))
.andReturn("method1\nmethod2\n");
mMockInvocationListener.testRunStarted(test1, 3);
+ mMockInvocationListener.testStarted(mDummyTest);
+ mMockInvocationListener.testEnded(
+ EasyMock.eq(mDummyTest), EasyMock.<HashMap<String, String>>anyObject());
mMockInvocationListener.testRunStarted(test2, 2);
+ mMockInvocationListener.testStarted(mDummyTest);
+ mMockInvocationListener.testEnded(
+ EasyMock.eq(mDummyTest), EasyMock.<HashMap<String, String>>anyObject());
mMockInvocationListener.testRunEnded(
EasyMock.anyLong(), EasyMock.<HashMap<String, Metric>>anyObject());
EasyMock.expectLastCall().times(2);
@@ -226,7 +244,6 @@
.andReturn("");
mMockITestDevice.executeShellCommand(EasyMock.contains(test1), EasyMock.same(mMockReceiver),
EasyMock.anyLong(), (TimeUnit)EasyMock.anyObject(), EasyMock.anyInt());
- EasyMock.expect(mMockITestDevice.isExecutable(nativeTestPath + "/test1")).andReturn(true);
EasyMock.expect(
mMockITestDevice.executeShellCommand(
String.format(
@@ -234,6 +251,9 @@
.andReturn("method1\nmethod2\nmethod3");
// Expect reportName instead of test name
mMockInvocationListener.testRunStarted(reportName, 3);
+ mMockInvocationListener.testStarted(mDummyTest);
+ mMockInvocationListener.testEnded(
+ EasyMock.eq(mDummyTest), EasyMock.<HashMap<String, String>>anyObject());
mMockInvocationListener.testRunEnded(
EasyMock.anyLong(), EasyMock.<HashMap<String, Metric>>anyObject());
EasyMock.expectLastCall();
@@ -260,7 +280,6 @@
mMockITestDevice.executeShellCommand(EasyMock.contains(test1), EasyMock.same(mMockReceiver),
EasyMock.anyLong(), (TimeUnit)EasyMock.anyObject(), EasyMock.anyInt());
EasyMock.expectLastCall().andThrow(new DeviceNotAvailableException("dnae", "serial"));
- EasyMock.expect(mMockITestDevice.isExecutable(nativeTestPath + "/test1")).andReturn(true);
EasyMock.expect(
mMockITestDevice.executeShellCommand(
String.format(
@@ -322,4 +341,217 @@
assertFalse(mGoogleBenchmarkTest.shouldSkipFile("/some/path/file/test.txt"));
}
+
+ /** Test getFilterFlagForFilters. */
+ public void testGetFilterFlagForFilters() {
+ Set<String> filters = new LinkedHashSet<>(Arrays.asList("filter1", "filter2"));
+ String filterFlag = mGoogleBenchmarkTest.getFilterFlagForFilters(filters);
+ assertEquals(
+ String.format(
+ " %s=%s",
+ GoogleBenchmarkTest.GBENCHMARK_FILTER_OPTION, "filter1\\|filter2"),
+ filterFlag);
+ }
+
+ /** Test getFilterFlagForFilters - no filters. */
+ public void testGetFilterFlagForFilters_noFilters() {
+ Set<String> filters = new LinkedHashSet<>();
+ String filterFlag = mGoogleBenchmarkTest.getFilterFlagForFilters(filters);
+ assertEquals("", filterFlag);
+ }
+
+ /** Test getFilterFlagForTests. */
+ public void testGetFilterFlagForTests() {
+ Set<String> tests = new LinkedHashSet<>(Arrays.asList("test1", "test2"));
+ String filterFlag = mGoogleBenchmarkTest.getFilterFlagForTests(tests);
+ assertEquals(
+ String.format(
+ " %s=%s",
+ GoogleBenchmarkTest.GBENCHMARK_FILTER_OPTION, "^test1$\\|^test2$"),
+ filterFlag);
+ }
+
+ /** Test getFilterFlagForTests - no tests. */
+ public void testGetFilterFlagForTests_noFilters() {
+ Set<String> tests = new LinkedHashSet<>();
+ String filterFlag = mGoogleBenchmarkTest.getFilterFlagForTests(tests);
+ assertEquals("", filterFlag);
+ }
+
+ /**
+ * Helper function to do the actual filtering test.
+ *
+ * @param incFilter filter flag for querying tests to include
+ * @param incTests tests to include
+ * @param excFilter filter flag for querying tests to exclude
+ * @param excTests tests to exclude
+ * @param testFilter filter flag for running tests
+ * @throws DeviceNotAvailableException
+ */
+ private void doTestFilter(String incTests, String excTests, Set<String> filteredTests)
+ throws DeviceNotAvailableException {
+ String nativeTestPath = GoogleBenchmarkTest.DEFAULT_TEST_PATH;
+ String testPath = nativeTestPath + "/test1";
+ // configure the mock file system to have a single test
+ MockFileUtil.setMockDirContents(mMockITestDevice, nativeTestPath, "test1");
+ EasyMock.expect(mMockITestDevice.doesFileExist(nativeTestPath)).andReturn(true);
+ EasyMock.expect(mMockITestDevice.isDirectory(nativeTestPath)).andReturn(true);
+ EasyMock.expect(mMockITestDevice.isDirectory(testPath)).andReturn(false);
+ EasyMock.expect(mMockITestDevice.executeShellCommand(EasyMock.contains("chmod")))
+ .andReturn("");
+ String[] files = new String[] {"test1"};
+ EasyMock.expect(mMockITestDevice.getChildren(nativeTestPath)).andReturn(files);
+
+ // List tests to include
+ if (mGoogleBenchmarkTest.getIncludeFilters().size() > 0) {
+ String incFilterFlag =
+ mGoogleBenchmarkTest.getFilterFlagForFilters(
+ mGoogleBenchmarkTest.getIncludeFilters());
+ EasyMock.expect(mMockITestDevice.executeShellCommand(EasyMock.contains(incFilterFlag)))
+ .andReturn(incTests);
+ } else {
+ EasyMock.expect(
+ mMockITestDevice.executeShellCommand(
+ EasyMock.not(
+ EasyMock.contains(
+ GoogleBenchmarkTest.GBENCHMARK_FILTER_OPTION))))
+ .andReturn(incTests);
+ }
+ if (mGoogleBenchmarkTest.getExcludeFilters().size() > 0) {
+ // List tests to exclude
+ String excFilterFlag =
+ mGoogleBenchmarkTest.getFilterFlagForFilters(
+ mGoogleBenchmarkTest.getExcludeFilters());
+ EasyMock.expect(mMockITestDevice.executeShellCommand(EasyMock.contains(excFilterFlag)))
+ .andReturn(excTests);
+ }
+ if (filteredTests != null && filteredTests.size() > 0) {
+ // Runningt filtered tests
+ String testFilterFlag = mGoogleBenchmarkTest.getFilterFlagForTests(filteredTests);
+ mMockITestDevice.executeShellCommand(
+ EasyMock.contains(testFilterFlag),
+ EasyMock.same(mMockReceiver),
+ EasyMock.anyLong(),
+ (TimeUnit) EasyMock.anyObject(),
+ EasyMock.anyInt());
+ mMockInvocationListener.testRunStarted("test1", filteredTests.size());
+ mMockInvocationListener.testStarted(mDummyTest);
+ mMockInvocationListener.testEnded(
+ EasyMock.eq(mDummyTest), EasyMock.<HashMap<String, String>>anyObject());
+ mMockInvocationListener.testRunEnded(
+ EasyMock.anyLong(), EasyMock.<HashMap<String, Metric>>anyObject());
+ EasyMock.expectLastCall();
+ }
+ replayMocks();
+
+ mGoogleBenchmarkTest.run(mTestInfo, mMockInvocationListener);
+ verifyMocks();
+ }
+
+ /** Test no matching tests for the filters. */
+ public void testNoMatchingTests() throws DeviceNotAvailableException {
+ Set<String> incFilters = new LinkedHashSet<>(Arrays.asList("X", "Y"));
+ String incTests = "Failed to match any benchmarks against regex: X|Y";
+
+ mGoogleBenchmarkTest.addAllIncludeFilters(incFilters);
+ doTestFilter(incTests, null /* excTests */, null /* testFilter */);
+ }
+
+ /** Test the include filtering of test methods. */
+ public void testIncludeFilter() throws DeviceNotAvailableException {
+ Set<String> incFilters = new LinkedHashSet<>(Arrays.asList("A", "B"));
+ String incTests = "A\nAa\nB\nBb";
+ Set<String> filteredTests = new LinkedHashSet<>(Arrays.asList("A", "Aa", "B", "Bb"));
+
+ mGoogleBenchmarkTest.addAllIncludeFilters(incFilters);
+ doTestFilter(incTests, null /* excTests */, filteredTests);
+ }
+
+ /** Test the exclude filtering of test methods. */
+ public void testExcludeFilter() throws DeviceNotAvailableException {
+ String incTests = "A\nAa\nB\nBb\nC\nCc";
+ Set<String> excFilters = new LinkedHashSet<>(Arrays.asList("Bb", "C"));
+ String excTests = "Bb\nC\nCc";
+ Set<String> filteredTests = new LinkedHashSet<>(Arrays.asList("A", "Aa", "B"));
+
+ mGoogleBenchmarkTest.addAllExcludeFilters(excFilters);
+ doTestFilter(incTests, excTests, filteredTests);
+ }
+
+ /** Test the include & exclude filtering of test methods. */
+ public void testIncludeAndExcludeFilter() throws DeviceNotAvailableException {
+ Set<String> incFilters = new LinkedHashSet<>(Arrays.asList("A", "B"));
+ String incTests = "A\nAa\nB\nBb";
+ Set<String> excFilters = new LinkedHashSet<>(Arrays.asList("Bb", "C"));
+ String excTests = "Bb\nC\nCc";
+ Set<String> filteredTests = new LinkedHashSet<>(Arrays.asList("A", "Aa", "B"));
+
+ mGoogleBenchmarkTest.addAllIncludeFilters(incFilters);
+ mGoogleBenchmarkTest.addAllExcludeFilters(excFilters);
+ doTestFilter(incTests, excTests, filteredTests);
+ }
+
+ /** Test the ITestDescription filter format "class#method". */
+ public void testClearFilter() throws DeviceNotAvailableException {
+ Set<String> incFilters = new LinkedHashSet<>(Arrays.asList("X#A", "X#B"));
+ Set<String> expectedIncFilters = new LinkedHashSet<>(Arrays.asList("A", "B"));
+ mGoogleBenchmarkTest.addAllIncludeFilters(incFilters);
+ assertEquals(expectedIncFilters, mGoogleBenchmarkTest.getIncludeFilters());
+ }
+
+ /** Test behavior for command lines too long to be run by ADB */
+ public void testCommandTooLong() throws DeviceNotAvailableException {
+ String deviceScriptPath = "/data/local/tmp/gbenchmarktest_script.sh";
+ StringBuilder testNameBuilder = new StringBuilder();
+ for (int i = 0; i < GoogleBenchmarkTest.ADB_CMD_CHAR_LIMIT; i++) {
+ testNameBuilder.append("a");
+ }
+ String testName = testNameBuilder.toString();
+ // filter string will be longer than GTest.ADB_CMD_CHAR_LIMIT
+
+ String nativeTestPath = GoogleBenchmarkTest.DEFAULT_TEST_PATH;
+ String testPath = nativeTestPath + "/" + testName;
+ // configure the mock file system to have a single test
+ MockFileUtil.setMockDirContents(mMockITestDevice, nativeTestPath, "test1");
+ EasyMock.expect(mMockITestDevice.doesFileExist(nativeTestPath)).andReturn(true);
+ EasyMock.expect(mMockITestDevice.isDirectory(nativeTestPath)).andReturn(true);
+ EasyMock.expect(mMockITestDevice.isDirectory(testPath)).andReturn(false);
+ EasyMock.expect(mMockITestDevice.executeShellCommand(EasyMock.contains("chmod")))
+ .andReturn("");
+ String[] files = new String[] {testName};
+ EasyMock.expect(mMockITestDevice.getChildren(nativeTestPath)).andReturn(files);
+ // List tests
+ EasyMock.expect(
+ mMockITestDevice.pushString(
+ EasyMock.<String>anyObject(), EasyMock.eq(deviceScriptPath)))
+ .andReturn(Boolean.TRUE);
+ EasyMock.expect(
+ mMockITestDevice.executeShellCommand(
+ EasyMock.eq(String.format("sh %s", deviceScriptPath))))
+ .andReturn("test");
+ mMockITestDevice.deleteFile(deviceScriptPath);
+ // Run tests
+ EasyMock.expect(
+ mMockITestDevice.pushString(
+ EasyMock.<String>anyObject(), EasyMock.eq(deviceScriptPath)))
+ .andReturn(Boolean.TRUE);
+ mMockITestDevice.executeShellCommand(
+ EasyMock.eq(String.format("sh %s", deviceScriptPath)),
+ EasyMock.same(mMockReceiver),
+ EasyMock.anyLong(),
+ (TimeUnit) EasyMock.anyObject(),
+ EasyMock.anyInt());
+ mMockITestDevice.deleteFile(deviceScriptPath);
+ mMockInvocationListener.testRunStarted(testName, 1);
+ mMockInvocationListener.testStarted(mDummyTest);
+ mMockInvocationListener.testEnded(
+ EasyMock.eq(mDummyTest), EasyMock.<HashMap<String, String>>anyObject());
+ mMockInvocationListener.testRunEnded(
+ EasyMock.anyLong(), EasyMock.<HashMap<String, Metric>>anyObject());
+ EasyMock.expectLastCall();
+ replayMocks();
+
+ mGoogleBenchmarkTest.run(mTestInfo, mMockInvocationListener);
+ verifyMocks();
+ }
}
diff --git a/tests/src/com/android/tradefed/testtype/UsbResetTestTest.java b/tests/src/com/android/tradefed/testtype/UsbResetTestTest.java
index 6d166e3..d8c199c 100644
--- a/tests/src/com/android/tradefed/testtype/UsbResetTestTest.java
+++ b/tests/src/com/android/tradefed/testtype/UsbResetTestTest.java
@@ -68,6 +68,7 @@
mTest.run(mTestInfo, null);
verify(usbDevice).reset();
+ verify(mDevice).waitForDeviceOnline();
verify(mDevice).reboot();
}