Merge "GoogleBenchmarkTest: Add "benchmark-filter" option."
diff --git a/test_framework/com/android/tradefed/testtype/GoogleBenchmarkTest.java b/test_framework/com/android/tradefed/testtype/GoogleBenchmarkTest.java
index 163b4fb..373844a 100644
--- a/test_framework/com/android/tradefed/testtype/GoogleBenchmarkTest.java
+++ b/test_framework/com/android/tradefed/testtype/GoogleBenchmarkTest.java
@@ -16,6 +16,7 @@
package com.android.tradefed.testtype;
import com.android.ddmlib.FileListingService;
+import com.android.ddmlib.IShellOutputReceiver;
import com.android.tradefed.config.Option;
import com.android.tradefed.config.OptionClass;
import com.android.tradefed.device.CollectingOutputReceiver;
@@ -26,24 +27,28 @@
import com.android.tradefed.result.ITestInvocationListener;
import com.android.tradefed.util.proto.TfMetricProtoUtil;
+import com.google.common.annotations.VisibleForTesting;
+
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import java.util.concurrent.TimeUnit;
-/**
- * A Test that runs a Google benchmark test package on given device.
- */
+/** A Test that runs a Google benchmark test package on given device. */
@OptionClass(alias = "gbenchmark")
-public class GoogleBenchmarkTest implements IDeviceTest, IRemoteTest {
+public class GoogleBenchmarkTest implements IDeviceTest, IRemoteTest, ITestFilterReceiver {
static final String DEFAULT_TEST_PATH = "/data/benchmarktest";
+ static final String GBENCHMARK_FILTER_OPTION = "--benchmark_filter";
+ static final int ADB_CMD_CHAR_LIMIT = 4000; // May apply to some old device models.
private static final String GBENCHMARK_JSON_OUTPUT_FORMAT = "--benchmark_format=json";
private static final String GBENCHMARK_LIST_TESTS_OPTION = "--benchmark_list_tests=true";
-
private static final List<String> DEFAULT_FILE_EXCLUDE_FILTERS = new ArrayList<>();
static {
@@ -75,6 +80,16 @@
"The maximum time to allow for each benchmark run in ms.", isTimeVal=true)
private long mMaxRunTime = 15 * 60 * 1000;
+ @Option(
+ name = "include-filter",
+ description = "The benchmark (regex) filters used to match benchmarks to include.")
+ private Set<String> mIncludeFilters = new LinkedHashSet<>();
+
+ @Option(
+ name = "exclude-filter",
+ description = "The benchmark (regex) filters used to match benchmarks to exclude.")
+ private Set<String> mExcludeFilters = new LinkedHashSet<>();
+
private ITestDevice mDevice = null;
/**
@@ -168,8 +183,11 @@
}
long startTime = System.currentTimeMillis();
+ Set<String> filteredTests = getFilteredTests(testDevice, root);
+ CLog.d("List that will be used: %s", Arrays.asList(filteredTests));
+
// Count expected number of tests
- int numTests = countExpectedTests(testDevice, root);
+ int numTests = filteredTests.size();
if (numTests == 0) {
CLog.d("No tests to run.");
return;
@@ -180,11 +198,15 @@
GoogleBenchmarkResultParser resultParser = createResultParser(runName, listener);
listener.testRunStarted(runName, numTests);
try {
- String cmd = String.format("%s %s", root, GBENCHMARK_JSON_OUTPUT_FORMAT);
+ String cmd =
+ String.format(
+ "%s%s %s",
+ root,
+ getFilterFlagForTests(filteredTests),
+ GBENCHMARK_JSON_OUTPUT_FORMAT);
CLog.i(String.format("Running google benchmark test on %s: %s",
mDevice.getSerialNumber(), cmd));
- testDevice.executeShellCommand(cmd, outputCollector,
- mMaxRunTime, TimeUnit.MILLISECONDS, 0);
+ executeCommand(testDevice, cmd, outputCollector);
metricMap = resultParser.parse(outputCollector);
} catch (DeviceNotAvailableException e) {
listener.testRunFailed(e.getMessage());
@@ -196,17 +218,60 @@
}
}
- private int countExpectedTests(ITestDevice testDevice, String fullBinaryPath)
+ /**
+ * Returns benchmark tests matching current filters.
+ *
+ * @param testDevice the device on which to run the command
+ * @param fullBinaryPath the full binary path
+ * @return matching benchmark tests.
+ * @throws DeviceNotAvailableException
+ */
+ private Set<String> getFilteredTests(ITestDevice testDevice, String fullBinaryPath)
throws DeviceNotAvailableException {
- if (!testDevice.isExecutable(fullBinaryPath)) {
- CLog.d("%s does not look like an executable", fullBinaryPath);
- return 0;
+ Set<String> filteredTests = getTestsForFilters(testDevice, fullBinaryPath, mIncludeFilters);
+ if (!mExcludeFilters.isEmpty()) {
+ filteredTests.removeAll(
+ getTestsForFilters(testDevice, fullBinaryPath, mExcludeFilters));
}
- String cmd = String.format("%s %s", fullBinaryPath, GBENCHMARK_LIST_TESTS_OPTION);
- String list_output = testDevice.executeShellCommand(cmd);
+ return filteredTests;
+ }
+
+ /**
+ * Returns benchmark tests matching the filters.
+ *
+ * @param testDevice the device on which to run the command
+ * @param fullBinaryPath the full binary path
+ * @param filters filters for matching benchmark tests
+ * @return matching benchmark tests.
+ * @throws DeviceNotAvailableException
+ */
+ private Set<String> getTestsForFilters(
+ ITestDevice testDevice, String fullBinaryPath, Set<String> filters)
+ throws DeviceNotAvailableException {
+ String cmd =
+ String.format(
+ "%s%s %s",
+ fullBinaryPath,
+ getFilterFlagForFilters(filters),
+ GBENCHMARK_LIST_TESTS_OPTION);
+ String list_output = executeCommand(testDevice, cmd, null /* outputReceiver */);
String[] list = list_output.trim().split("\n");
- CLog.d("List that will be used: %s", Arrays.asList(list));
- return list.length;
+ if (noMatchesFound(list)) {
+ list = new String[0];
+ }
+ return new LinkedHashSet<String>(Arrays.asList(list));
+ }
+
+ // ** Returns true if no matches found. */
+ private boolean noMatchesFound(String[] list) {
+ if (list.length == 0) {
+ return true;
+ }
+
+ // A benchmark name is a single word.
+ // A no-match output is "Failed to match any benchmarks ..."
+ // Consider no matches found if the output line has multiple-words.
+ return (list[0].indexOf(' ') >= 0);
}
/**
@@ -262,4 +327,162 @@
}
doRunAllTestsInSubdirectory(testPath, mDevice, listener);
}
+
+ /*
+ * Conforms filters using a {@link TestDescription} format to be recognized by the
+ * GoogleBenchmarkTest executable.
+ */
+ public String cleanFilter(String filter) {
+ Integer index = filter.indexOf('#');
+ if (index >= 0) {
+ // For "class#method", takes the "method" part as the benchmark name.
+ String benchmark = filter.substring(index + 1);
+ if (benchmark.isEmpty()) {
+ CLog.e("Invalid filter %s. Result could be unexpected.", filter);
+ } else {
+ return benchmark;
+ }
+ }
+ return filter;
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public void addIncludeFilter(String filter) {
+ mIncludeFilters.add(cleanFilter(filter));
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public void addAllIncludeFilters(Set<String> filters) {
+ for (String filter : filters) {
+ mIncludeFilters.add(cleanFilter(filter));
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public void addExcludeFilter(String filter) {
+ mExcludeFilters.add(cleanFilter(filter));
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public void addAllExcludeFilters(Set<String> filters) {
+ for (String filter : filters) {
+ mExcludeFilters.add(cleanFilter(filter));
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public Set<String> getIncludeFilters() {
+ return mIncludeFilters;
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public Set<String> getExcludeFilters() {
+ return mExcludeFilters;
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public void clearIncludeFilters() {
+ mIncludeFilters.clear();
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public void clearExcludeFilters() {
+ mExcludeFilters.clear();
+ }
+
+ @VisibleForTesting
+ protected String getFilterFlagForFilters(Set<String> filters) {
+ // Use single filter as only the last "--benchmark_filter" is recognized.
+ StringBuilder filterFlag = new StringBuilder();
+ Iterator<String> iterator = filters.iterator();
+ if (iterator.hasNext()) {
+ filterFlag.append(String.format(" %s=%s", GBENCHMARK_FILTER_OPTION, iterator.next()));
+ while (iterator.hasNext()) {
+ filterFlag.append(String.format("\\|%s", iterator.next()));
+ }
+ }
+ return filterFlag.toString();
+ }
+
+ @VisibleForTesting
+ protected String getFilterFlagForTests(Set<String> fitlererTests) {
+ // Pass the list of tests as filter since BenchmarkTest can't handle negative filters.
+ StringBuilder filterFlag = new StringBuilder();
+ Iterator<String> iterator = fitlererTests.iterator();
+ if (iterator.hasNext()) {
+ // Format benchmark as "^benchmark$" to avoid unintended regex partial matching.
+ filterFlag.append(String.format(" %s=^%s$", GBENCHMARK_FILTER_OPTION, iterator.next()));
+ while (iterator.hasNext()) {
+ filterFlag.append(String.format("\\|^%s$", iterator.next()));
+ }
+ }
+ return filterFlag.toString();
+ }
+
+ /**
+ * Helper method to run a benchmarktest command. If the command is too long to be run directly
+ * by adb, it runs from a temporary script.
+ *
+ * @param testDevice the device on which to run the command
+ * @param cmd the command string to run
+ * @param outputReceiver the output receiver for reading test results
+ * @return shell output if outputReceiver is null
+ */
+ protected String executeCommand(
+ final ITestDevice testDevice,
+ final String cmd,
+ final IShellOutputReceiver outputReceiver)
+ throws DeviceNotAvailableException {
+ // Ensure that command is not too long for adb
+ if (cmd.length() < ADB_CMD_CHAR_LIMIT) {
+ if (outputReceiver == null) {
+ return testDevice.executeShellCommand(cmd);
+ }
+
+ testDevice.executeShellCommand(
+ cmd,
+ outputReceiver,
+ mMaxRunTime /* maxTimeToShellOutputResponse */,
+ TimeUnit.MILLISECONDS,
+ 0 /* retryAttempts */);
+ return null;
+ }
+
+ // Wrap adb shell command in script if command is too long for direct execution
+ return executeCommandByScript(testDevice, cmd, outputReceiver);
+ }
+
+ /** Runs a command from a temporary script. */
+ private String executeCommandByScript(
+ final ITestDevice testDevice,
+ final String cmd,
+ final IShellOutputReceiver outputReceiver)
+ throws DeviceNotAvailableException {
+ String tmpFileDevice = "/data/local/tmp/gbenchmarktest_script.sh";
+ testDevice.pushString(String.format("#!/bin/bash\n%s", cmd), tmpFileDevice);
+ String shellOutput = null;
+ try {
+ if (outputReceiver == null) {
+ shellOutput = testDevice.executeShellCommand(String.format("sh %s", tmpFileDevice));
+ } else {
+ testDevice.executeShellCommand(
+ String.format("sh %s", tmpFileDevice),
+ outputReceiver,
+ mMaxRunTime /* maxTimeToShellOutputResponse */,
+ TimeUnit.MILLISECONDS,
+ 0 /* retry attempts */);
+ }
+ } finally {
+ testDevice.deleteFile(tmpFileDevice);
+ }
+ return shellOutput;
+ }
}
diff --git a/tests/src/com/android/tradefed/testtype/GoogleBenchmarkTestTest.java b/tests/src/com/android/tradefed/testtype/GoogleBenchmarkTestTest.java
index 9fbfa13..51877ce 100644
--- a/tests/src/com/android/tradefed/testtype/GoogleBenchmarkTestTest.java
+++ b/tests/src/com/android/tradefed/testtype/GoogleBenchmarkTestTest.java
@@ -22,14 +22,18 @@
import com.android.tradefed.invoker.TestInformation;
import com.android.tradefed.metrics.proto.MetricMeasurement.Metric;
import com.android.tradefed.result.ITestInvocationListener;
+import com.android.tradefed.result.TestDescription;
import junit.framework.TestCase;
import org.easymock.EasyMock;
+import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
+import java.util.LinkedHashSet;
import java.util.Map;
+import java.util.Set;
import java.util.concurrent.TimeUnit;
/**
@@ -42,6 +46,7 @@
private ITestDevice mMockITestDevice = null;
private GoogleBenchmarkTest mGoogleBenchmarkTest;
private TestInformation mTestInfo;
+ private TestDescription mDummyTest;
/**
* Helper to initialize the various EasyMocks we'll need.
@@ -52,23 +57,28 @@
mMockInvocationListener = EasyMock.createMock(ITestInvocationListener.class);
mMockReceiver = new CollectingOutputReceiver();
mMockITestDevice = EasyMock.createMock(ITestDevice.class);
+ mDummyTest = new TestDescription("Class", "method");
EasyMock.expect(mMockITestDevice.getSerialNumber()).andStubReturn("serial");
- mGoogleBenchmarkTest = new GoogleBenchmarkTest() {
- @Override
- CollectingOutputReceiver createOutputCollector() {
- return mMockReceiver;
- }
- @Override
- GoogleBenchmarkResultParser createResultParser(String runName,
- ITestInvocationListener listener) {
- return new GoogleBenchmarkResultParser(runName, listener) {
+ mGoogleBenchmarkTest =
+ new GoogleBenchmarkTest() {
@Override
- public Map<String, String> parse(CollectingOutputReceiver output) {
- return Collections.emptyMap();
+ CollectingOutputReceiver createOutputCollector() {
+ return mMockReceiver;
+ }
+
+ @Override
+ GoogleBenchmarkResultParser createResultParser(
+ String runName, ITestInvocationListener listener) {
+ return new GoogleBenchmarkResultParser(runName, listener) {
+ @Override
+ public Map<String, String> parse(CollectingOutputReceiver output) {
+ listener.testStarted(mDummyTest);
+ listener.testEnded(mDummyTest, Collections.emptyMap());
+ return Collections.emptyMap();
+ }
+ };
}
};
- }
- };
mGoogleBenchmarkTest.setDevice(mMockITestDevice);
mTestInfo = TestInformation.newBuilder().build();
}
@@ -108,13 +118,11 @@
mMockITestDevice.executeShellCommand(EasyMock.contains(test2), EasyMock.same(mMockReceiver),
EasyMock.anyLong(), (TimeUnit)EasyMock.anyObject(), EasyMock.anyInt());
- EasyMock.expect(mMockITestDevice.isExecutable(nativeTestPath + "/test1")).andReturn(true);
EasyMock.expect(
mMockITestDevice.executeShellCommand(
String.format(
"%s/test1 --benchmark_list_tests=true", nativeTestPath)))
.andReturn("method1\nmethod2\nmethod3");
- EasyMock.expect(mMockITestDevice.isExecutable(nativeTestPath + "/test2")).andReturn(true);
EasyMock.expect(
mMockITestDevice.executeShellCommand(
@@ -122,7 +130,13 @@
"%s/test2 --benchmark_list_tests=true", nativeTestPath)))
.andReturn("method1\nmethod2\n");
mMockInvocationListener.testRunStarted(test1, 3);
+ mMockInvocationListener.testStarted(mDummyTest);
+ mMockInvocationListener.testEnded(
+ EasyMock.eq(mDummyTest), EasyMock.<HashMap<String, String>>anyObject());
mMockInvocationListener.testRunStarted(test2, 2);
+ mMockInvocationListener.testStarted(mDummyTest);
+ mMockInvocationListener.testEnded(
+ EasyMock.eq(mDummyTest), EasyMock.<HashMap<String, String>>anyObject());
mMockInvocationListener.testRunEnded(
EasyMock.anyLong(), EasyMock.<HashMap<String, Metric>>anyObject());
EasyMock.expectLastCall().times(2);
@@ -185,20 +199,24 @@
EasyMock.anyLong(), (TimeUnit)EasyMock.anyObject(), EasyMock.anyInt());
mMockITestDevice.executeShellCommand(EasyMock.contains(test2), EasyMock.same(mMockReceiver),
EasyMock.anyLong(), (TimeUnit)EasyMock.anyObject(), EasyMock.anyInt());
- EasyMock.expect(mMockITestDevice.isExecutable(nativeTestPath + "/test1")).andReturn(true);
EasyMock.expect(
mMockITestDevice.executeShellCommand(
String.format(
"%s/test1 --benchmark_list_tests=true", nativeTestPath)))
.andReturn("\nmethod1\nmethod2\nmethod3\n\n");
- EasyMock.expect(mMockITestDevice.isExecutable(nativeTestPath + "/test2")).andReturn(true);
EasyMock.expect(
mMockITestDevice.executeShellCommand(
String.format(
"%s/test2 --benchmark_list_tests=true", nativeTestPath)))
.andReturn("method1\nmethod2\n");
mMockInvocationListener.testRunStarted(test1, 3);
+ mMockInvocationListener.testStarted(mDummyTest);
+ mMockInvocationListener.testEnded(
+ EasyMock.eq(mDummyTest), EasyMock.<HashMap<String, String>>anyObject());
mMockInvocationListener.testRunStarted(test2, 2);
+ mMockInvocationListener.testStarted(mDummyTest);
+ mMockInvocationListener.testEnded(
+ EasyMock.eq(mDummyTest), EasyMock.<HashMap<String, String>>anyObject());
mMockInvocationListener.testRunEnded(
EasyMock.anyLong(), EasyMock.<HashMap<String, Metric>>anyObject());
EasyMock.expectLastCall().times(2);
@@ -226,7 +244,6 @@
.andReturn("");
mMockITestDevice.executeShellCommand(EasyMock.contains(test1), EasyMock.same(mMockReceiver),
EasyMock.anyLong(), (TimeUnit)EasyMock.anyObject(), EasyMock.anyInt());
- EasyMock.expect(mMockITestDevice.isExecutable(nativeTestPath + "/test1")).andReturn(true);
EasyMock.expect(
mMockITestDevice.executeShellCommand(
String.format(
@@ -234,6 +251,9 @@
.andReturn("method1\nmethod2\nmethod3");
// Expect reportName instead of test name
mMockInvocationListener.testRunStarted(reportName, 3);
+ mMockInvocationListener.testStarted(mDummyTest);
+ mMockInvocationListener.testEnded(
+ EasyMock.eq(mDummyTest), EasyMock.<HashMap<String, String>>anyObject());
mMockInvocationListener.testRunEnded(
EasyMock.anyLong(), EasyMock.<HashMap<String, Metric>>anyObject());
EasyMock.expectLastCall();
@@ -260,7 +280,6 @@
mMockITestDevice.executeShellCommand(EasyMock.contains(test1), EasyMock.same(mMockReceiver),
EasyMock.anyLong(), (TimeUnit)EasyMock.anyObject(), EasyMock.anyInt());
EasyMock.expectLastCall().andThrow(new DeviceNotAvailableException("dnae", "serial"));
- EasyMock.expect(mMockITestDevice.isExecutable(nativeTestPath + "/test1")).andReturn(true);
EasyMock.expect(
mMockITestDevice.executeShellCommand(
String.format(
@@ -322,4 +341,217 @@
assertFalse(mGoogleBenchmarkTest.shouldSkipFile("/some/path/file/test.txt"));
}
+
+ /** Test getFilterFlagForFilters. */
+ public void testGetFilterFlagForFilters() {
+ Set<String> filters = new LinkedHashSet<>(Arrays.asList("filter1", "filter2"));
+ String filterFlag = mGoogleBenchmarkTest.getFilterFlagForFilters(filters);
+ assertEquals(
+ String.format(
+ " %s=%s",
+ GoogleBenchmarkTest.GBENCHMARK_FILTER_OPTION, "filter1\\|filter2"),
+ filterFlag);
+ }
+
+ /** Test getFilterFlagForFilters - no filters. */
+ public void testGetFilterFlagForFilters_noFilters() {
+ Set<String> filters = new LinkedHashSet<>();
+ String filterFlag = mGoogleBenchmarkTest.getFilterFlagForFilters(filters);
+ assertEquals("", filterFlag);
+ }
+
+ /** Test getFilterFlagForTests. */
+ public void testGetFilterFlagForTests() {
+ Set<String> tests = new LinkedHashSet<>(Arrays.asList("test1", "test2"));
+ String filterFlag = mGoogleBenchmarkTest.getFilterFlagForTests(tests);
+ assertEquals(
+ String.format(
+ " %s=%s",
+ GoogleBenchmarkTest.GBENCHMARK_FILTER_OPTION, "^test1$\\|^test2$"),
+ filterFlag);
+ }
+
+ /** Test getFilterFlagForTests - no tests. */
+ public void testGetFilterFlagForTests_noFilters() {
+ Set<String> tests = new LinkedHashSet<>();
+ String filterFlag = mGoogleBenchmarkTest.getFilterFlagForTests(tests);
+ assertEquals("", filterFlag);
+ }
+
+ /**
+ * Helper function to do the actual filtering test.
+ *
+ * @param incFilter filter flag for querying tests to include
+ * @param incTests tests to include
+ * @param excFilter filter flag for querying tests to exclude
+ * @param excTests tests to exclude
+ * @param testFilter filter flag for running tests
+ * @throws DeviceNotAvailableException
+ */
+ private void doTestFilter(String incTests, String excTests, Set<String> filteredTests)
+ throws DeviceNotAvailableException {
+ String nativeTestPath = GoogleBenchmarkTest.DEFAULT_TEST_PATH;
+ String testPath = nativeTestPath + "/test1";
+ // configure the mock file system to have a single test
+ MockFileUtil.setMockDirContents(mMockITestDevice, nativeTestPath, "test1");
+ EasyMock.expect(mMockITestDevice.doesFileExist(nativeTestPath)).andReturn(true);
+ EasyMock.expect(mMockITestDevice.isDirectory(nativeTestPath)).andReturn(true);
+ EasyMock.expect(mMockITestDevice.isDirectory(testPath)).andReturn(false);
+ EasyMock.expect(mMockITestDevice.executeShellCommand(EasyMock.contains("chmod")))
+ .andReturn("");
+ String[] files = new String[] {"test1"};
+ EasyMock.expect(mMockITestDevice.getChildren(nativeTestPath)).andReturn(files);
+
+ // List tests to include
+ if (mGoogleBenchmarkTest.getIncludeFilters().size() > 0) {
+ String incFilterFlag =
+ mGoogleBenchmarkTest.getFilterFlagForFilters(
+ mGoogleBenchmarkTest.getIncludeFilters());
+ EasyMock.expect(mMockITestDevice.executeShellCommand(EasyMock.contains(incFilterFlag)))
+ .andReturn(incTests);
+ } else {
+ EasyMock.expect(
+ mMockITestDevice.executeShellCommand(
+ EasyMock.not(
+ EasyMock.contains(
+ GoogleBenchmarkTest.GBENCHMARK_FILTER_OPTION))))
+ .andReturn(incTests);
+ }
+ if (mGoogleBenchmarkTest.getExcludeFilters().size() > 0) {
+ // List tests to exclude
+ String excFilterFlag =
+ mGoogleBenchmarkTest.getFilterFlagForFilters(
+ mGoogleBenchmarkTest.getExcludeFilters());
+ EasyMock.expect(mMockITestDevice.executeShellCommand(EasyMock.contains(excFilterFlag)))
+ .andReturn(excTests);
+ }
+ if (filteredTests != null && filteredTests.size() > 0) {
+ // Runningt filtered tests
+ String testFilterFlag = mGoogleBenchmarkTest.getFilterFlagForTests(filteredTests);
+ mMockITestDevice.executeShellCommand(
+ EasyMock.contains(testFilterFlag),
+ EasyMock.same(mMockReceiver),
+ EasyMock.anyLong(),
+ (TimeUnit) EasyMock.anyObject(),
+ EasyMock.anyInt());
+ mMockInvocationListener.testRunStarted("test1", filteredTests.size());
+ mMockInvocationListener.testStarted(mDummyTest);
+ mMockInvocationListener.testEnded(
+ EasyMock.eq(mDummyTest), EasyMock.<HashMap<String, String>>anyObject());
+ mMockInvocationListener.testRunEnded(
+ EasyMock.anyLong(), EasyMock.<HashMap<String, Metric>>anyObject());
+ EasyMock.expectLastCall();
+ }
+ replayMocks();
+
+ mGoogleBenchmarkTest.run(mTestInfo, mMockInvocationListener);
+ verifyMocks();
+ }
+
+ /** Test no matching tests for the filters. */
+ public void testNoMatchingTests() throws DeviceNotAvailableException {
+ Set<String> incFilters = new LinkedHashSet<>(Arrays.asList("X", "Y"));
+ String incTests = "Failed to match any benchmarks against regex: X|Y";
+
+ mGoogleBenchmarkTest.addAllIncludeFilters(incFilters);
+ doTestFilter(incTests, null /* excTests */, null /* testFilter */);
+ }
+
+ /** Test the include filtering of test methods. */
+ public void testIncludeFilter() throws DeviceNotAvailableException {
+ Set<String> incFilters = new LinkedHashSet<>(Arrays.asList("A", "B"));
+ String incTests = "A\nAa\nB\nBb";
+ Set<String> filteredTests = new LinkedHashSet<>(Arrays.asList("A", "Aa", "B", "Bb"));
+
+ mGoogleBenchmarkTest.addAllIncludeFilters(incFilters);
+ doTestFilter(incTests, null /* excTests */, filteredTests);
+ }
+
+ /** Test the exclude filtering of test methods. */
+ public void testExcludeFilter() throws DeviceNotAvailableException {
+ String incTests = "A\nAa\nB\nBb\nC\nCc";
+ Set<String> excFilters = new LinkedHashSet<>(Arrays.asList("Bb", "C"));
+ String excTests = "Bb\nC\nCc";
+ Set<String> filteredTests = new LinkedHashSet<>(Arrays.asList("A", "Aa", "B"));
+
+ mGoogleBenchmarkTest.addAllExcludeFilters(excFilters);
+ doTestFilter(incTests, excTests, filteredTests);
+ }
+
+ /** Test the include & exclude filtering of test methods. */
+ public void testIncludeAndExcludeFilter() throws DeviceNotAvailableException {
+ Set<String> incFilters = new LinkedHashSet<>(Arrays.asList("A", "B"));
+ String incTests = "A\nAa\nB\nBb";
+ Set<String> excFilters = new LinkedHashSet<>(Arrays.asList("Bb", "C"));
+ String excTests = "Bb\nC\nCc";
+ Set<String> filteredTests = new LinkedHashSet<>(Arrays.asList("A", "Aa", "B"));
+
+ mGoogleBenchmarkTest.addAllIncludeFilters(incFilters);
+ mGoogleBenchmarkTest.addAllExcludeFilters(excFilters);
+ doTestFilter(incTests, excTests, filteredTests);
+ }
+
+ /** Test the ITestDescription filter format "class#method". */
+ public void testClearFilter() throws DeviceNotAvailableException {
+ Set<String> incFilters = new LinkedHashSet<>(Arrays.asList("X#A", "X#B"));
+ Set<String> expectedIncFilters = new LinkedHashSet<>(Arrays.asList("A", "B"));
+ mGoogleBenchmarkTest.addAllIncludeFilters(incFilters);
+ assertEquals(expectedIncFilters, mGoogleBenchmarkTest.getIncludeFilters());
+ }
+
+ /** Test behavior for command lines too long to be run by ADB */
+ public void testCommandTooLong() throws DeviceNotAvailableException {
+ String deviceScriptPath = "/data/local/tmp/gbenchmarktest_script.sh";
+ StringBuilder testNameBuilder = new StringBuilder();
+ for (int i = 0; i < GoogleBenchmarkTest.ADB_CMD_CHAR_LIMIT; i++) {
+ testNameBuilder.append("a");
+ }
+ String testName = testNameBuilder.toString();
+ // filter string will be longer than GTest.ADB_CMD_CHAR_LIMIT
+
+ String nativeTestPath = GoogleBenchmarkTest.DEFAULT_TEST_PATH;
+ String testPath = nativeTestPath + "/" + testName;
+ // configure the mock file system to have a single test
+ MockFileUtil.setMockDirContents(mMockITestDevice, nativeTestPath, "test1");
+ EasyMock.expect(mMockITestDevice.doesFileExist(nativeTestPath)).andReturn(true);
+ EasyMock.expect(mMockITestDevice.isDirectory(nativeTestPath)).andReturn(true);
+ EasyMock.expect(mMockITestDevice.isDirectory(testPath)).andReturn(false);
+ EasyMock.expect(mMockITestDevice.executeShellCommand(EasyMock.contains("chmod")))
+ .andReturn("");
+ String[] files = new String[] {testName};
+ EasyMock.expect(mMockITestDevice.getChildren(nativeTestPath)).andReturn(files);
+ // List tests
+ EasyMock.expect(
+ mMockITestDevice.pushString(
+ EasyMock.<String>anyObject(), EasyMock.eq(deviceScriptPath)))
+ .andReturn(Boolean.TRUE);
+ EasyMock.expect(
+ mMockITestDevice.executeShellCommand(
+ EasyMock.eq(String.format("sh %s", deviceScriptPath))))
+ .andReturn("test");
+ mMockITestDevice.deleteFile(deviceScriptPath);
+ // Run tests
+ EasyMock.expect(
+ mMockITestDevice.pushString(
+ EasyMock.<String>anyObject(), EasyMock.eq(deviceScriptPath)))
+ .andReturn(Boolean.TRUE);
+ mMockITestDevice.executeShellCommand(
+ EasyMock.eq(String.format("sh %s", deviceScriptPath)),
+ EasyMock.same(mMockReceiver),
+ EasyMock.anyLong(),
+ (TimeUnit) EasyMock.anyObject(),
+ EasyMock.anyInt());
+ mMockITestDevice.deleteFile(deviceScriptPath);
+ mMockInvocationListener.testRunStarted(testName, 1);
+ mMockInvocationListener.testStarted(mDummyTest);
+ mMockInvocationListener.testEnded(
+ EasyMock.eq(mDummyTest), EasyMock.<HashMap<String, String>>anyObject());
+ mMockInvocationListener.testRunEnded(
+ EasyMock.anyLong(), EasyMock.<HashMap<String, Metric>>anyObject());
+ EasyMock.expectLastCall();
+ replayMocks();
+
+ mGoogleBenchmarkTest.run(mTestInfo, mMockInvocationListener);
+ verifyMocks();
+ }
}