Merge "GTestRunner: Detect and report duplicate tests for GTest."
diff --git a/test_framework/com/android/tradefed/testtype/GTest.java b/test_framework/com/android/tradefed/testtype/GTest.java
index 444a278..41c5dac 100644
--- a/test_framework/com/android/tradefed/testtype/GTest.java
+++ b/test_framework/com/android/tradefed/testtype/GTest.java
@@ -428,6 +428,7 @@
// Insert the coverage listener if code coverage collection is enabled.
listener = addNativeCoverageListenerIfEnabled(listener);
listener = addClangCoverageListenerIfEnabled(listener);
+ listener = getGTestListener(listener);
NativeCodeCoverageFlusher flusher =
new NativeCodeCoverageFlusher(mDevice, getCoverageOptions().getCoverageProcesses());
diff --git a/test_framework/com/android/tradefed/testtype/GTestBase.java b/test_framework/com/android/tradefed/testtype/GTestBase.java
index 194199c..3ee7179 100644
--- a/test_framework/com/android/tradefed/testtype/GTestBase.java
+++ b/test_framework/com/android/tradefed/testtype/GTestBase.java
@@ -163,6 +163,11 @@
+ "the same name as the binary with the .json extension.")
private String mTestFilterKey = null;
+ @Option(
+ name = "disable-duplicate-test-check",
+ description = "If set to true, it will not check that a method is only run once.")
+ private boolean mDisableDuplicateCheck = false;
+
// GTest flags...
protected static final String GTEST_FLAG_PRINT_TIME = "--gtest_print_time";
protected static final String GTEST_FLAG_FILTER = "--gtest_filter";
@@ -673,4 +678,17 @@
}
return new CoverageOptions();
}
+
+ /**
+ * Returns the {@link GTestListener} that provides extra debugging info, like detects and
+ * reports duplicate tests if mDisabledDuplicateCheck is false. Otherwise, returns the passed-in
+ * listener.
+ */
+ protected ITestInvocationListener getGTestListener(ITestInvocationListener listener) {
+ if (mDisableDuplicateCheck) {
+ return listener;
+ }
+
+ return new GTestListener(listener);
+ }
}
diff --git a/test_framework/com/android/tradefed/testtype/GTestListener.java b/test_framework/com/android/tradefed/testtype/GTestListener.java
new file mode 100644
index 0000000..c967d06
--- /dev/null
+++ b/test_framework/com/android/tradefed/testtype/GTestListener.java
@@ -0,0 +1,63 @@
+/*
+ * Copyright (C) 2020 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.android.tradefed.testtype;
+
+import com.android.tradefed.metrics.proto.MetricMeasurement.Metric;
+import com.android.tradefed.result.FailureDescription;
+import com.android.tradefed.result.ITestInvocationListener;
+import com.android.tradefed.result.ResultForwarder;
+import com.android.tradefed.result.TestDescription;
+import com.android.tradefed.result.proto.TestRecordProto.FailureStatus;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Set;
+
+/**
+ * Internal listener to Trade Federation for {@link GTest}. Detect and report if duplciated tests
+ * are run.
+ */
+final class GTestListener extends ResultForwarder {
+
+ private Set<TestDescription> mTests = new HashSet<>();
+ private Set<TestDescription> mDuplicateTests = new HashSet<>();
+
+ public GTestListener(ITestInvocationListener... listeners) {
+ super(listeners);
+ }
+
+ @Override
+ public void testStarted(TestDescription test, long startTime) {
+ super.testStarted(test, startTime);
+ if (!mTests.add(test)) {
+ mDuplicateTests.add(test);
+ }
+ }
+
+ @Override
+ public void testRunEnded(long elapsedTime, HashMap<String, Metric> runMetrics) {
+ if (!mDuplicateTests.isEmpty()) {
+ FailureDescription error =
+ FailureDescription.create(
+ String.format(
+ "The following tests ran more than once: %s.",
+ mDuplicateTests));
+ error.setFailureStatus(FailureStatus.TEST_FAILURE);
+ super.testRunFailed(error);
+ }
+ super.testRunEnded(elapsedTime, runMetrics);
+ }
+}
diff --git a/test_framework/com/android/tradefed/testtype/HostGTest.java b/test_framework/com/android/tradefed/testtype/HostGTest.java
index c4958b7..43d599e 100644
--- a/test_framework/com/android/tradefed/testtype/HostGTest.java
+++ b/test_framework/com/android/tradefed/testtype/HostGTest.java
@@ -246,6 +246,7 @@
String.format("%s is not executable!", gTestFile.getAbsolutePath()));
}
+ listener = getGTestListener(listener);
// TODO: Need to support XML test output based on isEnableXmlOutput
IShellOutputReceiver resultParser = createResultParser(gTestFile.getName(), listener);
String flags = getAllGTestFlags(gTestFile.getName());
diff --git a/tests/src/com/android/tradefed/UnitTests.java b/tests/src/com/android/tradefed/UnitTests.java
index ad894a9..ed1ba81 100644
--- a/tests/src/com/android/tradefed/UnitTests.java
+++ b/tests/src/com/android/tradefed/UnitTests.java
@@ -257,6 +257,7 @@
import com.android.tradefed.testtype.DeviceTestSuiteTest;
import com.android.tradefed.testtype.FakeTestTest;
import com.android.tradefed.testtype.GTestListTestParserTest;
+import com.android.tradefed.testtype.GTestListenerTest;
import com.android.tradefed.testtype.GTestResultParserTest;
import com.android.tradefed.testtype.GTestTest;
import com.android.tradefed.testtype.GTestXmlResultParserTest;
@@ -728,6 +729,7 @@
GoogleBenchmarkResultParserTest.class,
GoogleBenchmarkTestTest.class,
GTestListTestParserTest.class,
+ GTestListenerTest.class,
GTestResultParserTest.class,
GTestTest.class,
GTestXmlResultParserTest.class,
diff --git a/tests/src/com/android/tradefed/testtype/GTestListenerTest.java b/tests/src/com/android/tradefed/testtype/GTestListenerTest.java
new file mode 100644
index 0000000..392ee75
--- /dev/null
+++ b/tests/src/com/android/tradefed/testtype/GTestListenerTest.java
@@ -0,0 +1,132 @@
+/*
+ * Copyright (C) 2020 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.android.tradefed.testtype;
+
+import static org.junit.Assert.assertTrue;
+
+import com.android.tradefed.device.DeviceNotAvailableException;
+import com.android.tradefed.metrics.proto.MetricMeasurement.Metric;
+import com.android.tradefed.result.FailureDescription;
+import com.android.tradefed.result.ITestInvocationListener;
+import com.android.tradefed.result.TestDescription;
+import com.android.tradefed.result.proto.TestRecordProto.FailureStatus;
+
+import org.easymock.Capture;
+import org.easymock.EasyMock;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+import java.util.HashMap;
+
+/** Unit tests for {@link GTestListener}. */
+@RunWith(JUnit4.class)
+public class GTestListenerTest {
+ private ITestInvocationListener mMockListener = null;
+
+ /** Helper to initialize the various EasyMocks we'll need. */
+ @Before
+ public void setUp() throws Exception {
+ mMockListener = EasyMock.createMock(ITestInvocationListener.class);
+ }
+
+ /** Helper that replays all mocks. */
+ private void replayMocks() {
+ EasyMock.replay(mMockListener);
+ }
+
+ /** Helper that verifies all mocks. */
+ private void verifyMocks() {
+ EasyMock.verify(mMockListener);
+ }
+
+ /** Verify test passes without duplicate tests */
+ @Test
+ public void testNoDuplicateTests() throws DeviceNotAvailableException {
+ String moduleName = "testNoDuplicateTest";
+ String testClass = "testClass";
+ String testName1 = "testName1";
+ TestDescription testId1 = new TestDescription(testClass, testName1);
+ String testName2 = "testName2";
+ TestDescription testId2 = new TestDescription(testClass, testName2);
+
+ mMockListener.testRunStarted(EasyMock.eq(moduleName), EasyMock.eq(2));
+ mMockListener.testStarted(EasyMock.eq(testId1), EasyMock.anyLong());
+ mMockListener.testEnded(
+ EasyMock.eq(testId1),
+ EasyMock.anyLong(),
+ (HashMap<String, Metric>) EasyMock.anyObject());
+ mMockListener.testStarted(EasyMock.eq(testId2), EasyMock.anyLong());
+ mMockListener.testEnded(
+ EasyMock.eq(testId2),
+ EasyMock.anyLong(),
+ (HashMap<String, Metric>) EasyMock.anyObject());
+ mMockListener.testRunEnded(
+ EasyMock.anyLong(), (HashMap<String, Metric>) EasyMock.anyObject());
+ replayMocks();
+
+ HashMap<String, Metric> emptyMap = new HashMap<>();
+ GTestListener listener = new GTestListener(mMockListener);
+ listener.testRunStarted(moduleName, 2);
+ listener.testStarted(testId1);
+ listener.testEnded(testId1, emptyMap);
+ listener.testStarted(testId2);
+ listener.testEnded(testId2, emptyMap);
+ listener.testRunEnded(0, emptyMap);
+ verifyMocks();
+ }
+
+ /** Verify test with duplicate tests fails if option enabled */
+ @Test
+ public void testDuplicateTestsFailsWithOptionEnabled() throws DeviceNotAvailableException {
+ String moduleName = "testWithDuplicateTests";
+ String testClass = "testClass";
+ String testName1 = "testName1";
+ String duplicateTestsMessage = "The following tests ran more than once: ";
+ TestDescription testId1 = new TestDescription(testClass, testName1);
+
+ mMockListener.testRunStarted(EasyMock.eq(moduleName), EasyMock.eq(2));
+ mMockListener.testStarted(EasyMock.eq(testId1), EasyMock.anyLong());
+ mMockListener.testEnded(
+ EasyMock.eq(testId1),
+ EasyMock.anyLong(),
+ (HashMap<String, Metric>) EasyMock.anyObject());
+ mMockListener.testStarted(EasyMock.eq(testId1), EasyMock.anyLong());
+ mMockListener.testEnded(
+ EasyMock.eq(testId1),
+ EasyMock.anyLong(),
+ (HashMap<String, Metric>) EasyMock.anyObject());
+ Capture<FailureDescription> capturedFailureDescription = new Capture<>();
+ mMockListener.testRunFailed(EasyMock.capture(capturedFailureDescription));
+ mMockListener.testRunEnded(
+ EasyMock.anyLong(), (HashMap<String, Metric>) EasyMock.anyObject());
+ replayMocks();
+
+ HashMap<String, Metric> emptyMap = new HashMap<>();
+ GTestListener listener = new GTestListener(mMockListener);
+ listener.testRunStarted(moduleName, 2);
+ listener.testStarted(testId1);
+ listener.testEnded(testId1, emptyMap);
+ listener.testStarted(testId1);
+ listener.testEnded(testId1, emptyMap);
+ listener.testRunEnded(0, emptyMap);
+ FailureDescription failureDescription = capturedFailureDescription.getValue();
+ assertTrue(failureDescription.getErrorMessage().contains(duplicateTestsMessage));
+ assertTrue(FailureStatus.TEST_FAILURE.equals(failureDescription.getFailureStatus()));
+ verifyMocks();
+ }
+}