Merge "Create a SafeCleanerRule so cleanup failures don't hide test failures."
diff --git a/tests/autofillservice/src/android/autofillservice/cts/AutoFillServiceTestCase.java b/tests/autofillservice/src/android/autofillservice/cts/AutoFillServiceTestCase.java
index 74e168b..48b190b 100644
--- a/tests/autofillservice/src/android/autofillservice/cts/AutoFillServiceTestCase.java
+++ b/tests/autofillservice/src/android/autofillservice/cts/AutoFillServiceTestCase.java
@@ -39,8 +39,6 @@
import org.junit.Rule;
import org.junit.runner.RunWith;
-import java.util.List;
-
/**
* Base class for all other tests.
*/
@@ -62,6 +60,12 @@
public final RequiredFeatureRule mRequiredFeatureRule =
new RequiredFeatureRule(PackageManager.FEATURE_AUTOFILL);
+ @Rule
+ public final SafeCleanerRule mSafeCleanerRule = new SafeCleanerRule()
+ .run(() -> sReplier.assertNumberUnhandledFillRequests(0))
+ .run(() -> sReplier.assertNumberUnhandledSaveRequests(0))
+ .add(() -> { return sReplier.getExceptions(); });
+
protected final Context mContext;
protected final String mPackageName;
/**
@@ -130,24 +134,6 @@
}
}
- // TODO: we shouldn't throw exceptions on @After / @AfterClass because if the test failed, these
- // exceptions would mask the real cause. A better approach might be using a @Rule or some other
- // visitor pattern.
- @After
- public void assertNothingIsPending() throws Throwable {
- final MultipleExceptionsCatcher catcher = new MultipleExceptionsCatcher()
- .run(() -> sReplier.assertNumberUnhandledFillRequests(0))
- .run(() -> sReplier.assertNumberUnhandledSaveRequests(0));
-
- final List<Exception> replierExceptions = sReplier.getExceptions();
- if (replierExceptions != null) {
- for (Exception e : replierExceptions) {
- catcher.add(e);
- }
- }
- catcher.throwIfAny();
- }
-
@After
public void ignoreFurtherRequests() {
InstrumentedAutoFillService.setIgnoreUnexpectedRequests(true);
diff --git a/tests/autofillservice/src/android/autofillservice/cts/InstrumentedAutoFillService.java b/tests/autofillservice/src/android/autofillservice/cts/InstrumentedAutoFillService.java
index fbdec66..7532a13 100644
--- a/tests/autofillservice/src/android/autofillservice/cts/InstrumentedAutoFillService.java
+++ b/tests/autofillservice/src/android/autofillservice/cts/InstrumentedAutoFillService.java
@@ -249,7 +249,7 @@
private final BlockingQueue<FillRequest> mFillRequests = new LinkedBlockingQueue<>();
private final BlockingQueue<SaveRequest> mSaveRequests = new LinkedBlockingQueue<>();
- private List<Exception> mExceptions;
+ private List<Throwable> mExceptions;
private Replier() {
}
@@ -263,11 +263,11 @@
/**
* Gets the exceptions thrown asynchronously, if any.
*/
- @Nullable List<Exception> getExceptions() {
+ @Nullable List<Throwable> getExceptions() {
return mExceptions;
}
- private void addException(@Nullable Exception e) {
+ private void addException(@Nullable Throwable e) {
if (e == null) return;
if (mExceptions == null) {
diff --git a/tests/autofillservice/src/android/autofillservice/cts/MultipleExceptionsCatcher.java b/tests/autofillservice/src/android/autofillservice/cts/MultipleExceptionsCatcher.java
deleted file mode 100644
index 52c3bcc..0000000
--- a/tests/autofillservice/src/android/autofillservice/cts/MultipleExceptionsCatcher.java
+++ /dev/null
@@ -1,91 +0,0 @@
-/*
- * Copyright (C) 2017 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package android.autofillservice.cts;
-
-import android.support.annotation.NonNull;
-import android.support.annotation.Nullable;
-import android.util.Log;
-
-import java.io.IOException;
-import java.io.PrintWriter;
-import java.io.StringWriter;
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * Helper used to catch multiple exceptions that might have happened in a test case.
- */
-// TODO: move to common CTS code (and add test cases to it)
-public final class MultipleExceptionsCatcher {
-
- private static final String TAG = "MultipleExceptionsCatcher";
-
- private final List<Throwable> mThrowables = new ArrayList<>();
-
- /**
- * Runs {@code r} postponing any thrown exception to {@link #throwIfAny()}.
- */
- public MultipleExceptionsCatcher run(@NonNull Runnable r) {
- try {
- r.run();
- } catch (Throwable t) {
- mThrowables.add(t);
- }
- return this;
- }
-
- /**
- * Adds an exception - if it's not {@code null} to the exceptions thrown by
- * {@link #throwIfAny()}.
- */
- public MultipleExceptionsCatcher add(@Nullable Throwable t) {
- if (t != null) {
- mThrowables.add(t);
- }
- return this;
- }
-
- /**
- * Throws one exception merging all exceptions thrown or added so far, if any.
- */
- public void throwIfAny() throws Throwable {
- if (mThrowables.isEmpty()) return;
-
- final int numberExceptions = mThrowables.size();
- if (numberExceptions == 1) {
- throw mThrowables.get(0);
- }
-
- String msg = "D'OH!";
- try {
- try (StringWriter sw = new StringWriter(); PrintWriter pw = new PrintWriter(sw)) {
- sw.write("Caught " + numberExceptions + " exceptions\n");
- for (int i = 0; i < numberExceptions; i++) {
- sw.write("\n---- Begin of exception #" + (i + 1) + " ----\n");
- final Throwable exception = mThrowables.get(i);
- exception.printStackTrace(pw);
- sw.write("---- End of exception #" + (i + 1) + " ----\n\n");
- }
- msg = sw.toString();
- }
- } catch (IOException e) {
- // ignore close() errors - should not happen...
- Log.e(TAG, "Exception closing StringWriter: " + e);
- }
- throw new AssertionError(msg);
- }
-}
diff --git a/tests/autofillservice/src/android/autofillservice/cts/SafeCleanerRule.java b/tests/autofillservice/src/android/autofillservice/cts/SafeCleanerRule.java
new file mode 100644
index 0000000..bceb11b
--- /dev/null
+++ b/tests/autofillservice/src/android/autofillservice/cts/SafeCleanerRule.java
@@ -0,0 +1,142 @@
+/*
+ * Copyright (C) 2017 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package android.autofillservice.cts;
+
+import android.support.annotation.NonNull;
+import android.util.Log;
+
+import org.junit.rules.TestRule;
+import org.junit.runner.Description;
+import org.junit.runners.model.Statement;
+
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.io.StringWriter;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Callable;
+
+/**
+ * Rule used to safely run clean up code after a test is finished, so that exceptions thrown by
+ * the cleanup code don't hide exception thrown by the test body
+ */
+// TODO: move to common CTS code
+public final class SafeCleanerRule implements TestRule {
+
+ private static final String TAG = "SafeCleanerRule";
+
+ private final List<Runnable> mCleaners = new ArrayList<>();
+ private final List<Callable<List<Throwable>>> mExtraThrowables = new ArrayList<>();
+ private final List<Throwable> mThrowables = new ArrayList<>();
+
+ /**
+ * Runs {@code cleaner} after the test is finished, catching any {@link Throwable} thrown by it.
+ */
+ public SafeCleanerRule run(@NonNull Runnable cleaner) {
+ mCleaners.add(cleaner);
+ return this;
+ }
+
+ /**
+ * Adds exceptions directly.
+ *
+ * <p>Typically used when exceptions were caught asychronously during the test execution.
+ */
+ public SafeCleanerRule add(@NonNull Callable<List<Throwable>> exceptions) {
+ mExtraThrowables.add(exceptions);
+ return this;
+ }
+
+ @Override
+ public Statement apply(Statement base, Description description) {
+ return new Statement() {
+ @Override
+ public void evaluate() throws Throwable {
+ // First run the test
+ try {
+ base.evaluate();
+ } catch (Throwable t) {
+ Log.w(TAG, "Adding exception from main test");
+ mThrowables.add(t);
+ }
+
+ // Then the cleanup runners
+ for (Runnable runner : mCleaners) {
+ try {
+ runner.run();
+ } catch (Throwable t) {
+ Log.w(TAG, "Adding exception from cleaner");
+ mThrowables.add(t);
+ }
+ }
+
+ // And finally add the extra exceptions
+ for (Callable<List<Throwable>> extraThrowablesCallable : mExtraThrowables) {
+ final List<Throwable> extraThrowables = extraThrowablesCallable.call();
+ if (extraThrowables != null) {
+ Log.w(TAG, "Adding " + extraThrowables.size() + " extra exceptions");
+ mThrowables.addAll(extraThrowables);
+ }
+ }
+
+ // Finally, throw up!
+ if (mThrowables.isEmpty()) return;
+
+ final int numberExceptions = mThrowables.size();
+ if (numberExceptions == 1) {
+ throw mThrowables.get(0);
+ }
+ throw new MultipleExceptions(mThrowables);
+ }
+ };
+ }
+
+ private static String toMesssage(List<Throwable> throwables) {
+ String msg = "D'OH!";
+ try {
+ try (StringWriter sw = new StringWriter(); PrintWriter pw = new PrintWriter(sw)) {
+ sw.write("Caught " + throwables.size() + " exceptions\n");
+ for (int i = 0; i < throwables.size(); i++) {
+ sw.write("\n---- Begin of exception #" + (i + 1) + " ----\n");
+ final Throwable exception = throwables.get(i);
+ exception.printStackTrace(pw);
+ sw.write("---- End of exception #" + (i + 1) + " ----\n\n");
+ }
+ msg = sw.toString();
+ }
+ } catch (IOException e) {
+ // ignore close() errors - should not happen...
+ Log.e(TAG, "Exception closing StringWriter: " + e);
+ }
+ return msg;
+ }
+
+ // VisibleForTesting
+ static class MultipleExceptions extends AssertionError {
+ private final List<Throwable> mThrowables;
+
+ private MultipleExceptions(List<Throwable> throwables) {
+ super(toMesssage(throwables));
+
+ this.mThrowables = throwables;
+ }
+
+ List<Throwable> getThrowables() {
+ return mThrowables;
+ }
+ }
+}
diff --git a/tests/autofillservice/src/android/autofillservice/cts/SafeCleanerRuleTest.java b/tests/autofillservice/src/android/autofillservice/cts/SafeCleanerRuleTest.java
new file mode 100644
index 0000000..7fc021e
--- /dev/null
+++ b/tests/autofillservice/src/android/autofillservice/cts/SafeCleanerRuleTest.java
@@ -0,0 +1,148 @@
+/*
+ * Copyright (C) 2017 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package android.autofillservice.cts;
+
+import static com.google.common.truth.Truth.assertThat;
+
+import static org.mockito.Mockito.verify;
+import static org.testng.Assert.expectThrows;
+
+import com.google.common.collect.ImmutableList;
+
+import org.junit.Test;
+import org.junit.runner.Description;
+import org.junit.runner.RunWith;
+import org.junit.runners.model.Statement;
+import org.mockito.Mock;
+import org.mockito.junit.MockitoJUnitRunner;
+
+import java.util.List;
+import java.util.concurrent.Callable;
+
+@RunWith(MockitoJUnitRunner.class)
+public class SafeCleanerRuleTest {
+
+ private static class FailureStatement extends Statement {
+ private final Throwable mThrowable;
+
+ FailureStatement(Throwable t) {
+ mThrowable = t;
+ }
+
+ @Override
+ public void evaluate() throws Throwable {
+ throw mThrowable;
+ }
+ }
+
+ private final Description mDescription = Description.createSuiteDescription("Whatever");
+ private final RuntimeException mRuntimeException = new RuntimeException("D'OH!");
+
+ // Use mocks for objects that don't throw any exception.
+ @Mock private Runnable mGoodGuyRunner1;
+ @Mock private Runnable mGoodGuyRunner2;
+ @Mock private Callable<List<Throwable>> mGoodGuyExtraExceptions1;
+ @Mock private Callable<List<Throwable>> mGoodGuyExtraExceptions2;
+ @Mock private Statement mGoodGuyStatement;
+
+ @Test
+ public void testEmptyRule_testPass() throws Throwable {
+ final SafeCleanerRule rule = new SafeCleanerRule();
+ rule.apply(mGoodGuyStatement, mDescription).evaluate();
+ }
+
+ @Test
+ public void testEmptyRule_testFails() throws Throwable {
+ final SafeCleanerRule rule = new SafeCleanerRule();
+ final Throwable actualException = expectThrows(RuntimeException.class,
+ () -> rule.apply(new FailureStatement(mRuntimeException), mDescription).evaluate());
+ assertThat(actualException).isSameAs(mRuntimeException);
+ }
+
+ @Test
+ public void testOnlyTestFails() throws Throwable {
+ final SafeCleanerRule rule = new SafeCleanerRule()
+ .run(mGoodGuyRunner1)
+ .add(mGoodGuyExtraExceptions1);
+ final Throwable actualException = expectThrows(RuntimeException.class,
+ () -> rule.apply(new FailureStatement(mRuntimeException), mDescription).evaluate());
+ assertThat(actualException).isSameAs(mRuntimeException);
+ verify(mGoodGuyRunner1).run();
+ verify(mGoodGuyExtraExceptions1).call();
+ }
+
+ @Test
+ public void testTestPass_oneRunnerFails() throws Throwable {
+ final SafeCleanerRule rule = new SafeCleanerRule()
+ .run(mGoodGuyRunner1)
+ .run(() -> { throw mRuntimeException; })
+ .run(mGoodGuyRunner2)
+ .add(mGoodGuyExtraExceptions1);
+ final Throwable actualException = expectThrows(RuntimeException.class,
+ () -> rule.apply(mGoodGuyStatement, mDescription).evaluate());
+ assertThat(actualException).isSameAs(mRuntimeException);
+ verify(mGoodGuyRunner1).run();
+ verify(mGoodGuyRunner2).run();
+ verify(mGoodGuyExtraExceptions1).call();
+ }
+
+ @Test
+ public void testTestPass_oneExtraExceptionThrown() throws Throwable {
+ final SafeCleanerRule rule = new SafeCleanerRule()
+ .run(mGoodGuyRunner1)
+ .add(() -> { return ImmutableList.of(mRuntimeException); })
+ .add(mGoodGuyExtraExceptions1)
+ .run(mGoodGuyRunner2);
+ final Throwable actualException = expectThrows(RuntimeException.class,
+ () -> rule.apply(mGoodGuyStatement, mDescription).evaluate());
+ assertThat(actualException).isSameAs(mRuntimeException);
+ verify(mGoodGuyRunner1).run();
+ verify(mGoodGuyRunner2).run();
+ verify(mGoodGuyExtraExceptions1).call();
+ }
+
+ @Test
+ public void testThrowTheKitchenSinkAKAEverybodyThrows() throws Throwable {
+ final Exception extra1 = new Exception("1");
+ final Exception extra2 = new Exception("2");
+ final Exception extra3 = new Exception("3");
+ final Error error1 = new Error("one");
+ final Error error2 = new Error("two");
+ final RuntimeException testException = new RuntimeException("TEST, Y U NO PASS?");
+ final SafeCleanerRule rule = new SafeCleanerRule()
+ .run(mGoodGuyRunner1)
+ .add(mGoodGuyExtraExceptions1)
+ .add(() -> { return ImmutableList.of(extra1, extra2); })
+ .run(() -> {
+ throw error1;
+ })
+ .run(mGoodGuyRunner2)
+ .add(() -> { return ImmutableList.of(extra3); })
+ .add(mGoodGuyExtraExceptions2)
+ .run(() -> { throw error2; });
+
+ final SafeCleanerRule.MultipleExceptions actualException = expectThrows(
+ SafeCleanerRule.MultipleExceptions.class,
+ () -> rule.apply(new FailureStatement(testException), mDescription).evaluate());
+ assertThat(actualException.getThrowables())
+ .containsExactly(testException, error1, error2, extra1, extra2, extra3)
+ .inOrder();
+ verify(mGoodGuyRunner1).run();
+ verify(mGoodGuyRunner2).run();
+ verify(mGoodGuyExtraExceptions1).call();
+ }
+}