blob: 3e4cdf4685933dd9cd202f45b2c85dbace94fee0 [file] [log] [blame]
/*
* Copyright (C) 2008 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.android.ddmlib.testrunner;
import junit.framework.TestCase;
import org.easymock.Capture;
import org.easymock.EasyMock;
import java.util.Collections;
import java.util.Map;
/**
* Unit tests for {@link @InstrumentationResultParser}.
*/
@SuppressWarnings("unchecked")
public class InstrumentationResultParserTest extends TestCase {
private InstrumentationResultParser mParser;
private ITestRunListener mMockListener;
// static dummy test names to use for validation
private static final String RUN_NAME = "foo";
private static final String CLASS_NAME = "com.test.FooTest";
private static final String TEST_NAME = "testFoo";
private static final String STACK_TRACE = "java.lang.AssertionFailedException";
private static final TestIdentifier TEST_ID = new TestIdentifier(CLASS_NAME, TEST_NAME);
/**
* @param name - test name
*/
public InstrumentationResultParserTest(String name) {
super(name);
}
/**
* @see junit.framework.TestCase#setUp()
*/
@Override
protected void setUp() throws Exception {
super.setUp();
// use a strict mock to verify order of method calls
mMockListener = EasyMock.createStrictMock(ITestRunListener.class);
mParser = new InstrumentationResultParser(RUN_NAME, mMockListener);
}
/**
* Tests parsing empty output.
*/
public void testParse_empty() {
mMockListener.testRunStarted(RUN_NAME, 0);
mMockListener.testRunFailed(InstrumentationResultParser.NO_TEST_RESULTS_MSG);
mMockListener.testRunEnded(0, Collections.EMPTY_MAP);
injectAndVerifyTestString("");
}
/**
* Tests parsing output for a successful test run with no tests.
*/
public void testParse_noTests() {
StringBuilder output = new StringBuilder();
addLine(output, "INSTRUMENTATION_RESULT: stream=");
addLine(output, "Test results for InstrumentationTestRunner=");
addLine(output, "Time: 0.001");
addLine(output, "OK (0 tests)");
addLine(output, "INSTRUMENTATION_CODE: -1");
mMockListener.testRunStarted(RUN_NAME, 0);
mMockListener.testRunEnded(1, Collections.EMPTY_MAP);
injectAndVerifyTestString(output.toString());
}
/**
* Tests parsing output for a single successful test execution.
*/
public void testParse_singleTest() {
StringBuilder output = createSuccessTest();
mMockListener.testRunStarted(RUN_NAME, 1);
mMockListener.testStarted(TEST_ID);
mMockListener.testEnded(TEST_ID, Collections.EMPTY_MAP);
mMockListener.testRunEnded(0, Collections.EMPTY_MAP);
injectAndVerifyTestString(output.toString());
}
/**
* Tests parsing output for a successful test execution with metrics.
*/
public void testParse_testMetrics() {
StringBuilder output = buildCommonResult();
addStatusKey(output, "randomKey", "randomValue");
addSuccessCode(output);
final Capture<Map<String, String>> captureMetrics = new Capture<Map<String, String>>();
mMockListener.testRunStarted(RUN_NAME, 1);
mMockListener.testStarted(TEST_ID);
mMockListener.testEnded(EasyMock.eq(TEST_ID), EasyMock.capture(captureMetrics));
mMockListener.testRunEnded(0, Collections.EMPTY_MAP);
injectAndVerifyTestString(output.toString());
assertEquals("randomValue", captureMetrics.getValue().get("randomKey"));
}
/**
* Test parsing output for a test that produces repeated metrics values
* <p/>
* This mimics launch performance test output.
*/
public void testParse_repeatedTestMetrics() {
StringBuilder output = new StringBuilder();
// add test start output
addCommonStatus(output);
addStartCode(output);
addStatusKey(output, "currentiterations", "1");
addStatusCode(output, "2");
addStatusKey(output, "currentiterations", "2");
addStatusCode(output, "2");
addStatusKey(output, "currentiterations", "3");
addStatusCode(output, "2");
// add test end
addCommonStatus(output);
addStatusKey(output, "numiterations", "3");
addSuccessCode(output);
final Capture<Map<String, String>> captureMetrics = new Capture<Map<String, String>>();
mMockListener.testRunStarted(RUN_NAME, 1);
mMockListener.testStarted(TEST_ID);
mMockListener.testEnded(EasyMock.eq(TEST_ID), EasyMock.capture(captureMetrics));
mMockListener.testRunEnded(0, Collections.EMPTY_MAP);
injectAndVerifyTestString(output.toString());
assertEquals("3", captureMetrics.getValue().get("currentiterations"));
assertEquals("3", captureMetrics.getValue().get("numiterations"));
}
/**
* Test parsing output for a test failure.
*/
public void testParse_testFailed() {
StringBuilder output = buildCommonResult();
addStackTrace(output);
addFailureCode(output);
mMockListener.testRunStarted(RUN_NAME, 1);
mMockListener.testStarted(TEST_ID);
mMockListener.testFailed(TEST_ID, STACK_TRACE);
mMockListener.testEnded(TEST_ID, Collections.EMPTY_MAP);
mMockListener.testRunEnded(0, Collections.EMPTY_MAP);
injectAndVerifyTestString(output.toString());
}
/**
* Test parsing and conversion of time output that contains extra chars.
*/
public void testParse_timeBracket() {
StringBuilder output = createSuccessTest();
output.append("Time: 0.001)");
mMockListener.testRunStarted(RUN_NAME, 1);
mMockListener.testStarted(TEST_ID);
mMockListener.testEnded(TEST_ID, Collections.EMPTY_MAP);
mMockListener.testRunEnded(1, Collections.EMPTY_MAP);
injectAndVerifyTestString(output.toString());
}
/**
* Test parsing output for a test run failure.
*/
public void testParse_runFailed() {
StringBuilder output = new StringBuilder();
final String errorMessage = "Unable to find instrumentation info";
addStatusKey(output, "Error", errorMessage);
addStatusCode(output, "-1");
output.append("INSTRUMENTATION_FAILED: com.dummy/android.test.InstrumentationTestRunner");
addLineBreak(output);
mMockListener.testRunStarted(RUN_NAME, 0);
mMockListener.testRunFailed(errorMessage);
mMockListener.testRunEnded(0, Collections.EMPTY_MAP);
injectAndVerifyTestString(output.toString());
}
/**
* Test parsing output when a status code cannot be parsed
*/
public void testParse_invalidCode() {
StringBuilder output = new StringBuilder();
addLine(output, "android.util.AndroidException: INSTRUMENTATION_FAILED: foo/foo");
addLine(output, "INSTRUMENTATION_STATUS: id=ActivityManagerService");
addLine(output, "INSTRUMENTATION_STATUS: Error=Unable to find instrumentation target package: foo");
addLine(output, "INSTRUMENTATION_STATUS_CODE: -1at com.android.commands.am.Am.runInstrument(Am.java:532)");
addLine(output, "");
addLine(output, " at com.android.commands.am.Am.run(Am.java:111)");
addLineBreak(output);
mMockListener.testRunStarted(RUN_NAME, 0);
mMockListener.testRunFailed((String)EasyMock.anyObject());
mMockListener.testRunEnded(0, Collections.EMPTY_MAP);
injectAndVerifyTestString(output.toString());
}
/**
* Test parsing output for a test run failure, where an instrumentation component failed to
* load.
* <p/>
* Parsing input takes the from of INSTRUMENTATION_RESULT: fff
*/
public void testParse_failedResult() {
StringBuilder output = new StringBuilder();
final String errorMessage = "Unable to instantiate instrumentation";
output.append("INSTRUMENTATION_RESULT: shortMsg=");
output.append(errorMessage);
addLineBreak(output);
output.append("INSTRUMENTATION_CODE: 0");
addLineBreak(output);
mMockListener.testRunStarted(RUN_NAME, 0);
mMockListener.testRunFailed(EasyMock.contains(errorMessage));
mMockListener.testRunEnded(0, Collections.EMPTY_MAP);
injectAndVerifyTestString(output.toString());
}
/**
* Test parsing output for a test run that did not complete.
* <p/>
* This can occur if device spontaneously reboots, or if test method could not be found.
*/
public void testParse_incomplete() {
StringBuilder output = new StringBuilder();
// add a start test sequence, but without an end test sequence
addCommonStatus(output);
addStartCode(output);
mMockListener.testRunStarted(RUN_NAME, 1);
mMockListener.testStarted(TEST_ID);
mMockListener.testFailed(EasyMock.eq(TEST_ID),
EasyMock.startsWith(InstrumentationResultParser.INCOMPLETE_TEST_ERR_MSG_PREFIX));
mMockListener.testEnded(TEST_ID, Collections.EMPTY_MAP);
mMockListener.testRunFailed(EasyMock.startsWith(
InstrumentationResultParser.INCOMPLETE_RUN_ERR_MSG_PREFIX));
mMockListener.testRunEnded(0, Collections.EMPTY_MAP);
injectAndVerifyTestString(output.toString());
}
/**
* Test parsing output for a test run that did not start due to incorrect syntax supplied to am.
*/
public void testParse_amFailed() {
StringBuilder output = new StringBuilder();
addLine(output, "usage: am [subcommand] [options]");
addLine(output, "start an Activity: am start [-D] [-W] <INTENT>");
addLine(output, "-D: enable debugging");
addLine(output, "-W: wait for launch to complete");
addLine(output, "start a Service: am startservice <INTENT>");
addLine(output, "Error: Bad component name: wfsdafddfasasdf");
mMockListener.testRunStarted(RUN_NAME, 0);
mMockListener.testRunFailed(InstrumentationResultParser.NO_TEST_RESULTS_MSG);
mMockListener.testRunEnded(0, Collections.EMPTY_MAP);
injectAndVerifyTestString(output.toString());
}
/**
* Test parsing output for a test run that produces INSTRUMENTATION_RESULT output.
* <p/>
* This mimics launch performance test output.
*/
public void testParse_instrumentationResults() {
StringBuilder output = new StringBuilder();
addResultKey(output, "other_pss", "2390");
addResultKey(output, "java_allocated", "2539");
addResultKey(output, "foo", "bar");
addResultKey(output, "stream", "should not be captured");
addLine(output, "INSTRUMENTATION_CODE: -1");
Capture<Map<String, String>> captureMetrics = new Capture<Map<String, String>>();
mMockListener.testRunStarted(RUN_NAME, 0);
mMockListener.testRunEnded(EasyMock.anyLong(), EasyMock.capture(captureMetrics));
injectAndVerifyTestString(output.toString());
assertEquals("2390", captureMetrics.getValue().get("other_pss"));
assertEquals("2539", captureMetrics.getValue().get("java_allocated"));
assertEquals("bar", captureMetrics.getValue().get("foo"));
assertEquals(3, captureMetrics.getValue().size());
}
/**
* Builds a common test result using TEST_NAME and TEST_CLASS.
*/
private StringBuilder buildCommonResult() {
StringBuilder output = new StringBuilder();
// add test start bundle
addCommonStatus(output);
addStartCode(output);
// add end test bundle, without status
addCommonStatus(output);
return output;
}
/**
* Create instrumentation output for a successful single test case execution.
*/
private StringBuilder createSuccessTest() {
StringBuilder output = buildCommonResult();
addSuccessCode(output);
return output;
}
/**
* Adds common status results to the provided output.
*/
private void addCommonStatus(StringBuilder output) {
addStatusKey(output, "stream", "\r\n" + CLASS_NAME);
addStatusKey(output, "test", TEST_NAME);
addStatusKey(output, "class", CLASS_NAME);
addStatusKey(output, "current", "1");
addStatusKey(output, "numtests", "1");
addStatusKey(output, "id", "InstrumentationTestRunner");
}
/**
* Adds a stack trace status bundle to output.
*/
private void addStackTrace(StringBuilder output) {
addStatusKey(output, "stack", STACK_TRACE);
}
/**
* Helper method to add a status key-value bundle.
*/
private void addStatusKey(StringBuilder outputBuilder, String key,
String value) {
outputBuilder.append("INSTRUMENTATION_STATUS: ");
outputBuilder.append(key);
outputBuilder.append('=');
outputBuilder.append(value);
addLineBreak(outputBuilder);
}
/**
* Helper method to add a result key value bundle.
*/
private void addResultKey(StringBuilder outputBuilder, String key,
String value) {
outputBuilder.append("INSTRUMENTATION_RESULT: ");
outputBuilder.append(key);
outputBuilder.append('=');
outputBuilder.append(value);
addLineBreak(outputBuilder);
}
/**
* Append a line to output.
*/
private void addLine(StringBuilder outputBuilder, String lineContent) {
outputBuilder.append(lineContent);
addLineBreak(outputBuilder);
}
/**
* Append line break characters to output
*/
private void addLineBreak(StringBuilder outputBuilder) {
outputBuilder.append("\r\n");
}
private void addStartCode(StringBuilder outputBuilder) {
addStatusCode(outputBuilder, "1");
}
private void addSuccessCode(StringBuilder outputBuilder) {
addStatusCode(outputBuilder, "0");
}
private void addFailureCode(StringBuilder outputBuilder) {
addStatusCode(outputBuilder, "-2");
}
private void addStatusCode(StringBuilder outputBuilder, String value) {
outputBuilder.append("INSTRUMENTATION_STATUS_CODE: ");
outputBuilder.append(value);
addLineBreak(outputBuilder);
}
/**
* Inject a test string into the result parser, and verify the mock listener.
*
* @param result the string to inject into parser under test.
*/
private void injectAndVerifyTestString(String result) {
EasyMock.replay(mMockListener);
byte[] data = result.getBytes();
mParser.addOutput(data, 0, data.length);
mParser.flush();
EasyMock.verify(mMockListener);
}
}