Add support for dEQP test to tradefed.
Add new test type DeqpTest to Android CTS tradefed. This test type runs
dEQP tests through instrumentation and parses output events and reports
them to ITestRunListener.
Also supports collecting per test case dEQP logs by setting option
collect-deqp-logs to true.
Bug: 17388917
Change-Id: I67577b3faf625f3100e3e1c2cad0a4e410232ba7
Signed-off-by: Mika Isojärvi <misojarvi@google.com>
diff --git a/tools/tradefed-host/src/com/android/cts/tradefed/testtype/CtsTest.java b/tools/tradefed-host/src/com/android/cts/tradefed/testtype/CtsTest.java
index d90a61e..2e3fadc 100644
--- a/tools/tradefed-host/src/com/android/cts/tradefed/testtype/CtsTest.java
+++ b/tools/tradefed-host/src/com/android/cts/tradefed/testtype/CtsTest.java
@@ -172,6 +172,10 @@
"Should be an amount that can comfortably fit in memory.")
private int mMaxLogcatBytes = 500 * 1024; // 500K
+ @Option(name = "collect-deqp-logs", description =
+ "Collect dEQP logs from the device.")
+ private boolean mCollectDeqpLogs = false;
+
private long mPrevRebootTime; // last reboot time
/** data structure for a {@link IRemoteTest} and its known tests */
@@ -483,6 +487,9 @@
if (test instanceof InstrumentationTest) {
((InstrumentationTest)test).setForceAbi(mForceAbi);
}
+ if (test instanceof DeqpTest) {
+ ((DeqpTest)test).setCollectLogs(mCollectDeqpLogs);
+ }
forwardPackageDetails(knownTests.getPackageDef(), listener);
test.run(filter);
diff --git a/tools/tradefed-host/src/com/android/cts/tradefed/testtype/DeqpTest.java b/tools/tradefed-host/src/com/android/cts/tradefed/testtype/DeqpTest.java
new file mode 100644
index 0000000..5381973
--- /dev/null
+++ b/tools/tradefed-host/src/com/android/cts/tradefed/testtype/DeqpTest.java
@@ -0,0 +1,451 @@
+package com.android.cts.tradefed.testtype;
+
+import com.android.cts.tradefed.build.CtsBuildHelper;
+import com.android.ddmlib.IShellOutputReceiver;
+import com.android.ddmlib.MultiLineReceiver;
+import com.android.ddmlib.testrunner.ITestRunListener;
+import com.android.ddmlib.testrunner.TestIdentifier;
+import com.android.tradefed.build.IBuildInfo;
+import com.android.tradefed.config.Option;
+import com.android.tradefed.device.DeviceNotAvailableException;
+import com.android.tradefed.device.ITestDevice;
+import com.android.tradefed.result.ByteArrayInputStreamSource;
+import com.android.tradefed.result.ITestInvocationListener;
+import com.android.tradefed.result.LogDataType;
+import com.android.tradefed.testtype.IBuildReceiver;
+import com.android.tradefed.testtype.IDeviceTest;
+import com.android.tradefed.testtype.IRemoteTest;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileReader;
+import java.io.IOException;
+import java.lang.Thread;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+
+/**
+ * Test runner for dEQP tests
+ *
+ * Supports running drawElements Quality Program tests found under external/deqp.
+ */
+public class DeqpTest implements IDeviceTest, IRemoteTest {
+ final private int TESTCASE_BATCH_LIMIT = 1000;
+
+ private boolean mLogData;
+
+ private ITestDevice mDevice;
+
+ private final String mUri;
+ private Collection<TestIdentifier> mTests;
+
+ private TestIdentifier mCurrentTestId;
+ private boolean mGotTestResult;
+ private String mCurrentTestLog;
+
+ private ITestInvocationListener mListener;
+
+ public DeqpTest(String uri, Collection<TestIdentifier> tests) {
+ mUri = uri;
+ mTests = tests;
+ mLogData = false;
+ }
+
+ /**
+ * Enable or disable raw dEQP test log collection.
+ */
+ public void setCollectLogs(boolean logData) {
+ mLogData = logData;
+ }
+
+ /**
+ * dEQP instrumentation parser
+ */
+ class InstrumentationParser extends MultiLineReceiver {
+ private DeqpTest mDeqpTests;
+
+ private Map<String, String> mValues;
+ private String mCurrentName;
+ private String mCurrentValue;
+
+
+ public InstrumentationParser(DeqpTest tests) {
+ mDeqpTests = tests;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void processNewLines(String[] lines) {
+ for (String line : lines)
+ {
+ if (mValues == null) mValues = new HashMap<String, String>();
+
+ if (line.startsWith("INSTRUMENTATION_STATUS_CODE: ")) {
+ if (mCurrentName != null) {
+ mValues.put(mCurrentName, mCurrentValue);
+
+ mCurrentName = null;
+ mCurrentValue = null;
+ }
+
+ mDeqpTests.handleStatus(mValues);
+ mValues = null;
+ } else if (line.startsWith("INSTRUMENTATION_STATUS: dEQP-")) {
+ if (mCurrentName != null) {
+ mValues.put(mCurrentName, mCurrentValue);
+
+ mCurrentValue = null;
+ mCurrentName = null;
+ }
+
+ String prefix = "INSTRUMENTATION_STATUS: ";
+ int nameBegin = prefix.length();
+ int nameEnd = line.indexOf('=');
+ int valueBegin = nameEnd + 1;
+
+ mCurrentName = line.substring(nameBegin, nameEnd);
+ mCurrentValue = line.substring(valueBegin);
+ } else if (mCurrentValue != null) {
+ mCurrentValue = mCurrentValue + line;
+ }
+ }
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void done() {
+ if (mCurrentName != null) {
+ mValues.put(mCurrentName, mCurrentValue);
+
+ mCurrentName = null;
+ mCurrentValue = null;
+ }
+
+ if (mValues != null) {
+ mDeqpTests.handleStatus(mValues);
+ mValues = null;
+ }
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public boolean isCancelled() {
+ return false;
+ }
+ }
+
+ /**
+ * Converts dEQP testcase path to TestIdentifier.
+ */
+ private TestIdentifier pathToIdentifier(String testPath)
+ {
+ String[] components = testPath.split("\\.");
+ String name = components[components.length - 1];
+ String className = null;
+
+ for (int i = 0; i < components.length - 1; i++) {
+ if (className == null) {
+ className = components[i];
+ } else {
+ className = className + "." + components[i];
+ }
+ }
+
+ return new TestIdentifier(className, name);
+ }
+
+ /**
+ * Handles beginning of dEQP session.
+ */
+ private void handleBeginSession(Map<String, String> values) {
+ mListener.testRunStarted(mUri, mTests.size());
+ }
+
+ /**
+ * Handles end of dEQP session.
+ */
+ private void handleEndSession(Map<String, String> values) {
+ Map <String, String> emptyMap = Collections.emptyMap();
+ mListener.testRunEnded(0, emptyMap);
+ }
+
+ /**
+ * Handles beginning of dEQP testcase.
+ */
+ private void handleBeginTestCase(Map<String, String> values) {
+ mCurrentTestId = pathToIdentifier(values.get("dEQP-BeginTestCase-TestCasePath"));
+ mCurrentTestLog = "";
+ mGotTestResult = false;
+
+ mListener.testStarted(mCurrentTestId);
+ mTests.remove(mCurrentTestId);
+ }
+
+ /**
+ * Handles end of dEQP testcase.
+ */
+ private void handleEndTestCase(Map<String, String> values) {
+ Map <String, String> emptyMap = Collections.emptyMap();
+
+ if (!mGotTestResult) {
+ mListener.testFailed(ITestRunListener.TestFailure.ERROR, mCurrentTestId,
+ "Log doesn't contain test result");
+ }
+
+ if (mLogData && mCurrentTestLog != null && mCurrentTestLog.length() > 0) {
+ ByteArrayInputStreamSource source
+ = new ByteArrayInputStreamSource(mCurrentTestLog.getBytes());
+
+ mListener.testLog(mCurrentTestId.getClassName() + "."
+ + mCurrentTestId.getTestName(), LogDataType.XML, source);
+
+ source.cancel();
+ }
+
+ mListener.testEnded(mCurrentTestId, emptyMap);
+ mCurrentTestId = null;
+ }
+
+ /**
+ * Handles dEQP testcase result.
+ */
+ private void handleTestCaseResult(Map<String, String> values) {
+ String code = values.get("dEQP-TestCaseResult-Code");
+ String details = values.get("dEQP-TestCaseResult-Details");
+
+ if (code.compareTo("Pass") == 0) {
+ mGotTestResult = true;
+ } else if (code.compareTo("NotSupported") == 0) {
+ mGotTestResult = true;
+ } else if (code.compareTo("QualityWarning") == 0) {
+ mGotTestResult = true;
+ } else if (code.compareTo("CompatibilityWarning") == 0) {
+ mGotTestResult = true;
+ } else if (code.compareTo("Fail") == 0 || code.compareTo("ResourceError") == 0
+ || code.compareTo("InternalError") == 0 || code.compareTo("Crash") == 0
+ || code.compareTo("Timeout") == 0) {
+ mListener.testFailed(ITestRunListener.TestFailure.ERROR, mCurrentTestId,
+ code + ":" + details);
+ mGotTestResult = true;
+ } else {
+ mListener.testFailed(ITestRunListener.TestFailure.ERROR, mCurrentTestId,
+ "Unknown result code: " + code + ":" + details);
+ mGotTestResult = true;
+ }
+ }
+
+ /**
+ * Handles terminated dEQP testcase.
+ */
+ private void handleTestCaseTerminate(Map<String, String> values) {
+ Map <String, String> emptyMap = Collections.emptyMap();
+
+ String reason = values.get("dEQP-TerminateTestCase-Reason");
+ mListener.testFailed(ITestRunListener.TestFailure.ERROR, mCurrentTestId,
+ "Terminated: " + reason);
+ mListener.testEnded(mCurrentTestId, emptyMap);
+
+ if (mLogData && mCurrentTestLog != null && mCurrentTestLog.length() > 0) {
+ ByteArrayInputStreamSource source
+ = new ByteArrayInputStreamSource(mCurrentTestLog.getBytes());
+
+ mListener.testLog(mCurrentTestId.getClassName() + "."
+ + mCurrentTestId.getTestName(), LogDataType.XML, source);
+
+ source.cancel();
+ }
+
+ mCurrentTestId = null;
+ mGotTestResult = true;
+ }
+
+ /**
+ * Handles dEQP testlog data.
+ */
+ private void handleTestLogData(Map<String, String> values) {
+ mCurrentTestLog = mCurrentTestLog + values.get("dEQP-TestLogData-Log");
+ }
+
+ /**
+ * Handles new instrumentation status message.
+ */
+ public void handleStatus(Map<String, String> values) {
+ String eventType = values.get("dEQP-EventType");
+
+ if (eventType == null)
+ return;
+
+ if (eventType.compareTo("BeginSession") == 0) {
+ handleBeginSession(values);
+ } else if (eventType.compareTo("EndSession") == 0) {
+ handleEndSession(values);
+ } else if (eventType.compareTo("BeginTestCase") == 0) {
+ handleBeginTestCase(values);
+ } else if (eventType.compareTo("EndTestCase") == 0) {
+ handleEndTestCase(values);
+ } else if (eventType.compareTo("TestCaseResult") == 0) {
+ handleTestCaseResult(values);
+ } else if (eventType.compareTo("TerminateTestCase") == 0) {
+ handleTestCaseTerminate(values);
+ } else if (eventType.compareTo("TestLogData") == 0) {
+ handleTestLogData(values);
+ }
+ }
+
+ /**
+ * Generates tescase trie from dEQP testcase paths. Used to define which testcases to execute.
+ */
+ private String generateTestCaseTrieFromPaths(Collection<String> tests) {
+ String result = "{";
+ boolean first = true;
+
+ // Add testcases to results
+ for (Iterator<String> iter = tests.iterator(); iter.hasNext();) {
+ String test = iter.next();
+ String[] components = test.split("\\.");
+
+ if (components.length == 1) {
+ if (!first) {
+ result = result + ",";
+ }
+ first = false;
+
+ result += components[0];
+ iter.remove();
+ }
+ }
+
+ if (!tests.isEmpty()) {
+ HashMap<String, ArrayList<String> > testGroups = new HashMap();
+
+ // Collect all sub testgroups
+ for (String test : tests) {
+ String[] components = test.split("\\.");
+ ArrayList<String> testGroup = testGroups.get(components[0]);
+
+ if (testGroup == null) {
+ testGroup = new ArrayList();
+ testGroups.put(components[0], testGroup);
+ }
+
+ testGroup.add(test.substring(components[0].length()+1));
+ }
+
+ for (String testGroup : testGroups.keySet()) {
+ if (!first) {
+ result = result + ",";
+ }
+
+ first = false;
+ result = result + testGroup
+ + generateTestCaseTrieFromPaths(testGroups.get(testGroup));
+ }
+ }
+
+ return result + "}";
+ }
+
+ /**
+ * Generates testacase trie from TestIdentifiers.
+ */
+ private String generateTestCaseTrie(Collection<TestIdentifier> tests) {
+ ArrayList<String> testPaths = new ArrayList();
+
+ for (TestIdentifier test : tests) {
+ testPaths.add(test.getClassName() + "." + test.getTestName());
+
+ // Limit number of testcases for each run
+ if (testPaths.size() > TESTCASE_BATCH_LIMIT)
+ break;
+ }
+
+ return generateTestCaseTrieFromPaths(testPaths);
+ }
+
+ /**
+ * Executes tests on the device.
+ */
+ private void executeTests(ITestInvocationListener listener) throws DeviceNotAvailableException {
+ InstrumentationParser parser = new InstrumentationParser(this);
+ String caseListFileName = "/sdcard/dEQP-TestCaseList.txt";
+ String logFileName = "/sdcard/TestLog.qpa";
+ String testCases = generateTestCaseTrie(mTests);
+
+ mDevice.executeShellCommand("rm " + caseListFileName);
+ mDevice.executeShellCommand("rm " + logFileName);
+ mDevice.pushString(testCases + "\n", caseListFileName);
+
+ String instrumentationName =
+ "com.drawelements.deqp/com.drawelements.deqp.testercore.DeqpInstrumentation";
+ String command = "am instrument -w -e deqpLogFileName \"" + logFileName
+ + "\" -e deqpCmdLine \"--deqp-caselist-file=" + caseListFileName + "\" "
+ + (mLogData ? "-e deqpLogData \"true\" " : "") + instrumentationName;
+
+ mDevice.executeShellCommand(command, parser);
+ parser.flush();
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void run(ITestInvocationListener listener) throws DeviceNotAvailableException {
+ mListener = listener;
+
+ while (!mTests.isEmpty()) {
+ executeTests(listener);
+
+ // Set test to failed if it didn't receive test result
+ if (mCurrentTestId != null) {
+ Map <String, String> emptyMap = Collections.emptyMap();
+
+ if (mLogData && mCurrentTestLog != null && mCurrentTestLog.length() > 0) {
+ ByteArrayInputStreamSource source
+ = new ByteArrayInputStreamSource(mCurrentTestLog.getBytes());
+
+ mListener.testLog(mCurrentTestId.getClassName() + "."
+ + mCurrentTestId.getTestName(), LogDataType.XML, source);
+
+ source.cancel();
+ }
+
+
+ if (!mGotTestResult) {
+ mListener.testFailed(ITestRunListener.TestFailure.ERROR, mCurrentTestId,
+ "Log doesn't contain test result");
+ }
+
+ mListener.testEnded(mCurrentTestId, emptyMap);
+ mCurrentTestId = null;
+ mListener.testRunEnded(0, emptyMap);
+ }
+ }
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void setDevice(ITestDevice device) {
+ mDevice = device;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public ITestDevice getDevice() {
+ return mDevice;
+ }
+}
diff --git a/tools/tradefed-host/src/com/android/cts/tradefed/testtype/TestPackageDef.java b/tools/tradefed-host/src/com/android/cts/tradefed/testtype/TestPackageDef.java
index c477585..4fa3b2b 100644
--- a/tools/tradefed-host/src/com/android/cts/tradefed/testtype/TestPackageDef.java
+++ b/tools/tradefed-host/src/com/android/cts/tradefed/testtype/TestPackageDef.java
@@ -46,6 +46,7 @@
public static final String NATIVE_TEST = "native";
public static final String WRAPPED_NATIVE_TEST = "wrappednative";
public static final String VM_HOST_TEST = "vmHostTest";
+ public static final String DEQP_TEST = "deqpTest";
public static final String ACCESSIBILITY_TEST =
"com.android.cts.tradefed.testtype.AccessibilityTestRunner";
public static final String ACCESSIBILITY_SERVICE_TEST =
@@ -233,6 +234,8 @@
vmHostTest.setTests(mTests);
mDigest = generateDigest(testCaseDir, mJarPath);
return vmHostTest;
+ } else if (DEQP_TEST.equals(mTestType)) {
+ return new DeqpTest(mUri, mTests);
} else if (NATIVE_TEST.equals(mTestType)) {
return new GeeTest(mUri, mName);
} else if (WRAPPED_NATIVE_TEST.equals(mTestType)) {
diff --git a/tools/tradefed-host/tests/src/com/android/cts/tradefed/UnitTests.java b/tools/tradefed-host/tests/src/com/android/cts/tradefed/UnitTests.java
index 3c36a3d..65528b7 100644
--- a/tools/tradefed-host/tests/src/com/android/cts/tradefed/UnitTests.java
+++ b/tools/tradefed-host/tests/src/com/android/cts/tradefed/UnitTests.java
@@ -22,6 +22,7 @@
import com.android.cts.tradefed.result.TestSummaryXmlTest;
import com.android.cts.tradefed.result.TestTest;
import com.android.cts.tradefed.testtype.CtsTestTest;
+import com.android.cts.tradefed.testtype.DeqpTestTest;
import com.android.cts.tradefed.testtype.JarHostTestTest;
import com.android.cts.tradefed.testtype.TestFilterTest;
import com.android.cts.tradefed.testtype.TestPackageDefTest;
@@ -59,6 +60,7 @@
addTestSuite(TestPackageXmlParserTest.class);
addTestSuite(TestPlanTest.class);
addTestSuite(WrappedGTestResultParserTest.class);
+ addTestSuite(DeqpTestTest.class);
}
public static Test suite() {
diff --git a/tools/tradefed-host/tests/src/com/android/cts/tradefed/testtype/DeqpTestTest.java b/tools/tradefed-host/tests/src/com/android/cts/tradefed/testtype/DeqpTestTest.java
new file mode 100644
index 0000000..b6e2806
--- /dev/null
+++ b/tools/tradefed-host/tests/src/com/android/cts/tradefed/testtype/DeqpTestTest.java
@@ -0,0 +1,380 @@
+/*
+ * Copyright (C) 2014 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.android.cts.tradefed.testtype;
+
+import com.android.ddmlib.IShellOutputReceiver;
+import com.android.ddmlib.testrunner.ITestRunListener;
+import com.android.ddmlib.testrunner.TestIdentifier;
+import com.android.tradefed.device.ITestDevice;
+import com.android.tradefed.result.ITestInvocationListener;
+
+import junit.framework.TestCase;
+
+import org.easymock.EasyMock;
+import org.easymock.IAnswer;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Map;
+
+/**
+ * Unit tests for {@link DeqpTest}.
+ */
+public class DeqpTestTest extends TestCase {
+ private static final String URI = "dEQP-GLES3";
+ private static final String CASE_LIST_FILE_NAME = "/sdcard/dEQP-TestCaseList.txt";
+ private static final String LOG_FILE_NAME = "/sdcard/TestLog.qpa";
+ private static final String INSTRUMENTATION_NAME =
+ "com.drawelements.deqp/com.drawelements.deqp.testercore.DeqpInstrumentation";
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected void setUp() throws Exception {
+ super.setUp();
+ }
+
+ /**
+ * Test that result code produces correctly pass or fail.
+ */
+ private void testResultCode(final String resultCode, boolean pass) throws Exception {
+ final TestIdentifier testId = new TestIdentifier("dEQP-GLES3.info", "version");
+ final String testPath = "dEQP-GLES3.info.version";
+ final String testTrie = "{dEQP-GLES3{info{version}}}";
+
+ /* MultiLineReceiver expects "\r\n" line ending. */
+ final String output = "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=" + testPath + "\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=" + resultCode + "\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Detail" + resultCode + "\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_CODE: 0\r\n";
+
+ ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class);
+ ITestInvocationListener mockListener
+ = EasyMock.createStrictMock(ITestInvocationListener.class);
+ Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>();
+
+ tests.add(testId);
+
+ DeqpTest deqpTest = new DeqpTest(URI, tests);
+
+ EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("rm " + CASE_LIST_FILE_NAME)))
+ .andReturn("").once();
+
+ EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("rm " + LOG_FILE_NAME)))
+ .andReturn("").once();
+
+ EasyMock.expect(mockDevice.pushString(testTrie + "\n", CASE_LIST_FILE_NAME)).andReturn(true)
+ .once();
+
+ String command = "am instrument -w -e deqpLogFileName \"" + LOG_FILE_NAME
+ + "\" -e deqpCmdLine \"--deqp-caselist-file=" + CASE_LIST_FILE_NAME + "\" "
+ + INSTRUMENTATION_NAME;
+
+ mockDevice.executeShellCommand(EasyMock.eq(command),
+ EasyMock.<IShellOutputReceiver>notNull());
+
+ EasyMock.expectLastCall().andAnswer(new IAnswer<Object>() {
+ @Override
+ public Object answer() {
+ IShellOutputReceiver receiver
+ = (IShellOutputReceiver)EasyMock.getCurrentArguments()[1];
+
+ receiver.addOutput(output.getBytes(), 0, output.length());
+ receiver.flush();
+
+ return null;
+ }
+ });
+
+ mockListener.testRunStarted(URI, 1);
+ EasyMock.expectLastCall().once();
+
+ mockListener.testStarted(EasyMock.eq(testId));
+ EasyMock.expectLastCall().once();
+
+ if (!pass) {
+ mockListener.testFailed(ITestRunListener.TestFailure.ERROR, testId,
+ resultCode + ":Detail" + resultCode);
+
+ EasyMock.expectLastCall().once();
+ }
+
+ mockListener.testEnded(EasyMock.eq(testId), EasyMock.<Map<String, String>>notNull());
+ EasyMock.expectLastCall().once();
+
+ mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<Map<String, String>>notNull());
+ EasyMock.expectLastCall().once();
+
+ EasyMock.replay(mockDevice);
+ EasyMock.replay(mockListener);
+
+ deqpTest.setDevice(mockDevice);
+ deqpTest.run(mockListener);
+
+ EasyMock.verify(mockListener);
+ EasyMock.verify(mockDevice);
+ }
+
+ /**
+ * Test running multiple test cases.
+ */
+ public void testRun_multipleTets() throws Exception {
+ /* MultiLineReceiver expects "\r\n" line ending. */
+ final String output = "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.vendor\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.renderer\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.version\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.shading_language_version\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.extensions\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.render_target\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_CODE: 0\r\n";
+
+ final TestIdentifier[] testIds = {
+ new TestIdentifier("dEQP-GLES3.info", "vendor"),
+ new TestIdentifier("dEQP-GLES3.info", "renderer"),
+ new TestIdentifier("dEQP-GLES3.info", "version"),
+ new TestIdentifier("dEQP-GLES3.info", "shading_language_version"),
+ new TestIdentifier("dEQP-GLES3.info", "extensions"),
+ new TestIdentifier("dEQP-GLES3.info", "render_target")
+ };
+
+ final String[] testPaths = {
+ "dEQP-GLES3.info.vendor",
+ "dEQP-GLES3.info.renderer",
+ "dEQP-GLES3.info.version",
+ "dEQP-GLES3.info.shading_language_version",
+ "dEQP-GLES3.info.extensions",
+ "dEQP-GLES3.info.render_target"
+ };
+
+ final String testTrie
+ = "{dEQP-GLES3{info{vendor,renderer,version,shading_language_version,extensions,render_target}}}";
+
+ ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class);
+ ITestInvocationListener mockListener = EasyMock.createStrictMock(ITestInvocationListener.class);
+ Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>();
+
+ for (TestIdentifier id : testIds) {
+ tests.add(id);
+ }
+
+ DeqpTest deqpTest = new DeqpTest(URI, tests);
+
+ EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("rm " + CASE_LIST_FILE_NAME)))
+ .andReturn("").once();
+
+ EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("rm " + LOG_FILE_NAME)))
+ .andReturn("").once();
+
+ EasyMock.expect(mockDevice.pushString(testTrie + "\n", CASE_LIST_FILE_NAME))
+ .andReturn(true).once();
+
+ String command = "am instrument -w -e deqpLogFileName \"" + LOG_FILE_NAME
+ + "\" -e deqpCmdLine \"--deqp-caselist-file=" + CASE_LIST_FILE_NAME + "\" "
+ + INSTRUMENTATION_NAME;
+
+ mockDevice.executeShellCommand(EasyMock.eq(command),
+ EasyMock.<IShellOutputReceiver>notNull());
+
+ EasyMock.expectLastCall().andAnswer(new IAnswer<Object>() {
+ @Override
+ public Object answer() {
+ IShellOutputReceiver receiver
+ = (IShellOutputReceiver)EasyMock.getCurrentArguments()[1];
+
+ receiver.addOutput(output.getBytes(), 0, output.length());
+ receiver.flush();
+
+ return null;
+ }
+ });
+
+ mockListener.testRunStarted(URI, testPaths.length);
+ EasyMock.expectLastCall().once();
+
+ for (int i = 0; i < testPaths.length; i++) {
+ mockListener.testStarted(EasyMock.eq(testIds[i]));
+ EasyMock.expectLastCall().once();
+
+ mockListener.testEnded(EasyMock.eq(testIds[i]),
+ EasyMock.<Map<String, String>>notNull());
+
+ EasyMock.expectLastCall().once();
+ }
+
+ mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<Map<String, String>>notNull());
+ EasyMock.expectLastCall().once();
+
+ EasyMock.replay(mockDevice);
+ EasyMock.replay(mockListener);
+
+ deqpTest.setDevice(mockDevice);
+ deqpTest.run(mockListener);
+
+ EasyMock.verify(mockListener);
+ EasyMock.verify(mockDevice);
+ }
+
+ /**
+ * Test dEQP Pass result code.
+ */
+ public void testRun_resultPass() throws Exception {
+ testResultCode("Pass", true);
+ }
+
+ /**
+ * Test dEQP Fail result code.
+ */
+ public void testRun_resultFail() throws Exception {
+ testResultCode("Fail", false);
+ }
+
+ /**
+ * Test dEQP NotSupported result code.
+ */
+ public void testRun_resultNotSupported() throws Exception {
+ testResultCode("NotSupported", true);
+ }
+
+ /**
+ * Test dEQP QualityWarning result code.
+ */
+ public void testRun_resultQualityWarning() throws Exception {
+ testResultCode("QualityWarning", true);
+ }
+
+ /**
+ * Test dEQP CompatibilityWarning result code.
+ */
+ public void testRun_resultCompatibilityWarning() throws Exception {
+ testResultCode("CompatibilityWarning", true);
+ }
+
+ /**
+ * Test dEQP ResourceError result code.
+ */
+ public void testRun_resultResourceError() throws Exception {
+ testResultCode("ResourceError", false);
+ }
+
+ /**
+ * Test dEQP InternalError result code.
+ */
+ public void testRun_resultInternalError() throws Exception {
+ testResultCode("InternalError", false);
+ }
+
+ /**
+ * Test dEQP Crash result code.
+ */
+ public void testRun_resultCrash() throws Exception {
+ testResultCode("Crash", false);
+ }
+
+ /**
+ * Test dEQP Timeout result code.
+ */
+ public void testRun_resultTimeout() throws Exception {
+ testResultCode("Timeout", false);
+ }
+}