Merge "Add support for dEQP test to tradefed." into lmp-dev
diff --git a/tools/tradefed-host/src/com/android/cts/tradefed/testtype/CtsTest.java b/tools/tradefed-host/src/com/android/cts/tradefed/testtype/CtsTest.java
index d90a61e..2e3fadc 100644
--- a/tools/tradefed-host/src/com/android/cts/tradefed/testtype/CtsTest.java
+++ b/tools/tradefed-host/src/com/android/cts/tradefed/testtype/CtsTest.java
@@ -172,6 +172,10 @@
"Should be an amount that can comfortably fit in memory.")
private int mMaxLogcatBytes = 500 * 1024; // 500K
+ @Option(name = "collect-deqp-logs", description =
+ "Collect dEQP logs from the device.")
+ private boolean mCollectDeqpLogs = false;
+
private long mPrevRebootTime; // last reboot time
/** data structure for a {@link IRemoteTest} and its known tests */
@@ -483,6 +487,9 @@
if (test instanceof InstrumentationTest) {
((InstrumentationTest)test).setForceAbi(mForceAbi);
}
+ if (test instanceof DeqpTest) {
+ ((DeqpTest)test).setCollectLogs(mCollectDeqpLogs);
+ }
forwardPackageDetails(knownTests.getPackageDef(), listener);
test.run(filter);
diff --git a/tools/tradefed-host/src/com/android/cts/tradefed/testtype/DeqpTest.java b/tools/tradefed-host/src/com/android/cts/tradefed/testtype/DeqpTest.java
new file mode 100644
index 0000000..5381973
--- /dev/null
+++ b/tools/tradefed-host/src/com/android/cts/tradefed/testtype/DeqpTest.java
@@ -0,0 +1,451 @@
+package com.android.cts.tradefed.testtype;
+
+import com.android.cts.tradefed.build.CtsBuildHelper;
+import com.android.ddmlib.IShellOutputReceiver;
+import com.android.ddmlib.MultiLineReceiver;
+import com.android.ddmlib.testrunner.ITestRunListener;
+import com.android.ddmlib.testrunner.TestIdentifier;
+import com.android.tradefed.build.IBuildInfo;
+import com.android.tradefed.config.Option;
+import com.android.tradefed.device.DeviceNotAvailableException;
+import com.android.tradefed.device.ITestDevice;
+import com.android.tradefed.result.ByteArrayInputStreamSource;
+import com.android.tradefed.result.ITestInvocationListener;
+import com.android.tradefed.result.LogDataType;
+import com.android.tradefed.testtype.IBuildReceiver;
+import com.android.tradefed.testtype.IDeviceTest;
+import com.android.tradefed.testtype.IRemoteTest;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileReader;
+import java.io.IOException;
+import java.lang.Thread;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+
+/**
+ * Test runner for dEQP tests
+ *
+ * Supports running drawElements Quality Program tests found under external/deqp.
+ */
+public class DeqpTest implements IDeviceTest, IRemoteTest {
+ final private int TESTCASE_BATCH_LIMIT = 1000;
+
+ private boolean mLogData;
+
+ private ITestDevice mDevice;
+
+ private final String mUri;
+ private Collection<TestIdentifier> mTests;
+
+ private TestIdentifier mCurrentTestId;
+ private boolean mGotTestResult;
+ private String mCurrentTestLog;
+
+ private ITestInvocationListener mListener;
+
+ public DeqpTest(String uri, Collection<TestIdentifier> tests) {
+ mUri = uri;
+ mTests = tests;
+ mLogData = false;
+ }
+
+ /**
+ * Enable or disable raw dEQP test log collection.
+ */
+ public void setCollectLogs(boolean logData) {
+ mLogData = logData;
+ }
+
+ /**
+ * dEQP instrumentation parser
+ */
+ class InstrumentationParser extends MultiLineReceiver {
+ private DeqpTest mDeqpTests;
+
+ private Map<String, String> mValues;
+ private String mCurrentName;
+ private String mCurrentValue;
+
+
+ public InstrumentationParser(DeqpTest tests) {
+ mDeqpTests = tests;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void processNewLines(String[] lines) {
+ for (String line : lines)
+ {
+ if (mValues == null) mValues = new HashMap<String, String>();
+
+ if (line.startsWith("INSTRUMENTATION_STATUS_CODE: ")) {
+ if (mCurrentName != null) {
+ mValues.put(mCurrentName, mCurrentValue);
+
+ mCurrentName = null;
+ mCurrentValue = null;
+ }
+
+ mDeqpTests.handleStatus(mValues);
+ mValues = null;
+ } else if (line.startsWith("INSTRUMENTATION_STATUS: dEQP-")) {
+ if (mCurrentName != null) {
+ mValues.put(mCurrentName, mCurrentValue);
+
+ mCurrentValue = null;
+ mCurrentName = null;
+ }
+
+ String prefix = "INSTRUMENTATION_STATUS: ";
+ int nameBegin = prefix.length();
+ int nameEnd = line.indexOf('=');
+ int valueBegin = nameEnd + 1;
+
+ mCurrentName = line.substring(nameBegin, nameEnd);
+ mCurrentValue = line.substring(valueBegin);
+ } else if (mCurrentValue != null) {
+ mCurrentValue = mCurrentValue + line;
+ }
+ }
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void done() {
+ if (mCurrentName != null) {
+ mValues.put(mCurrentName, mCurrentValue);
+
+ mCurrentName = null;
+ mCurrentValue = null;
+ }
+
+ if (mValues != null) {
+ mDeqpTests.handleStatus(mValues);
+ mValues = null;
+ }
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public boolean isCancelled() {
+ return false;
+ }
+ }
+
+ /**
+ * Converts dEQP testcase path to TestIdentifier.
+ */
+ private TestIdentifier pathToIdentifier(String testPath)
+ {
+ String[] components = testPath.split("\\.");
+ String name = components[components.length - 1];
+ String className = null;
+
+ for (int i = 0; i < components.length - 1; i++) {
+ if (className == null) {
+ className = components[i];
+ } else {
+ className = className + "." + components[i];
+ }
+ }
+
+ return new TestIdentifier(className, name);
+ }
+
+ /**
+ * Handles beginning of dEQP session.
+ */
+ private void handleBeginSession(Map<String, String> values) {
+ mListener.testRunStarted(mUri, mTests.size());
+ }
+
+ /**
+ * Handles end of dEQP session.
+ */
+ private void handleEndSession(Map<String, String> values) {
+ Map <String, String> emptyMap = Collections.emptyMap();
+ mListener.testRunEnded(0, emptyMap);
+ }
+
+ /**
+ * Handles beginning of dEQP testcase.
+ */
+ private void handleBeginTestCase(Map<String, String> values) {
+ mCurrentTestId = pathToIdentifier(values.get("dEQP-BeginTestCase-TestCasePath"));
+ mCurrentTestLog = "";
+ mGotTestResult = false;
+
+ mListener.testStarted(mCurrentTestId);
+ mTests.remove(mCurrentTestId);
+ }
+
+ /**
+ * Handles end of dEQP testcase.
+ */
+ private void handleEndTestCase(Map<String, String> values) {
+ Map <String, String> emptyMap = Collections.emptyMap();
+
+ if (!mGotTestResult) {
+ mListener.testFailed(ITestRunListener.TestFailure.ERROR, mCurrentTestId,
+ "Log doesn't contain test result");
+ }
+
+ if (mLogData && mCurrentTestLog != null && mCurrentTestLog.length() > 0) {
+ ByteArrayInputStreamSource source
+ = new ByteArrayInputStreamSource(mCurrentTestLog.getBytes());
+
+ mListener.testLog(mCurrentTestId.getClassName() + "."
+ + mCurrentTestId.getTestName(), LogDataType.XML, source);
+
+ source.cancel();
+ }
+
+ mListener.testEnded(mCurrentTestId, emptyMap);
+ mCurrentTestId = null;
+ }
+
+ /**
+ * Handles dEQP testcase result.
+ */
+ private void handleTestCaseResult(Map<String, String> values) {
+ String code = values.get("dEQP-TestCaseResult-Code");
+ String details = values.get("dEQP-TestCaseResult-Details");
+
+ if (code.compareTo("Pass") == 0) {
+ mGotTestResult = true;
+ } else if (code.compareTo("NotSupported") == 0) {
+ mGotTestResult = true;
+ } else if (code.compareTo("QualityWarning") == 0) {
+ mGotTestResult = true;
+ } else if (code.compareTo("CompatibilityWarning") == 0) {
+ mGotTestResult = true;
+ } else if (code.compareTo("Fail") == 0 || code.compareTo("ResourceError") == 0
+ || code.compareTo("InternalError") == 0 || code.compareTo("Crash") == 0
+ || code.compareTo("Timeout") == 0) {
+ mListener.testFailed(ITestRunListener.TestFailure.ERROR, mCurrentTestId,
+ code + ":" + details);
+ mGotTestResult = true;
+ } else {
+ mListener.testFailed(ITestRunListener.TestFailure.ERROR, mCurrentTestId,
+ "Unknown result code: " + code + ":" + details);
+ mGotTestResult = true;
+ }
+ }
+
+ /**
+ * Handles terminated dEQP testcase.
+ */
+ private void handleTestCaseTerminate(Map<String, String> values) {
+ Map <String, String> emptyMap = Collections.emptyMap();
+
+ String reason = values.get("dEQP-TerminateTestCase-Reason");
+ mListener.testFailed(ITestRunListener.TestFailure.ERROR, mCurrentTestId,
+ "Terminated: " + reason);
+ mListener.testEnded(mCurrentTestId, emptyMap);
+
+ if (mLogData && mCurrentTestLog != null && mCurrentTestLog.length() > 0) {
+ ByteArrayInputStreamSource source
+ = new ByteArrayInputStreamSource(mCurrentTestLog.getBytes());
+
+ mListener.testLog(mCurrentTestId.getClassName() + "."
+ + mCurrentTestId.getTestName(), LogDataType.XML, source);
+
+ source.cancel();
+ }
+
+ mCurrentTestId = null;
+ mGotTestResult = true;
+ }
+
+ /**
+ * Handles dEQP testlog data.
+ */
+ private void handleTestLogData(Map<String, String> values) {
+ mCurrentTestLog = mCurrentTestLog + values.get("dEQP-TestLogData-Log");
+ }
+
+ /**
+ * Handles new instrumentation status message.
+ */
+ public void handleStatus(Map<String, String> values) {
+ String eventType = values.get("dEQP-EventType");
+
+ if (eventType == null)
+ return;
+
+ if (eventType.compareTo("BeginSession") == 0) {
+ handleBeginSession(values);
+ } else if (eventType.compareTo("EndSession") == 0) {
+ handleEndSession(values);
+ } else if (eventType.compareTo("BeginTestCase") == 0) {
+ handleBeginTestCase(values);
+ } else if (eventType.compareTo("EndTestCase") == 0) {
+ handleEndTestCase(values);
+ } else if (eventType.compareTo("TestCaseResult") == 0) {
+ handleTestCaseResult(values);
+ } else if (eventType.compareTo("TerminateTestCase") == 0) {
+ handleTestCaseTerminate(values);
+ } else if (eventType.compareTo("TestLogData") == 0) {
+ handleTestLogData(values);
+ }
+ }
+
+ /**
+ * Generates tescase trie from dEQP testcase paths. Used to define which testcases to execute.
+ */
+ private String generateTestCaseTrieFromPaths(Collection<String> tests) {
+ String result = "{";
+ boolean first = true;
+
+ // Add testcases to results
+ for (Iterator<String> iter = tests.iterator(); iter.hasNext();) {
+ String test = iter.next();
+ String[] components = test.split("\\.");
+
+ if (components.length == 1) {
+ if (!first) {
+ result = result + ",";
+ }
+ first = false;
+
+ result += components[0];
+ iter.remove();
+ }
+ }
+
+ if (!tests.isEmpty()) {
+ HashMap<String, ArrayList<String> > testGroups = new HashMap();
+
+ // Collect all sub testgroups
+ for (String test : tests) {
+ String[] components = test.split("\\.");
+ ArrayList<String> testGroup = testGroups.get(components[0]);
+
+ if (testGroup == null) {
+ testGroup = new ArrayList();
+ testGroups.put(components[0], testGroup);
+ }
+
+ testGroup.add(test.substring(components[0].length()+1));
+ }
+
+ for (String testGroup : testGroups.keySet()) {
+ if (!first) {
+ result = result + ",";
+ }
+
+ first = false;
+ result = result + testGroup
+ + generateTestCaseTrieFromPaths(testGroups.get(testGroup));
+ }
+ }
+
+ return result + "}";
+ }
+
+ /**
+ * Generates testacase trie from TestIdentifiers.
+ */
+ private String generateTestCaseTrie(Collection<TestIdentifier> tests) {
+ ArrayList<String> testPaths = new ArrayList();
+
+ for (TestIdentifier test : tests) {
+ testPaths.add(test.getClassName() + "." + test.getTestName());
+
+ // Limit number of testcases for each run
+ if (testPaths.size() > TESTCASE_BATCH_LIMIT)
+ break;
+ }
+
+ return generateTestCaseTrieFromPaths(testPaths);
+ }
+
+ /**
+ * Executes tests on the device.
+ */
+ private void executeTests(ITestInvocationListener listener) throws DeviceNotAvailableException {
+ InstrumentationParser parser = new InstrumentationParser(this);
+ String caseListFileName = "/sdcard/dEQP-TestCaseList.txt";
+ String logFileName = "/sdcard/TestLog.qpa";
+ String testCases = generateTestCaseTrie(mTests);
+
+ mDevice.executeShellCommand("rm " + caseListFileName);
+ mDevice.executeShellCommand("rm " + logFileName);
+ mDevice.pushString(testCases + "\n", caseListFileName);
+
+ String instrumentationName =
+ "com.drawelements.deqp/com.drawelements.deqp.testercore.DeqpInstrumentation";
+ String command = "am instrument -w -e deqpLogFileName \"" + logFileName
+ + "\" -e deqpCmdLine \"--deqp-caselist-file=" + caseListFileName + "\" "
+ + (mLogData ? "-e deqpLogData \"true\" " : "") + instrumentationName;
+
+ mDevice.executeShellCommand(command, parser);
+ parser.flush();
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void run(ITestInvocationListener listener) throws DeviceNotAvailableException {
+ mListener = listener;
+
+ while (!mTests.isEmpty()) {
+ executeTests(listener);
+
+ // Set test to failed if it didn't receive test result
+ if (mCurrentTestId != null) {
+ Map <String, String> emptyMap = Collections.emptyMap();
+
+ if (mLogData && mCurrentTestLog != null && mCurrentTestLog.length() > 0) {
+ ByteArrayInputStreamSource source
+ = new ByteArrayInputStreamSource(mCurrentTestLog.getBytes());
+
+ mListener.testLog(mCurrentTestId.getClassName() + "."
+ + mCurrentTestId.getTestName(), LogDataType.XML, source);
+
+ source.cancel();
+ }
+
+
+ if (!mGotTestResult) {
+ mListener.testFailed(ITestRunListener.TestFailure.ERROR, mCurrentTestId,
+ "Log doesn't contain test result");
+ }
+
+ mListener.testEnded(mCurrentTestId, emptyMap);
+ mCurrentTestId = null;
+ mListener.testRunEnded(0, emptyMap);
+ }
+ }
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void setDevice(ITestDevice device) {
+ mDevice = device;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public ITestDevice getDevice() {
+ return mDevice;
+ }
+}
diff --git a/tools/tradefed-host/src/com/android/cts/tradefed/testtype/TestPackageDef.java b/tools/tradefed-host/src/com/android/cts/tradefed/testtype/TestPackageDef.java
index c477585..4fa3b2b 100644
--- a/tools/tradefed-host/src/com/android/cts/tradefed/testtype/TestPackageDef.java
+++ b/tools/tradefed-host/src/com/android/cts/tradefed/testtype/TestPackageDef.java
@@ -46,6 +46,7 @@
public static final String NATIVE_TEST = "native";
public static final String WRAPPED_NATIVE_TEST = "wrappednative";
public static final String VM_HOST_TEST = "vmHostTest";
+ public static final String DEQP_TEST = "deqpTest";
public static final String ACCESSIBILITY_TEST =
"com.android.cts.tradefed.testtype.AccessibilityTestRunner";
public static final String ACCESSIBILITY_SERVICE_TEST =
@@ -233,6 +234,8 @@
vmHostTest.setTests(mTests);
mDigest = generateDigest(testCaseDir, mJarPath);
return vmHostTest;
+ } else if (DEQP_TEST.equals(mTestType)) {
+ return new DeqpTest(mUri, mTests);
} else if (NATIVE_TEST.equals(mTestType)) {
return new GeeTest(mUri, mName);
} else if (WRAPPED_NATIVE_TEST.equals(mTestType)) {
diff --git a/tools/tradefed-host/tests/src/com/android/cts/tradefed/UnitTests.java b/tools/tradefed-host/tests/src/com/android/cts/tradefed/UnitTests.java
index 3c36a3d..65528b7 100644
--- a/tools/tradefed-host/tests/src/com/android/cts/tradefed/UnitTests.java
+++ b/tools/tradefed-host/tests/src/com/android/cts/tradefed/UnitTests.java
@@ -22,6 +22,7 @@
import com.android.cts.tradefed.result.TestSummaryXmlTest;
import com.android.cts.tradefed.result.TestTest;
import com.android.cts.tradefed.testtype.CtsTestTest;
+import com.android.cts.tradefed.testtype.DeqpTestTest;
import com.android.cts.tradefed.testtype.JarHostTestTest;
import com.android.cts.tradefed.testtype.TestFilterTest;
import com.android.cts.tradefed.testtype.TestPackageDefTest;
@@ -59,6 +60,7 @@
addTestSuite(TestPackageXmlParserTest.class);
addTestSuite(TestPlanTest.class);
addTestSuite(WrappedGTestResultParserTest.class);
+ addTestSuite(DeqpTestTest.class);
}
public static Test suite() {
diff --git a/tools/tradefed-host/tests/src/com/android/cts/tradefed/testtype/DeqpTestTest.java b/tools/tradefed-host/tests/src/com/android/cts/tradefed/testtype/DeqpTestTest.java
new file mode 100644
index 0000000..b6e2806
--- /dev/null
+++ b/tools/tradefed-host/tests/src/com/android/cts/tradefed/testtype/DeqpTestTest.java
@@ -0,0 +1,380 @@
+/*
+ * Copyright (C) 2014 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.android.cts.tradefed.testtype;
+
+import com.android.ddmlib.IShellOutputReceiver;
+import com.android.ddmlib.testrunner.ITestRunListener;
+import com.android.ddmlib.testrunner.TestIdentifier;
+import com.android.tradefed.device.ITestDevice;
+import com.android.tradefed.result.ITestInvocationListener;
+
+import junit.framework.TestCase;
+
+import org.easymock.EasyMock;
+import org.easymock.IAnswer;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Map;
+
+/**
+ * Unit tests for {@link DeqpTest}.
+ */
+public class DeqpTestTest extends TestCase {
+ private static final String URI = "dEQP-GLES3";
+ private static final String CASE_LIST_FILE_NAME = "/sdcard/dEQP-TestCaseList.txt";
+ private static final String LOG_FILE_NAME = "/sdcard/TestLog.qpa";
+ private static final String INSTRUMENTATION_NAME =
+ "com.drawelements.deqp/com.drawelements.deqp.testercore.DeqpInstrumentation";
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected void setUp() throws Exception {
+ super.setUp();
+ }
+
+ /**
+ * Test that result code produces correctly pass or fail.
+ */
+ private void testResultCode(final String resultCode, boolean pass) throws Exception {
+ final TestIdentifier testId = new TestIdentifier("dEQP-GLES3.info", "version");
+ final String testPath = "dEQP-GLES3.info.version";
+ final String testTrie = "{dEQP-GLES3{info{version}}}";
+
+ /* MultiLineReceiver expects "\r\n" line ending. */
+ final String output = "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=" + testPath + "\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=" + resultCode + "\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Detail" + resultCode + "\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_CODE: 0\r\n";
+
+ ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class);
+ ITestInvocationListener mockListener
+ = EasyMock.createStrictMock(ITestInvocationListener.class);
+ Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>();
+
+ tests.add(testId);
+
+ DeqpTest deqpTest = new DeqpTest(URI, tests);
+
+ EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("rm " + CASE_LIST_FILE_NAME)))
+ .andReturn("").once();
+
+ EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("rm " + LOG_FILE_NAME)))
+ .andReturn("").once();
+
+ EasyMock.expect(mockDevice.pushString(testTrie + "\n", CASE_LIST_FILE_NAME)).andReturn(true)
+ .once();
+
+ String command = "am instrument -w -e deqpLogFileName \"" + LOG_FILE_NAME
+ + "\" -e deqpCmdLine \"--deqp-caselist-file=" + CASE_LIST_FILE_NAME + "\" "
+ + INSTRUMENTATION_NAME;
+
+ mockDevice.executeShellCommand(EasyMock.eq(command),
+ EasyMock.<IShellOutputReceiver>notNull());
+
+ EasyMock.expectLastCall().andAnswer(new IAnswer<Object>() {
+ @Override
+ public Object answer() {
+ IShellOutputReceiver receiver
+ = (IShellOutputReceiver)EasyMock.getCurrentArguments()[1];
+
+ receiver.addOutput(output.getBytes(), 0, output.length());
+ receiver.flush();
+
+ return null;
+ }
+ });
+
+ mockListener.testRunStarted(URI, 1);
+ EasyMock.expectLastCall().once();
+
+ mockListener.testStarted(EasyMock.eq(testId));
+ EasyMock.expectLastCall().once();
+
+ if (!pass) {
+ mockListener.testFailed(ITestRunListener.TestFailure.ERROR, testId,
+ resultCode + ":Detail" + resultCode);
+
+ EasyMock.expectLastCall().once();
+ }
+
+ mockListener.testEnded(EasyMock.eq(testId), EasyMock.<Map<String, String>>notNull());
+ EasyMock.expectLastCall().once();
+
+ mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<Map<String, String>>notNull());
+ EasyMock.expectLastCall().once();
+
+ EasyMock.replay(mockDevice);
+ EasyMock.replay(mockListener);
+
+ deqpTest.setDevice(mockDevice);
+ deqpTest.run(mockListener);
+
+ EasyMock.verify(mockListener);
+ EasyMock.verify(mockDevice);
+ }
+
+ /**
+ * Test running multiple test cases.
+ */
+ public void testRun_multipleTets() throws Exception {
+ /* MultiLineReceiver expects "\r\n" line ending. */
+ final String output = "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.vendor\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.renderer\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.version\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.shading_language_version\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.extensions\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.render_target\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_CODE: 0\r\n";
+
+ final TestIdentifier[] testIds = {
+ new TestIdentifier("dEQP-GLES3.info", "vendor"),
+ new TestIdentifier("dEQP-GLES3.info", "renderer"),
+ new TestIdentifier("dEQP-GLES3.info", "version"),
+ new TestIdentifier("dEQP-GLES3.info", "shading_language_version"),
+ new TestIdentifier("dEQP-GLES3.info", "extensions"),
+ new TestIdentifier("dEQP-GLES3.info", "render_target")
+ };
+
+ final String[] testPaths = {
+ "dEQP-GLES3.info.vendor",
+ "dEQP-GLES3.info.renderer",
+ "dEQP-GLES3.info.version",
+ "dEQP-GLES3.info.shading_language_version",
+ "dEQP-GLES3.info.extensions",
+ "dEQP-GLES3.info.render_target"
+ };
+
+ final String testTrie
+ = "{dEQP-GLES3{info{vendor,renderer,version,shading_language_version,extensions,render_target}}}";
+
+ ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class);
+ ITestInvocationListener mockListener = EasyMock.createStrictMock(ITestInvocationListener.class);
+ Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>();
+
+ for (TestIdentifier id : testIds) {
+ tests.add(id);
+ }
+
+ DeqpTest deqpTest = new DeqpTest(URI, tests);
+
+ EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("rm " + CASE_LIST_FILE_NAME)))
+ .andReturn("").once();
+
+ EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("rm " + LOG_FILE_NAME)))
+ .andReturn("").once();
+
+ EasyMock.expect(mockDevice.pushString(testTrie + "\n", CASE_LIST_FILE_NAME))
+ .andReturn(true).once();
+
+ String command = "am instrument -w -e deqpLogFileName \"" + LOG_FILE_NAME
+ + "\" -e deqpCmdLine \"--deqp-caselist-file=" + CASE_LIST_FILE_NAME + "\" "
+ + INSTRUMENTATION_NAME;
+
+ mockDevice.executeShellCommand(EasyMock.eq(command),
+ EasyMock.<IShellOutputReceiver>notNull());
+
+ EasyMock.expectLastCall().andAnswer(new IAnswer<Object>() {
+ @Override
+ public Object answer() {
+ IShellOutputReceiver receiver
+ = (IShellOutputReceiver)EasyMock.getCurrentArguments()[1];
+
+ receiver.addOutput(output.getBytes(), 0, output.length());
+ receiver.flush();
+
+ return null;
+ }
+ });
+
+ mockListener.testRunStarted(URI, testPaths.length);
+ EasyMock.expectLastCall().once();
+
+ for (int i = 0; i < testPaths.length; i++) {
+ mockListener.testStarted(EasyMock.eq(testIds[i]));
+ EasyMock.expectLastCall().once();
+
+ mockListener.testEnded(EasyMock.eq(testIds[i]),
+ EasyMock.<Map<String, String>>notNull());
+
+ EasyMock.expectLastCall().once();
+ }
+
+ mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<Map<String, String>>notNull());
+ EasyMock.expectLastCall().once();
+
+ EasyMock.replay(mockDevice);
+ EasyMock.replay(mockListener);
+
+ deqpTest.setDevice(mockDevice);
+ deqpTest.run(mockListener);
+
+ EasyMock.verify(mockListener);
+ EasyMock.verify(mockDevice);
+ }
+
+ /**
+ * Test dEQP Pass result code.
+ */
+ public void testRun_resultPass() throws Exception {
+ testResultCode("Pass", true);
+ }
+
+ /**
+ * Test dEQP Fail result code.
+ */
+ public void testRun_resultFail() throws Exception {
+ testResultCode("Fail", false);
+ }
+
+ /**
+ * Test dEQP NotSupported result code.
+ */
+ public void testRun_resultNotSupported() throws Exception {
+ testResultCode("NotSupported", true);
+ }
+
+ /**
+ * Test dEQP QualityWarning result code.
+ */
+ public void testRun_resultQualityWarning() throws Exception {
+ testResultCode("QualityWarning", true);
+ }
+
+ /**
+ * Test dEQP CompatibilityWarning result code.
+ */
+ public void testRun_resultCompatibilityWarning() throws Exception {
+ testResultCode("CompatibilityWarning", true);
+ }
+
+ /**
+ * Test dEQP ResourceError result code.
+ */
+ public void testRun_resultResourceError() throws Exception {
+ testResultCode("ResourceError", false);
+ }
+
+ /**
+ * Test dEQP InternalError result code.
+ */
+ public void testRun_resultInternalError() throws Exception {
+ testResultCode("InternalError", false);
+ }
+
+ /**
+ * Test dEQP Crash result code.
+ */
+ public void testRun_resultCrash() throws Exception {
+ testResultCode("Crash", false);
+ }
+
+ /**
+ * Test dEQP Timeout result code.
+ */
+ public void testRun_resultTimeout() throws Exception {
+ testResultCode("Timeout", false);
+ }
+}