CTS v2 integration for deqp
Move the runner from the CTS tree into the deqp tree and modify the
runner to use the parametrization conventions of the CTS v2. Add support
for filters.
Generate the test configuration with build_android_mustpass.py utility
Ported tests from the old runner and added tests for filtering.
Change-Id: Ibc87b78f302e363b878f2631ef58defa222a2110
diff --git a/Android.mk b/Android.mk
index e4c6997..038f891 100644
--- a/Android.mk
+++ b/Android.mk
@@ -746,4 +746,8 @@
include $(BUILD_SHARED_LIBRARY)
-include $(LOCAL_PATH)/android/package/Android.mk
+
+# Build the test APKs using their own makefiles
+# include $(call all-makefiles-under,$(LOCAL_PATH)/android)
+
+include $(LOCAL_PATH)/android/package/Android.mk $(LOCAL_PATH)/android/cts/Android.mk
diff --git a/android/cts/Android.mk b/android/cts/Android.mk
new file mode 100644
index 0000000..b3f167a
--- /dev/null
+++ b/android/cts/Android.mk
@@ -0,0 +1,40 @@
+# Copyright (C) 2015 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+LOCAL_PATH := $(call my-dir)
+
+include $(CLEAR_VARS)
+
+LOCAL_MODULE := CtsDeqpTestCases
+
+LOCAL_MODULE_TAGS := optional
+
+# Tag this module as a cts_v2 test artifact
+LOCAL_COMPATIBILITY_SUITE := cts_v2
+
+LOCAL_SDK_VERSION := 22
+
+LOCAL_SRC_FILES := $(call all-java-files-under, runner/src)
+LOCAL_JAVA_LIBRARIES := cts-tradefed_v2 compatibility-host-util tradefed-prebuilt
+
+DEQP_CASELISTS:=$(sort $(patsubst mnc/%,%, \
+ $(shell cd $(LOCAL_PATH) ; \
+ find -L mnc -maxdepth 1 -name "*.txt") \
+ ))
+LOCAL_COMPATIBILITY_SUPPORT_FILES := $(foreach file, $(DEQP_CASELISTS), ./mnc/$(file):$(file))
+
+
+include $(BUILD_HOST_JAVA_LIBRARY)
+
+include $(call all-makefiles-under,$(LOCAL_PATH))
diff --git a/android/cts/AndroidTest.xml b/android/cts/AndroidTest.xml
new file mode 100644
index 0000000..c61f982
--- /dev/null
+++ b/android/cts/AndroidTest.xml
@@ -0,0 +1,137 @@
+<?xml version="1.0" encoding="utf-8"?>
+<configuration>
+ <!--
+ Copyright (C) 2015 The Android Open Source Project
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+ <!--
+ This file has been automatically generated. Edit with caution.
+ -->
+ <target_preparer class="com.android.compatibility.common.tradefed.targetprep.ApkInstaller">
+ <option name="cleanup-apks" value="true"/>
+ <option name="test-file-name" value="com.drawelements.deqp.apk"/>
+ </target_preparer>
+ <test class="com.drawelements.deqp.runner.DeqpTestRunner">
+ <option name="deqp-package" value="dEQP-EGL"/>
+ <option name="deqp-caselist-file" value="egl-master.txt"/>
+ <option name="deqp-gl-config-name" value="rgba8888d24s8ms0"/>
+ <option name="deqp-surface-type" value="window"/>
+ <option name="deqp-screen-rotation" value="unspecified"/>
+ </test>
+ <test class="com.drawelements.deqp.runner.DeqpTestRunner">
+ <option name="deqp-package" value="dEQP-GLES2"/>
+ <option name="deqp-caselist-file" value="gles2-master.txt"/>
+ <option name="deqp-gl-config-name" value="rgba8888d24s8ms0"/>
+ <option name="deqp-surface-type" value="window"/>
+ <option name="deqp-screen-rotation" value="unspecified"/>
+ </test>
+ <test class="com.drawelements.deqp.runner.DeqpTestRunner">
+ <option name="deqp-package" value="dEQP-GLES3"/>
+ <option name="deqp-caselist-file" value="gles3-master.txt"/>
+ <option name="deqp-gl-config-name" value="rgba8888d24s8ms0"/>
+ <option name="deqp-surface-type" value="window"/>
+ <option name="deqp-screen-rotation" value="unspecified"/>
+ </test>
+ <test class="com.drawelements.deqp.runner.DeqpTestRunner">
+ <option name="deqp-package" value="dEQP-GLES3"/>
+ <option name="deqp-caselist-file" value="gles3-rotate-portrait.txt"/>
+ <option name="deqp-gl-config-name" value="rgba8888d24s8ms0"/>
+ <option name="deqp-surface-type" value="window"/>
+ <option name="deqp-screen-rotation" value="0"/>
+ </test>
+ <test class="com.drawelements.deqp.runner.DeqpTestRunner">
+ <option name="deqp-package" value="dEQP-GLES3"/>
+ <option name="deqp-caselist-file" value="gles3-rotate-landscape.txt"/>
+ <option name="deqp-gl-config-name" value="rgba8888d24s8ms0"/>
+ <option name="deqp-surface-type" value="window"/>
+ <option name="deqp-screen-rotation" value="90"/>
+ </test>
+ <test class="com.drawelements.deqp.runner.DeqpTestRunner">
+ <option name="deqp-package" value="dEQP-GLES3"/>
+ <option name="deqp-caselist-file" value="gles3-rotate-reverse-portrait.txt"/>
+ <option name="deqp-gl-config-name" value="rgba8888d24s8ms0"/>
+ <option name="deqp-surface-type" value="window"/>
+ <option name="deqp-screen-rotation" value="180"/>
+ </test>
+ <test class="com.drawelements.deqp.runner.DeqpTestRunner">
+ <option name="deqp-package" value="dEQP-GLES3"/>
+ <option name="deqp-caselist-file" value="gles3-rotate-reverse-landscape.txt"/>
+ <option name="deqp-gl-config-name" value="rgba8888d24s8ms0"/>
+ <option name="deqp-surface-type" value="window"/>
+ <option name="deqp-screen-rotation" value="270"/>
+ </test>
+ <test class="com.drawelements.deqp.runner.DeqpTestRunner">
+ <option name="deqp-package" value="dEQP-GLES3"/>
+ <option name="deqp-caselist-file" value="gles3-multisample.txt"/>
+ <option name="deqp-gl-config-name" value="rgba8888d24s8ms4"/>
+ <option name="deqp-surface-type" value="window"/>
+ <option name="deqp-screen-rotation" value="unspecified"/>
+ </test>
+ <test class="com.drawelements.deqp.runner.DeqpTestRunner">
+ <option name="deqp-package" value="dEQP-GLES3"/>
+ <option name="deqp-caselist-file" value="gles3-565-no-depth-no-stencil.txt"/>
+ <option name="deqp-gl-config-name" value="rgb565d0s0ms0"/>
+ <option name="deqp-surface-type" value="window"/>
+ <option name="deqp-screen-rotation" value="unspecified"/>
+ </test>
+ <test class="com.drawelements.deqp.runner.DeqpTestRunner">
+ <option name="deqp-package" value="dEQP-GLES31"/>
+ <option name="deqp-caselist-file" value="gles31-master.txt"/>
+ <option name="deqp-gl-config-name" value="rgba8888d24s8ms0"/>
+ <option name="deqp-surface-type" value="window"/>
+ <option name="deqp-screen-rotation" value="unspecified"/>
+ </test>
+ <test class="com.drawelements.deqp.runner.DeqpTestRunner">
+ <option name="deqp-package" value="dEQP-GLES31"/>
+ <option name="deqp-caselist-file" value="gles31-rotate-portrait.txt"/>
+ <option name="deqp-gl-config-name" value="rgba8888d24s8ms0"/>
+ <option name="deqp-surface-type" value="window"/>
+ <option name="deqp-screen-rotation" value="0"/>
+ </test>
+ <test class="com.drawelements.deqp.runner.DeqpTestRunner">
+ <option name="deqp-package" value="dEQP-GLES31"/>
+ <option name="deqp-caselist-file" value="gles31-rotate-landscape.txt"/>
+ <option name="deqp-gl-config-name" value="rgba8888d24s8ms0"/>
+ <option name="deqp-surface-type" value="window"/>
+ <option name="deqp-screen-rotation" value="90"/>
+ </test>
+ <test class="com.drawelements.deqp.runner.DeqpTestRunner">
+ <option name="deqp-package" value="dEQP-GLES31"/>
+ <option name="deqp-caselist-file" value="gles31-rotate-reverse-portrait.txt"/>
+ <option name="deqp-gl-config-name" value="rgba8888d24s8ms0"/>
+ <option name="deqp-surface-type" value="window"/>
+ <option name="deqp-screen-rotation" value="180"/>
+ </test>
+ <test class="com.drawelements.deqp.runner.DeqpTestRunner">
+ <option name="deqp-package" value="dEQP-GLES31"/>
+ <option name="deqp-caselist-file" value="gles31-rotate-reverse-landscape.txt"/>
+ <option name="deqp-gl-config-name" value="rgba8888d24s8ms0"/>
+ <option name="deqp-surface-type" value="window"/>
+ <option name="deqp-screen-rotation" value="270"/>
+ </test>
+ <test class="com.drawelements.deqp.runner.DeqpTestRunner">
+ <option name="deqp-package" value="dEQP-GLES31"/>
+ <option name="deqp-caselist-file" value="gles31-multisample.txt"/>
+ <option name="deqp-gl-config-name" value="rgba8888d24s8ms4"/>
+ <option name="deqp-surface-type" value="window"/>
+ <option name="deqp-screen-rotation" value="unspecified"/>
+ </test>
+ <test class="com.drawelements.deqp.runner.DeqpTestRunner">
+ <option name="deqp-package" value="dEQP-GLES31"/>
+ <option name="deqp-caselist-file" value="gles31-565-no-depth-no-stencil.txt"/>
+ <option name="deqp-gl-config-name" value="rgb565d0s0ms0"/>
+ <option name="deqp-surface-type" value="window"/>
+ <option name="deqp-screen-rotation" value="unspecified"/>
+ </test>
+</configuration>
diff --git a/android/cts/lmp-mr1/com.drawelements.deqp.gles3.xml b/android/cts/lmp-mr1/com.drawelements.deqp.gles3.xml
index be78487..b41f4e4 100644
--- a/android/cts/lmp-mr1/com.drawelements.deqp.gles3.xml
+++ b/android/cts/lmp-mr1/com.drawelements.deqp.gles3.xml
@@ -1,5 +1,23 @@
<?xml version="1.0" encoding="utf-8"?>
<TestPackage appPackageName="com.drawelements.deqp.gles3" deqp:glesVersion="196608" name="dEQP-GLES3" testType="deqpTest" xmlns:deqp="http://drawelements.com/deqp">
+ <!--
+ Copyright (C) 2015 The Android Open Source Project
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+ <!--
+ This file has been automatically generated. Edit with caution.
+ -->
<TestSuite name="dEQP-GLES3">
<TestCase name="info">
<Test name="vendor">
diff --git a/android/cts/lmp-mr1/com.drawelements.deqp.gles31.xml b/android/cts/lmp-mr1/com.drawelements.deqp.gles31.xml
index 3af235b..985279b 100644
--- a/android/cts/lmp-mr1/com.drawelements.deqp.gles31.xml
+++ b/android/cts/lmp-mr1/com.drawelements.deqp.gles31.xml
@@ -1,5 +1,23 @@
<?xml version="1.0" encoding="utf-8"?>
<TestPackage appPackageName="com.drawelements.deqp.gles31" deqp:glesVersion="196609" name="dEQP-GLES31" testType="deqpTest" xmlns:deqp="http://drawelements.com/deqp">
+ <!--
+ Copyright (C) 2015 The Android Open Source Project
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+ <!--
+ This file has been automatically generated. Edit with caution.
+ -->
<TestSuite name="dEQP-GLES31">
<TestCase name="info">
<Test name="vendor">
diff --git a/android/cts/lmp-mr1/mustpass.xml b/android/cts/lmp-mr1/mustpass.xml
index bae28d3..76e34cc 100644
--- a/android/cts/lmp-mr1/mustpass.xml
+++ b/android/cts/lmp-mr1/mustpass.xml
@@ -1,5 +1,23 @@
<?xml version="1.0" encoding="utf-8"?>
<Mustpass version="lmp-mr1">
+ <!--
+ Copyright (C) 2015 The Android Open Source Project
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+ <!--
+ This file has been automatically generated. Edit with caution.
+ -->
<TestPackage name="dEQP-GLES3">
<Configuration caseListFile="gles3-master.txt" commandLine="--deqp-gl-config-name=rgba8888d24s8ms0 --deqp-screen-rotation=unspecified --deqp-surface-type=window --deqp-watchdog=enable" name="master"/>
</TestPackage>
diff --git a/android/cts/lmp/com.drawelements.deqp.gles3.xml b/android/cts/lmp/com.drawelements.deqp.gles3.xml
index b068e3b..c417472 100644
--- a/android/cts/lmp/com.drawelements.deqp.gles3.xml
+++ b/android/cts/lmp/com.drawelements.deqp.gles3.xml
@@ -1,5 +1,23 @@
<?xml version="1.0" encoding="utf-8"?>
<TestPackage appPackageName="com.drawelements.deqp.gles3" deqp:glesVersion="196608" name="dEQP-GLES3" testType="deqpTest" xmlns:deqp="http://drawelements.com/deqp">
+ <!--
+ Copyright (C) 2015 The Android Open Source Project
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+ <!--
+ This file has been automatically generated. Edit with caution.
+ -->
<TestSuite name="dEQP-GLES3">
<TestCase name="info">
<Test name="vendor">
diff --git a/android/cts/lmp/com.drawelements.deqp.gles31.xml b/android/cts/lmp/com.drawelements.deqp.gles31.xml
index dde2070..00c367c 100644
--- a/android/cts/lmp/com.drawelements.deqp.gles31.xml
+++ b/android/cts/lmp/com.drawelements.deqp.gles31.xml
@@ -1,5 +1,23 @@
<?xml version="1.0" encoding="utf-8"?>
<TestPackage appPackageName="com.drawelements.deqp.gles31" deqp:glesVersion="196609" name="dEQP-GLES31" testType="deqpTest" xmlns:deqp="http://drawelements.com/deqp">
+ <!--
+ Copyright (C) 2015 The Android Open Source Project
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+ <!--
+ This file has been automatically generated. Edit with caution.
+ -->
<TestSuite name="dEQP-GLES31">
<TestCase name="info">
<Test name="vendor">
diff --git a/android/cts/lmp/mustpass.xml b/android/cts/lmp/mustpass.xml
index abc191f..e32573f 100644
--- a/android/cts/lmp/mustpass.xml
+++ b/android/cts/lmp/mustpass.xml
@@ -1,5 +1,23 @@
<?xml version="1.0" encoding="utf-8"?>
<Mustpass version="lmp">
+ <!--
+ Copyright (C) 2015 The Android Open Source Project
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+ <!--
+ This file has been automatically generated. Edit with caution.
+ -->
<TestPackage name="dEQP-GLES3">
<Configuration caseListFile="gles3-master.txt" commandLine="--deqp-gl-config-name=rgba8888d24s8ms0 --deqp-screen-rotation=unspecified --deqp-surface-type=window --deqp-watchdog=enable" name="master"/>
</TestPackage>
diff --git a/android/cts/master/com.drawelements.deqp.egl.xml b/android/cts/master/com.drawelements.deqp.egl.xml
index 58cdf61..c801d48 100644
--- a/android/cts/master/com.drawelements.deqp.egl.xml
+++ b/android/cts/master/com.drawelements.deqp.egl.xml
@@ -1,5 +1,23 @@
<?xml version="1.0" encoding="utf-8"?>
<TestPackage appPackageName="com.drawelements.deqp.egl" deqp:glesVersion="131072" name="dEQP-EGL" testType="deqpTest" xmlns:deqp="http://drawelements.com/deqp">
+ <!--
+ Copyright (C) 2015 The Android Open Source Project
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+ <!--
+ This file has been automatically generated. Edit with caution.
+ -->
<TestSuite name="dEQP-EGL">
<TestCase name="info">
<Test name="version">
diff --git a/android/cts/master/com.drawelements.deqp.gles2.xml b/android/cts/master/com.drawelements.deqp.gles2.xml
index adbc036..88e9dc8 100644
--- a/android/cts/master/com.drawelements.deqp.gles2.xml
+++ b/android/cts/master/com.drawelements.deqp.gles2.xml
@@ -1,5 +1,23 @@
<?xml version="1.0" encoding="utf-8"?>
<TestPackage appPackageName="com.drawelements.deqp.gles2" deqp:glesVersion="131072" name="dEQP-GLES2" testType="deqpTest" xmlns:deqp="http://drawelements.com/deqp">
+ <!--
+ Copyright (C) 2015 The Android Open Source Project
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+ <!--
+ This file has been automatically generated. Edit with caution.
+ -->
<TestSuite name="dEQP-GLES2">
<TestCase name="info">
<Test name="vendor">
diff --git a/android/cts/master/com.drawelements.deqp.gles3.xml b/android/cts/master/com.drawelements.deqp.gles3.xml
index e7bef92..e58122e 100644
--- a/android/cts/master/com.drawelements.deqp.gles3.xml
+++ b/android/cts/master/com.drawelements.deqp.gles3.xml
@@ -1,5 +1,23 @@
<?xml version="1.0" encoding="utf-8"?>
<TestPackage appPackageName="com.drawelements.deqp.gles3" deqp:glesVersion="196608" name="dEQP-GLES3" testType="deqpTest" xmlns:deqp="http://drawelements.com/deqp">
+ <!--
+ Copyright (C) 2015 The Android Open Source Project
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+ <!--
+ This file has been automatically generated. Edit with caution.
+ -->
<TestSuite name="dEQP-GLES3">
<TestCase name="info">
<Test name="vendor">
diff --git a/android/cts/master/com.drawelements.deqp.gles31.xml b/android/cts/master/com.drawelements.deqp.gles31.xml
index b56b6b9..df29b69 100644
--- a/android/cts/master/com.drawelements.deqp.gles31.xml
+++ b/android/cts/master/com.drawelements.deqp.gles31.xml
@@ -1,5 +1,23 @@
<?xml version="1.0" encoding="utf-8"?>
<TestPackage appPackageName="com.drawelements.deqp.gles31" deqp:glesVersion="196609" name="dEQP-GLES31" testType="deqpTest" xmlns:deqp="http://drawelements.com/deqp">
+ <!--
+ Copyright (C) 2015 The Android Open Source Project
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+ <!--
+ This file has been automatically generated. Edit with caution.
+ -->
<TestSuite name="dEQP-GLES31">
<TestCase name="info">
<Test name="vendor">
diff --git a/android/cts/master/mustpass.xml b/android/cts/master/mustpass.xml
index f8c924a..3388e4c 100644
--- a/android/cts/master/mustpass.xml
+++ b/android/cts/master/mustpass.xml
@@ -1,5 +1,23 @@
<?xml version="1.0" encoding="utf-8"?>
<Mustpass version="master">
+ <!--
+ Copyright (C) 2015 The Android Open Source Project
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+ <!--
+ This file has been automatically generated. Edit with caution.
+ -->
<TestPackage name="dEQP-EGL">
<Configuration caseListFile="egl-master.txt" commandLine="--deqp-gl-config-name=rgba8888d24s8ms0 --deqp-screen-rotation=unspecified --deqp-surface-type=window --deqp-watchdog=enable" name="master"/>
</TestPackage>
diff --git a/android/cts/mnc/com.drawelements.deqp.egl.xml b/android/cts/mnc/com.drawelements.deqp.egl.xml
index 73784a3..4694698 100644
--- a/android/cts/mnc/com.drawelements.deqp.egl.xml
+++ b/android/cts/mnc/com.drawelements.deqp.egl.xml
@@ -1,5 +1,23 @@
<?xml version="1.0" encoding="utf-8"?>
<TestPackage appPackageName="com.drawelements.deqp.egl" deqp:glesVersion="131072" name="dEQP-EGL" testType="deqpTest" xmlns:deqp="http://drawelements.com/deqp">
+ <!--
+ Copyright (C) 2015 The Android Open Source Project
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+ <!--
+ This file has been automatically generated. Edit with caution.
+ -->
<TestSuite name="dEQP-EGL">
<TestCase name="info">
<Test name="version">
diff --git a/android/cts/mnc/com.drawelements.deqp.gles2.xml b/android/cts/mnc/com.drawelements.deqp.gles2.xml
index adbc036..88e9dc8 100644
--- a/android/cts/mnc/com.drawelements.deqp.gles2.xml
+++ b/android/cts/mnc/com.drawelements.deqp.gles2.xml
@@ -1,5 +1,23 @@
<?xml version="1.0" encoding="utf-8"?>
<TestPackage appPackageName="com.drawelements.deqp.gles2" deqp:glesVersion="131072" name="dEQP-GLES2" testType="deqpTest" xmlns:deqp="http://drawelements.com/deqp">
+ <!--
+ Copyright (C) 2015 The Android Open Source Project
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+ <!--
+ This file has been automatically generated. Edit with caution.
+ -->
<TestSuite name="dEQP-GLES2">
<TestCase name="info">
<Test name="vendor">
diff --git a/android/cts/mnc/com.drawelements.deqp.gles3.xml b/android/cts/mnc/com.drawelements.deqp.gles3.xml
index d8f6c4b..171ca14 100644
--- a/android/cts/mnc/com.drawelements.deqp.gles3.xml
+++ b/android/cts/mnc/com.drawelements.deqp.gles3.xml
@@ -1,5 +1,23 @@
<?xml version="1.0" encoding="utf-8"?>
<TestPackage appPackageName="com.drawelements.deqp.gles3" deqp:glesVersion="196608" name="dEQP-GLES3" testType="deqpTest" xmlns:deqp="http://drawelements.com/deqp">
+ <!--
+ Copyright (C) 2015 The Android Open Source Project
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+ <!--
+ This file has been automatically generated. Edit with caution.
+ -->
<TestSuite name="dEQP-GLES3">
<TestCase name="info">
<Test name="vendor">
diff --git a/android/cts/mnc/com.drawelements.deqp.gles31.xml b/android/cts/mnc/com.drawelements.deqp.gles31.xml
index 5891262..a30add5 100644
--- a/android/cts/mnc/com.drawelements.deqp.gles31.xml
+++ b/android/cts/mnc/com.drawelements.deqp.gles31.xml
@@ -1,5 +1,23 @@
<?xml version="1.0" encoding="utf-8"?>
<TestPackage appPackageName="com.drawelements.deqp.gles31" deqp:glesVersion="196609" name="dEQP-GLES31" testType="deqpTest" xmlns:deqp="http://drawelements.com/deqp">
+ <!--
+ Copyright (C) 2015 The Android Open Source Project
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+ <!--
+ This file has been automatically generated. Edit with caution.
+ -->
<TestSuite name="dEQP-GLES31">
<TestCase name="info">
<Test name="vendor">
diff --git a/android/cts/mnc/mustpass.xml b/android/cts/mnc/mustpass.xml
index dbaf3e8..bd5d337 100644
--- a/android/cts/mnc/mustpass.xml
+++ b/android/cts/mnc/mustpass.xml
@@ -1,5 +1,23 @@
<?xml version="1.0" encoding="utf-8"?>
<Mustpass version="mnc">
+ <!--
+ Copyright (C) 2015 The Android Open Source Project
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+ <!--
+ This file has been automatically generated. Edit with caution.
+ -->
<TestPackage name="dEQP-EGL">
<Configuration caseListFile="egl-master.txt" commandLine="--deqp-gl-config-name=rgba8888d24s8ms0 --deqp-screen-rotation=unspecified --deqp-surface-type=window --deqp-watchdog=enable" name="master"/>
</TestPackage>
diff --git a/android/cts/runner/Android.mk b/android/cts/runner/Android.mk
new file mode 100644
index 0000000..cb61b2a
--- /dev/null
+++ b/android/cts/runner/Android.mk
@@ -0,0 +1,20 @@
+# Copyright (C) 2015 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+LOCAL_PATH := $(call my-dir)
+
+include $(CLEAR_VARS)
+
+# Build all sub-directories
+include $(call all-makefiles-under,$(LOCAL_PATH))
diff --git a/android/cts/runner/src/com/drawelements/deqp/runner/DeqpTestRunner.java b/android/cts/runner/src/com/drawelements/deqp/runner/DeqpTestRunner.java
new file mode 100644
index 0000000..79d8d13
--- /dev/null
+++ b/android/cts/runner/src/com/drawelements/deqp/runner/DeqpTestRunner.java
@@ -0,0 +1,1959 @@
+/*
+ * Copyright (C) 2014 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.drawelements.deqp.runner;
+
+import com.android.compatibility.common.tradefed.build.CompatibilityBuildHelper;
+import com.android.compatibility.common.util.AbiUtils;
+import com.android.ddmlib.AdbCommandRejectedException;
+import com.android.ddmlib.IShellOutputReceiver;
+import com.android.ddmlib.MultiLineReceiver;
+import com.android.ddmlib.ShellCommandUnresponsiveException;
+import com.android.ddmlib.TimeoutException;
+import com.android.ddmlib.testrunner.TestIdentifier;
+import com.android.tradefed.build.IBuildInfo;
+import com.android.tradefed.build.IFolderBuildInfo;
+import com.android.tradefed.config.Option;
+import com.android.tradefed.config.OptionClass;
+import com.android.tradefed.device.DeviceNotAvailableException;
+import com.android.tradefed.device.ITestDevice;
+import com.android.tradefed.log.LogUtil.CLog;
+import com.android.tradefed.result.ByteArrayInputStreamSource;
+import com.android.tradefed.result.ITestInvocationListener;
+import com.android.tradefed.result.LogDataType;
+import com.android.tradefed.testtype.IAbi;
+import com.android.tradefed.testtype.IBuildReceiver;
+import com.android.tradefed.testtype.IAbiReceiver;
+import com.android.tradefed.testtype.IDeviceTest;
+import com.android.tradefed.testtype.IRemoteTest;
+import com.android.tradefed.testtype.ITestFilterReceiver;
+import com.android.tradefed.util.IRunUtil;
+import com.android.tradefed.util.RunInterruptedException;
+import com.android.tradefed.util.RunUtil;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileReader;
+import java.io.IOException;
+import java.io.Reader;
+import java.lang.reflect.Method;
+import java.lang.reflect.InvocationTargetException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.regex.Pattern;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Test runner for dEQP tests
+ *
+ * Supports running drawElements Quality Program tests found under external/deqp.
+ */
+@OptionClass(alias="deqp-test-runner")
+public class DeqpTestRunner implements IBuildReceiver, IDeviceTest, IRemoteTest, ITestFilterReceiver, IAbiReceiver {
+
+ private static final String DEQP_ONDEVICE_APK = "com.drawelements.deqp.apk";
+ private static final String DEQP_ONDEVICE_PKG = "com.drawelements.deqp";
+ private static final String INCOMPLETE_LOG_MESSAGE = "Crash: Incomplete test log";
+ private static final String SKIPPED_INSTANCE_LOG_MESSAGE = "Configuration skipped";
+ private static final String NOT_EXECUTABLE_LOG_MESSAGE = "Abort: Test cannot be executed";
+ private static final String CASE_LIST_FILE_NAME = "/sdcard/dEQP-TestCaseList.txt";
+ private static final String LOG_FILE_NAME = "/sdcard/TestLog.qpa";
+ public static final String FEATURE_LANDSCAPE = "android.hardware.screen.landscape";
+ public static final String FEATURE_PORTRAIT = "android.hardware.screen.portrait";
+
+ private static final int TESTCASE_BATCH_LIMIT = 1000;
+ private static final BatchRunConfiguration DEFAULT_CONFIG =
+ new BatchRunConfiguration("rgba8888d24s8", "unspecified", "window");
+
+ private static final int UNRESPOSIVE_CMD_TIMEOUT_MS = 60000; // one minute
+
+ @Option(name="deqp-package", description="Name of the deqp module used. Determines GLES version.", importance=Option.Importance.ALWAYS)
+ private String mDeqpPackage;
+ @Option(name="deqp-gl-config-name", description="GL render target config. See deqp documentation for syntax. ", importance=Option.Importance.ALWAYS)
+ private String mConfigName;
+ @Option(name="deqp-caselist-file", description="File listing the names of the cases to be run.", importance=Option.Importance.ALWAYS)
+ private String mCaselistFile;
+ @Option(name="deqp-screen-rotation", description="Screen orientation. Defaults to 'unspecified'", importance=Option.Importance.NEVER)
+ private String mScreenRotation = "unspecified";
+ @Option(name="deqp-surface-type", description="Surface type ('window', 'pbuffer', 'fbo'). Defaults to 'window'", importance=Option.Importance.NEVER)
+ private String mSurfaceType = "window";
+
+ @Option(name = "include-filter",
+ description="Test include filter. '*' is zero or more letters. '.' has no special meaning.")
+ private List<String> mIncludeFilters = new ArrayList<>();
+ @Option(name = "exclude-filter",
+ description="Test exclude filter. '*' is zero or more letters. '.' has no special meaning.")
+ private List<String> mExcludeFilters = new ArrayList<>();
+ private Collection<TestIdentifier> mRemainingTests = null;
+ private Map<TestIdentifier, Set<BatchRunConfiguration>> mTestInstances = null;
+ private final TestInstanceResultListener mInstanceListerner = new TestInstanceResultListener();
+ private final Map<TestIdentifier, Integer> mTestInstabilityRatings = new HashMap<>();
+ private IAbi mAbi;
+ private CompatibilityBuildHelper mBuildHelper;
+ private boolean mLogData = false;
+ private ITestDevice mDevice;
+ private Set<String> mDeviceFeatures;
+ private Map<String, Boolean> mConfigQuerySupportCache = new HashMap<>();
+ private IRunUtil mRunUtil = RunUtil.getDefault();
+ // When set will override the mCaselistFile for testing purposes.
+ private Reader mCaselistReader = null;
+
+ private IRecovery mDeviceRecovery = new Recovery();
+ {
+ mDeviceRecovery.setSleepProvider(new SleepProvider());
+ }
+
+ /**
+ * @param abi the ABI to run the test on
+ */
+ @Override
+ public void setAbi(IAbi abi) {
+ mAbi = abi;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void setBuild(IBuildInfo buildInfo) {
+ setBuildHelper(new CompatibilityBuildHelper((IFolderBuildInfo)buildInfo));
+ }
+
+ /**
+ * Exposed for better mockability during testing. In real use, always flows from
+ * setBuild() called by the framework
+ */
+ public void setBuildHelper(CompatibilityBuildHelper helper) {
+ mBuildHelper = helper;
+ }
+
+ /**
+ * Enable or disable raw dEQP test log collection.
+ */
+ public void setCollectLogs(boolean logData) {
+ mLogData = logData;
+ }
+
+ /**
+ * Get the deqp-package option contents.
+ */
+ public String getPackageName() {
+ return mDeqpPackage;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void setDevice(ITestDevice device) {
+ mDevice = device;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public ITestDevice getDevice() {
+ return mDevice;
+ }
+
+ /**
+ * Set recovery handler.
+ *
+ * Exposed for unit testing.
+ */
+ public void setRecovery(IRecovery deviceRecovery) {
+ mDeviceRecovery = deviceRecovery;
+ }
+
+ /**
+ * Set IRunUtil.
+ *
+ * Exposed for unit testing.
+ */
+ public void setRunUtil(IRunUtil runUtil) {
+ mRunUtil = runUtil;
+ }
+
+ /**
+ * Exposed for unit testing
+ */
+ public void setCaselistReader(Reader caselistReader) {
+ mCaselistReader = caselistReader;
+ }
+
+ private static final class CapabilityQueryFailureException extends Exception {
+ }
+
+ /**
+ * Test configuration of dEPQ test instance execution.
+ * Exposed for unit testing
+ */
+ public static final class BatchRunConfiguration {
+ public static final String ROTATION_UNSPECIFIED = "unspecified";
+ public static final String ROTATION_PORTRAIT = "0";
+ public static final String ROTATION_LANDSCAPE = "90";
+ public static final String ROTATION_REVERSE_PORTRAIT = "180";
+ public static final String ROTATION_REVERSE_LANDSCAPE = "270";
+
+ private final String mGlConfig;
+ private final String mRotation;
+ private final String mSurfaceType;
+
+ public BatchRunConfiguration(String glConfig, String rotation, String surfaceType) {
+ mGlConfig = glConfig;
+ mRotation = rotation;
+ mSurfaceType = surfaceType;
+ }
+
+ /**
+ * Get string that uniquely identifies this config
+ */
+ public String getId() {
+ return String.format("{glformat=%s,rotation=%s,surfacetype=%s}",
+ mGlConfig, mRotation, mSurfaceType);
+ }
+
+ /**
+ * Get the GL config used in this configuration.
+ */
+ public String getGlConfig() {
+ return mGlConfig;
+ }
+
+ /**
+ * Get the screen rotation used in this configuration.
+ */
+ public String getRotation() {
+ return mRotation;
+ }
+
+ /**
+ * Get the surface type used in this configuration.
+ */
+ public String getSurfaceType() {
+ return mSurfaceType;
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == null) {
+ return false;
+ } else if (!(other instanceof BatchRunConfiguration)) {
+ return false;
+ } else {
+ return getId().equals(((BatchRunConfiguration)other).getId());
+ }
+ }
+
+ @Override
+ public int hashCode() {
+ return getId().hashCode();
+ }
+ }
+
+ /**
+ * dEQP test instance listerer and invocation result forwarded
+ */
+ private class TestInstanceResultListener {
+ private ITestInvocationListener mSink;
+ private BatchRunConfiguration mRunConfig;
+
+ private TestIdentifier mCurrentTestId;
+ private boolean mGotTestResult;
+ private String mCurrentTestLog;
+
+ private class PendingResult {
+ boolean allInstancesPassed;
+ Map<BatchRunConfiguration, String> testLogs;
+ Map<BatchRunConfiguration, String> errorMessages;
+ Set<BatchRunConfiguration> remainingConfigs;
+ }
+
+ private final Map<TestIdentifier, PendingResult> mPendingResults = new HashMap<>();
+
+ public void setSink(ITestInvocationListener sink) {
+ mSink = sink;
+ }
+
+ public void setCurrentConfig(BatchRunConfiguration runConfig) {
+ mRunConfig = runConfig;
+ }
+
+ /**
+ * Get currently processed test id, or null if not currently processing a test case
+ */
+ public TestIdentifier getCurrentTestId() {
+ return mCurrentTestId;
+ }
+
+ /**
+ * Forward result to sink
+ */
+ private void forwardFinalizedPendingResult(TestIdentifier testId) {
+ if (mRemainingTests.contains(testId)) {
+ final PendingResult result = mPendingResults.get(testId);
+
+ mPendingResults.remove(testId);
+ mRemainingTests.remove(testId);
+
+ // Forward results to the sink
+ mSink.testStarted(testId);
+
+ // Test Log
+ if (mLogData) {
+ for (Map.Entry<BatchRunConfiguration, String> entry :
+ result.testLogs.entrySet()) {
+ final ByteArrayInputStreamSource source
+ = new ByteArrayInputStreamSource(entry.getValue().getBytes());
+
+ mSink.testLog(testId.getClassName() + "." + testId.getTestName() + "@"
+ + entry.getKey().getId(), LogDataType.XML, source);
+
+ source.cancel();
+ }
+ }
+
+ // Error message
+ if (!result.allInstancesPassed) {
+ final StringBuilder errorLog = new StringBuilder();
+
+ for (Map.Entry<BatchRunConfiguration, String> entry :
+ result.errorMessages.entrySet()) {
+ if (errorLog.length() > 0) {
+ errorLog.append('\n');
+ }
+ errorLog.append(String.format("=== with config %s ===\n",
+ entry.getKey().getId()));
+ errorLog.append(entry.getValue());
+ }
+
+ mSink.testFailed(testId, errorLog.toString());
+ }
+
+ final Map<String, String> emptyMap = Collections.emptyMap();
+ mSink.testEnded(testId, emptyMap);
+ }
+ }
+
+ /**
+ * Declare existence of a test and instances
+ */
+ public void setTestInstances(TestIdentifier testId, Set<BatchRunConfiguration> configs) {
+ // Test instances cannot change at runtime, ignore if we have already set this
+ if (!mPendingResults.containsKey(testId)) {
+ final PendingResult pendingResult = new PendingResult();
+ pendingResult.allInstancesPassed = true;
+ pendingResult.testLogs = new LinkedHashMap<>();
+ pendingResult.errorMessages = new LinkedHashMap<>();
+ pendingResult.remainingConfigs = new HashSet<>(configs); // avoid mutating argument
+ mPendingResults.put(testId, pendingResult);
+ }
+ }
+
+ /**
+ * Query if test instance has not yet been executed
+ */
+ public boolean isPendingTestInstance(TestIdentifier testId,
+ BatchRunConfiguration config) {
+ final PendingResult result = mPendingResults.get(testId);
+ if (result == null) {
+ // test is not in the current working batch of the runner, i.e. it cannot be
+ // "partially" completed.
+ if (!mRemainingTests.contains(testId)) {
+ // The test has been fully executed. Not pending.
+ return false;
+ } else {
+ // Test has not yet been executed. Check if such instance exists
+ return mTestInstances.get(testId).contains(config);
+ }
+ } else {
+ // could be partially completed, check this particular config
+ return result.remainingConfigs.contains(config);
+ }
+ }
+
+ /**
+ * Fake execution of an instance with current config
+ */
+ public void skipTest(TestIdentifier testId) {
+ final PendingResult result = mPendingResults.get(testId);
+
+ result.errorMessages.put(mRunConfig, SKIPPED_INSTANCE_LOG_MESSAGE);
+ result.remainingConfigs.remove(mRunConfig);
+
+ // Pending result finished, report result
+ if (result.remainingConfigs.isEmpty()) {
+ forwardFinalizedPendingResult(testId);
+ }
+ }
+
+ /**
+ * Fake failure of an instance with current config
+ */
+ public void abortTest(TestIdentifier testId, String errorMessage) {
+ final PendingResult result = mPendingResults.get(testId);
+
+ // Mark as executed
+ result.allInstancesPassed = false;
+ result.errorMessages.put(mRunConfig, errorMessage);
+ result.remainingConfigs.remove(mRunConfig);
+
+ // Pending result finished, report result
+ if (result.remainingConfigs.isEmpty()) {
+ forwardFinalizedPendingResult(testId);
+ }
+
+ if (testId.equals(mCurrentTestId)) {
+ mCurrentTestId = null;
+ }
+ }
+
+ /**
+ * Handles beginning of dEQP session.
+ */
+ private void handleBeginSession(Map<String, String> values) {
+ // ignore
+ }
+
+ /**
+ * Handles end of dEQP session.
+ */
+ private void handleEndSession(Map<String, String> values) {
+ // ignore
+ }
+
+ /**
+ * Handles beginning of dEQP testcase.
+ */
+ private void handleBeginTestCase(Map<String, String> values) {
+ mCurrentTestId = pathToIdentifier(values.get("dEQP-BeginTestCase-TestCasePath"));
+ mCurrentTestLog = "";
+ mGotTestResult = false;
+
+ // mark instance as started
+ if (mPendingResults.get(mCurrentTestId) != null) {
+ mPendingResults.get(mCurrentTestId).remainingConfigs.remove(mRunConfig);
+ } else {
+ CLog.w("Got unexpected start of %s", mCurrentTestId);
+ }
+ }
+
+ /**
+ * Handles end of dEQP testcase.
+ */
+ private void handleEndTestCase(Map<String, String> values) {
+ final PendingResult result = mPendingResults.get(mCurrentTestId);
+
+ if (result != null) {
+ if (!mGotTestResult) {
+ result.allInstancesPassed = false;
+ result.errorMessages.put(mRunConfig, INCOMPLETE_LOG_MESSAGE);
+ }
+
+ if (mLogData && mCurrentTestLog != null && mCurrentTestLog.length() > 0) {
+ result.testLogs.put(mRunConfig, mCurrentTestLog);
+ }
+
+ // Pending result finished, report result
+ if (result.remainingConfigs.isEmpty()) {
+ forwardFinalizedPendingResult(mCurrentTestId);
+ }
+ } else {
+ CLog.w("Got unexpected end of %s", mCurrentTestId);
+ }
+ mCurrentTestId = null;
+ }
+
+ /**
+ * Handles dEQP testcase result.
+ */
+ private void handleTestCaseResult(Map<String, String> values) {
+ String code = values.get("dEQP-TestCaseResult-Code");
+ String details = values.get("dEQP-TestCaseResult-Details");
+
+ if (mPendingResults.get(mCurrentTestId) == null) {
+ CLog.w("Got unexpected result for %s", mCurrentTestId);
+ mGotTestResult = true;
+ return;
+ }
+
+ if (code.compareTo("Pass") == 0) {
+ mGotTestResult = true;
+ } else if (code.compareTo("NotSupported") == 0) {
+ mGotTestResult = true;
+ } else if (code.compareTo("QualityWarning") == 0) {
+ mGotTestResult = true;
+ } else if (code.compareTo("CompatibilityWarning") == 0) {
+ mGotTestResult = true;
+ } else if (code.compareTo("Fail") == 0 || code.compareTo("ResourceError") == 0
+ || code.compareTo("InternalError") == 0 || code.compareTo("Crash") == 0
+ || code.compareTo("Timeout") == 0) {
+ mPendingResults.get(mCurrentTestId).allInstancesPassed = false;
+ mPendingResults.get(mCurrentTestId)
+ .errorMessages.put(mRunConfig, code + ": " + details);
+ mGotTestResult = true;
+ } else {
+ String codeError = "Unknown result code: " + code;
+ mPendingResults.get(mCurrentTestId).allInstancesPassed = false;
+ mPendingResults.get(mCurrentTestId)
+ .errorMessages.put(mRunConfig, codeError + ": " + details);
+ mGotTestResult = true;
+ }
+ }
+
+ /**
+ * Handles terminated dEQP testcase.
+ */
+ private void handleTestCaseTerminate(Map<String, String> values) {
+ final PendingResult result = mPendingResults.get(mCurrentTestId);
+
+ if (result != null) {
+ String reason = values.get("dEQP-TerminateTestCase-Reason");
+ mPendingResults.get(mCurrentTestId).allInstancesPassed = false;
+ mPendingResults.get(mCurrentTestId)
+ .errorMessages.put(mRunConfig, "Terminated: " + reason);
+
+ // Pending result finished, report result
+ if (result.remainingConfigs.isEmpty()) {
+ forwardFinalizedPendingResult(mCurrentTestId);
+ }
+ } else {
+ CLog.w("Got unexpected termination of %s", mCurrentTestId);
+ }
+
+ mCurrentTestId = null;
+ mGotTestResult = true;
+ }
+
+ /**
+ * Handles dEQP testlog data.
+ */
+ private void handleTestLogData(Map<String, String> values) {
+ mCurrentTestLog = mCurrentTestLog + values.get("dEQP-TestLogData-Log");
+ }
+
+ /**
+ * Handles new instrumentation status message.
+ */
+ public void handleStatus(Map<String, String> values) {
+ String eventType = values.get("dEQP-EventType");
+
+ if (eventType == null) {
+ return;
+ }
+
+ if (eventType.compareTo("BeginSession") == 0) {
+ handleBeginSession(values);
+ } else if (eventType.compareTo("EndSession") == 0) {
+ handleEndSession(values);
+ } else if (eventType.compareTo("BeginTestCase") == 0) {
+ handleBeginTestCase(values);
+ } else if (eventType.compareTo("EndTestCase") == 0) {
+ handleEndTestCase(values);
+ } else if (eventType.compareTo("TestCaseResult") == 0) {
+ handleTestCaseResult(values);
+ } else if (eventType.compareTo("TerminateTestCase") == 0) {
+ handleTestCaseTerminate(values);
+ } else if (eventType.compareTo("TestLogData") == 0) {
+ handleTestLogData(values);
+ }
+ }
+
+ /**
+ * Signal listener that batch ended and forget incomplete results.
+ */
+ public void endBatch() {
+ // end open test if when stream ends
+ if (mCurrentTestId != null) {
+ // Current instance was removed from remainingConfigs when case
+ // started. Mark current instance as pending.
+ if (mPendingResults.get(mCurrentTestId) != null) {
+ mPendingResults.get(mCurrentTestId).remainingConfigs.add(mRunConfig);
+ } else {
+ CLog.w("Got unexpected internal state of %s", mCurrentTestId);
+ }
+ }
+ mCurrentTestId = null;
+ }
+ }
+
+ /**
+ * dEQP instrumentation parser
+ */
+ private static class InstrumentationParser extends MultiLineReceiver {
+ private TestInstanceResultListener mListener;
+
+ private Map<String, String> mValues;
+ private String mCurrentName;
+ private String mCurrentValue;
+ private int mResultCode;
+ private boolean mGotExitValue = false;
+
+
+ public InstrumentationParser(TestInstanceResultListener listener) {
+ mListener = listener;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void processNewLines(String[] lines) {
+ for (String line : lines) {
+ if (mValues == null) mValues = new HashMap<String, String>();
+
+ if (line.startsWith("INSTRUMENTATION_STATUS_CODE: ")) {
+ if (mCurrentName != null) {
+ mValues.put(mCurrentName, mCurrentValue);
+
+ mCurrentName = null;
+ mCurrentValue = null;
+ }
+
+ mListener.handleStatus(mValues);
+ mValues = null;
+ } else if (line.startsWith("INSTRUMENTATION_STATUS: dEQP-")) {
+ if (mCurrentName != null) {
+ mValues.put(mCurrentName, mCurrentValue);
+
+ mCurrentValue = null;
+ mCurrentName = null;
+ }
+
+ String prefix = "INSTRUMENTATION_STATUS: ";
+ int nameBegin = prefix.length();
+ int nameEnd = line.indexOf('=');
+ int valueBegin = nameEnd + 1;
+
+ mCurrentName = line.substring(nameBegin, nameEnd);
+ mCurrentValue = line.substring(valueBegin);
+ } else if (line.startsWith("INSTRUMENTATION_CODE: ")) {
+ try {
+ mResultCode = Integer.parseInt(line.substring(22));
+ mGotExitValue = true;
+ } catch (NumberFormatException ex) {
+ CLog.w("Instrumentation code format unexpected");
+ }
+ } else if (mCurrentValue != null) {
+ mCurrentValue = mCurrentValue + line;
+ }
+ }
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void done() {
+ if (mCurrentName != null) {
+ mValues.put(mCurrentName, mCurrentValue);
+
+ mCurrentName = null;
+ mCurrentValue = null;
+ }
+
+ if (mValues != null) {
+ mListener.handleStatus(mValues);
+ mValues = null;
+ }
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public boolean isCancelled() {
+ return false;
+ }
+
+ /**
+ * Returns whether target instrumentation exited normally.
+ */
+ public boolean wasSuccessful() {
+ return mGotExitValue;
+ }
+
+ /**
+ * Returns Instrumentation return code
+ */
+ public int getResultCode() {
+ return mResultCode;
+ }
+ }
+
+ /**
+ * dEQP platfom query instrumentation parser
+ */
+ private static class PlatformQueryInstrumentationParser extends MultiLineReceiver {
+ private Map<String,String> mResultMap = new LinkedHashMap<>();
+ private int mResultCode;
+ private boolean mGotExitValue = false;
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void processNewLines(String[] lines) {
+ for (String line : lines) {
+ if (line.startsWith("INSTRUMENTATION_RESULT: ")) {
+ final String parts[] = line.substring(24).split("=",2);
+ if (parts.length == 2) {
+ mResultMap.put(parts[0], parts[1]);
+ } else {
+ CLog.w("Instrumentation status format unexpected");
+ }
+ } else if (line.startsWith("INSTRUMENTATION_CODE: ")) {
+ try {
+ mResultCode = Integer.parseInt(line.substring(22));
+ mGotExitValue = true;
+ } catch (NumberFormatException ex) {
+ CLog.w("Instrumentation code format unexpected");
+ }
+ }
+ }
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public boolean isCancelled() {
+ return false;
+ }
+
+ /**
+ * Returns whether target instrumentation exited normally.
+ */
+ public boolean wasSuccessful() {
+ return mGotExitValue;
+ }
+
+ /**
+ * Returns Instrumentation return code
+ */
+ public int getResultCode() {
+ return mResultCode;
+ }
+
+ public Map<String,String> getResultMap() {
+ return mResultMap;
+ }
+ }
+
+ /**
+ * Interface for sleeping.
+ *
+ * Exposed for unit testing
+ */
+ public static interface ISleepProvider {
+ public void sleep(int milliseconds);
+ }
+
+ private static class SleepProvider implements ISleepProvider {
+ public void sleep(int milliseconds) {
+ try {
+ Thread.sleep(milliseconds);
+ } catch (InterruptedException ex) {
+ }
+ }
+ }
+
+ /**
+ * Interface for failure recovery.
+ *
+ * Exposed for unit testing
+ */
+ public static interface IRecovery {
+ /**
+ * Sets the sleep provider IRecovery works on
+ */
+ public void setSleepProvider(ISleepProvider sleepProvider);
+
+ /**
+ * Sets the device IRecovery works on
+ */
+ public void setDevice(ITestDevice device);
+
+ /**
+ * Informs Recovery that test execution has progressed since the last recovery
+ */
+ public void onExecutionProgressed();
+
+ /**
+ * Tries to recover device after failed refused connection.
+ *
+ * @throws DeviceNotAvailableException if recovery did not succeed
+ */
+ public void recoverConnectionRefused() throws DeviceNotAvailableException;
+
+ /**
+ * Tries to recover device after abnormal execution termination or link failure.
+ *
+ * @param progressedSinceLastCall true if test execution has progressed since last call
+ * @throws DeviceNotAvailableException if recovery did not succeed
+ */
+ public void recoverComLinkKilled() throws DeviceNotAvailableException;
+ };
+
+ /**
+ * State machine for execution failure recovery.
+ *
+ * Exposed for unit testing
+ */
+ public static class Recovery implements IRecovery {
+ private int RETRY_COOLDOWN_MS = 6000; // 6 seconds
+ private int PROCESS_KILL_WAIT_MS = 1000; // 1 second
+
+ private static enum MachineState {
+ WAIT, // recover by waiting
+ RECOVER, // recover by calling recover()
+ REBOOT, // recover by rebooting
+ FAIL, // cannot recover
+ };
+
+ private MachineState mState = MachineState.WAIT;
+ private ITestDevice mDevice;
+ private ISleepProvider mSleepProvider;
+
+ private static class ProcessKillFailureException extends Exception {
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ public void setSleepProvider(ISleepProvider sleepProvider) {
+ mSleepProvider = sleepProvider;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void setDevice(ITestDevice device) {
+ mDevice = device;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void onExecutionProgressed() {
+ mState = MachineState.WAIT;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void recoverConnectionRefused() throws DeviceNotAvailableException {
+ switch (mState) {
+ case WAIT: // not a valid stratedy for connection refusal, fallthrough
+ case RECOVER:
+ // First failure, just try to recover
+ CLog.w("ADB connection failed, trying to recover");
+ mState = MachineState.REBOOT; // the next step is to reboot
+
+ try {
+ recoverDevice();
+ } catch (DeviceNotAvailableException ex) {
+ // chain forward
+ recoverConnectionRefused();
+ }
+ break;
+
+ case REBOOT:
+ // Second failure in a row, try to reboot
+ CLog.w("ADB connection failed after recovery, rebooting device");
+ mState = MachineState.FAIL; // the next step is to fail
+
+ try {
+ rebootDevice();
+ } catch (DeviceNotAvailableException ex) {
+ // chain forward
+ recoverConnectionRefused();
+ }
+ break;
+
+ case FAIL:
+ // Third failure in a row, just fail
+ CLog.w("Cannot recover ADB connection");
+ throw new DeviceNotAvailableException("failed to connect after reboot");
+ }
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void recoverComLinkKilled() throws DeviceNotAvailableException {
+ switch (mState) {
+ case WAIT:
+ // First failure, just try to wait and try again
+ CLog.w("ADB link failed, retrying after a cooldown period");
+ mState = MachineState.RECOVER; // the next step is to recover the device
+
+ waitCooldown();
+
+ // even if the link to deqp on-device process was killed, the process might
+ // still be alive. Locate and terminate such unwanted processes.
+ try {
+ killDeqpProcess();
+ } catch (DeviceNotAvailableException ex) {
+ // chain forward
+ recoverComLinkKilled();
+ } catch (ProcessKillFailureException ex) {
+ // chain forward
+ recoverComLinkKilled();
+ }
+ break;
+
+ case RECOVER:
+ // Second failure, just try to recover
+ CLog.w("ADB link failed, trying to recover");
+ mState = MachineState.REBOOT; // the next step is to reboot
+
+ try {
+ recoverDevice();
+ killDeqpProcess();
+ } catch (DeviceNotAvailableException ex) {
+ // chain forward
+ recoverComLinkKilled();
+ } catch (ProcessKillFailureException ex) {
+ // chain forward
+ recoverComLinkKilled();
+ }
+ break;
+
+ case REBOOT:
+ // Third failure in a row, try to reboot
+ CLog.w("ADB link failed after recovery, rebooting device");
+ mState = MachineState.FAIL; // the next step is to fail
+
+ try {
+ rebootDevice();
+ } catch (DeviceNotAvailableException ex) {
+ // chain forward
+ recoverComLinkKilled();
+ }
+ break;
+
+ case FAIL:
+ // Fourth failure in a row, just fail
+ CLog.w("Cannot recover ADB connection");
+ throw new DeviceNotAvailableException("link killed after reboot");
+ }
+ }
+
+ private void waitCooldown() {
+ mSleepProvider.sleep(RETRY_COOLDOWN_MS);
+ }
+
+ private Iterable<Integer> getDeqpProcessPids() throws DeviceNotAvailableException {
+ final List<Integer> pids = new ArrayList<Integer>(2);
+ final String processes = mDevice.executeShellCommand("ps | grep com.drawelements");
+ final String[] lines = processes.split("(\\r|\\n)+");
+ for (String line : lines) {
+ final String[] fields = line.split("\\s+");
+ if (fields.length < 2) {
+ continue;
+ }
+
+ try {
+ final int processId = Integer.parseInt(fields[1], 10);
+ pids.add(processId);
+ } catch (NumberFormatException ex) {
+ continue;
+ }
+ }
+ return pids;
+ }
+
+ private void killDeqpProcess() throws DeviceNotAvailableException,
+ ProcessKillFailureException {
+ for (Integer processId : getDeqpProcessPids()) {
+ mDevice.executeShellCommand(String.format("kill -9 %d", processId));
+ }
+
+ mSleepProvider.sleep(PROCESS_KILL_WAIT_MS);
+
+ // check that processes actually died
+ if (getDeqpProcessPids().iterator().hasNext()) {
+ // a process is still alive, killing failed
+ throw new ProcessKillFailureException();
+ }
+ }
+
+ public void recoverDevice() throws DeviceNotAvailableException {
+ // Work around the API. We need to call recoverDevice() on the test device and
+ // we know that mDevice is a TestDevice. However even though the recoverDevice()
+ // method is public suggesting it should be publicly accessible, the class itself
+ // and its super-interface (IManagedTestDevice) are package-private.
+ final Method recoverDeviceMethod;
+ try {
+ recoverDeviceMethod = mDevice.getClass().getMethod("recoverDevice");
+ recoverDeviceMethod.setAccessible(true);
+ } catch (NoSuchMethodException ex) {
+ throw new AssertionError("Test device must have recoverDevice()");
+ }
+
+ try {
+ recoverDeviceMethod.invoke(mDevice);
+ } catch (InvocationTargetException ex) {
+ if (ex.getCause() instanceof DeviceNotAvailableException) {
+ throw (DeviceNotAvailableException)ex.getCause();
+ } else if (ex.getCause() instanceof RuntimeException) {
+ throw (RuntimeException)ex.getCause();
+ } else {
+ throw new AssertionError("unexpected throw", ex);
+ }
+ } catch (IllegalAccessException ex) {
+ throw new AssertionError("unexpected throw", ex);
+ }
+ }
+
+ private void rebootDevice() throws DeviceNotAvailableException {
+ mDevice.reboot();
+ }
+ }
+
+ private static Map<TestIdentifier, Set<BatchRunConfiguration>> generateTestInstances(
+ Reader testlist, String configName, String screenRotation, String surfaceType) throws FileNotFoundException {
+ // Note: This is specifically a LinkedHashMap to guarantee that tests are iterated
+ // in the insertion order.
+ final Map<TestIdentifier, Set<BatchRunConfiguration>> instances = new LinkedHashMap<>();
+ try {
+ BufferedReader testlistReader = new BufferedReader(testlist);
+ String testName;
+ while ((testName = testlistReader.readLine()) != null) {
+ // Test name -> testId -> only one config -> done.
+ final Set<BatchRunConfiguration> testInstanceSet = new LinkedHashSet<>();
+ BatchRunConfiguration config = new BatchRunConfiguration(configName, screenRotation, surfaceType);
+ testInstanceSet.add(config);
+ TestIdentifier test = pathToIdentifier(testName);
+ instances.put(test, testInstanceSet);
+ }
+ testlistReader.close();
+ }
+ catch (IOException e)
+ {
+ throw new RuntimeException("Failure while reading the test case list for deqp: " + e.getMessage());
+ }
+
+ return instances;
+ }
+
+ private Set<BatchRunConfiguration> getTestRunConfigs (TestIdentifier testId) {
+ return mTestInstances.get(testId);
+ }
+
+ /**
+ * Converts dEQP testcase path to TestIdentifier.
+ */
+ private static TestIdentifier pathToIdentifier(String testPath) {
+ int indexOfLastDot = testPath.lastIndexOf('.');
+ String className = testPath.substring(0, indexOfLastDot);
+ String testName = testPath.substring(indexOfLastDot+1);
+
+ return new TestIdentifier(className, testName);
+ }
+
+ // \todo [2015-10-16 kalle] How unique should this be?
+ private String getId() {
+ return AbiUtils.createId(mAbi.getName(), mDeqpPackage);
+ }
+
+ /**
+ * Generates tescase trie from dEQP testcase paths. Used to define which testcases to execute.
+ */
+ private static String generateTestCaseTrieFromPaths(Collection<String> tests) {
+ String result = "{";
+ boolean first = true;
+
+ // Add testcases to results
+ for (Iterator<String> iter = tests.iterator(); iter.hasNext();) {
+ String test = iter.next();
+ String[] components = test.split("\\.");
+
+ if (components.length == 1) {
+ if (!first) {
+ result = result + ",";
+ }
+ first = false;
+
+ result += components[0];
+ iter.remove();
+ }
+ }
+
+ if (!tests.isEmpty()) {
+ HashMap<String, ArrayList<String> > testGroups = new HashMap<>();
+
+ // Collect all sub testgroups
+ for (String test : tests) {
+ String[] components = test.split("\\.");
+ ArrayList<String> testGroup = testGroups.get(components[0]);
+
+ if (testGroup == null) {
+ testGroup = new ArrayList<String>();
+ testGroups.put(components[0], testGroup);
+ }
+
+ testGroup.add(test.substring(components[0].length()+1));
+ }
+
+ for (String testGroup : testGroups.keySet()) {
+ if (!first) {
+ result = result + ",";
+ }
+
+ first = false;
+ result = result + testGroup
+ + generateTestCaseTrieFromPaths(testGroups.get(testGroup));
+ }
+ }
+
+ return result + "}";
+ }
+
+ /**
+ * Generates testcase trie from TestIdentifiers.
+ */
+ private static String generateTestCaseTrie(Collection<TestIdentifier> tests) {
+ ArrayList<String> testPaths = new ArrayList<String>();
+
+ for (TestIdentifier test : tests) {
+ testPaths.add(test.getClassName() + "." + test.getTestName());
+ }
+
+ return generateTestCaseTrieFromPaths(testPaths);
+ }
+
+ private static class TestBatch {
+ public BatchRunConfiguration config;
+ public List<TestIdentifier> tests;
+ }
+
+ /**
+ * Creates a TestBatch from the given tests or null if not tests remaining.
+ *
+ * @param pool List of tests to select from
+ * @param requiredConfig Select only instances with pending requiredConfig, or null to select
+ * any run configuration.
+ */
+ private TestBatch selectRunBatch(Collection<TestIdentifier> pool,
+ BatchRunConfiguration requiredConfig) {
+ // select one test (leading test) that is going to be executed and then pack along as many
+ // other compatible instances as possible.
+
+ TestIdentifier leadingTest = null;
+ for (TestIdentifier test : pool) {
+ if (!mRemainingTests.contains(test)) {
+ continue;
+ }
+ if (requiredConfig != null &&
+ !mInstanceListerner.isPendingTestInstance(test, requiredConfig)) {
+ continue;
+ }
+ leadingTest = test;
+ break;
+ }
+
+ // no remaining tests?
+ if (leadingTest == null) {
+ return null;
+ }
+
+ BatchRunConfiguration leadingTestConfig = null;
+ if (requiredConfig != null) {
+ leadingTestConfig = requiredConfig;
+ } else {
+ for (BatchRunConfiguration runConfig : getTestRunConfigs(leadingTest)) {
+ if (mInstanceListerner.isPendingTestInstance(leadingTest, runConfig)) {
+ leadingTestConfig = runConfig;
+ break;
+ }
+ }
+ }
+
+ // test pending <=> test has a pending config
+ if (leadingTestConfig == null) {
+ throw new AssertionError("search postcondition failed");
+ }
+
+ final int leadingInstability = getTestInstabilityRating(leadingTest);
+
+ final TestBatch runBatch = new TestBatch();
+ runBatch.config = leadingTestConfig;
+ runBatch.tests = new ArrayList<>();
+ runBatch.tests.add(leadingTest);
+
+ for (TestIdentifier test : pool) {
+ if (test == leadingTest) {
+ // do not re-select the leading tests
+ continue;
+ }
+ if (!mInstanceListerner.isPendingTestInstance(test, leadingTestConfig)) {
+ // select only compatible
+ continue;
+ }
+ if (getTestInstabilityRating(test) != leadingInstability) {
+ // pack along only cases in the same stability category. Packing more dangerous
+ // tests along jeopardizes the stability of this run. Packing more stable tests
+ // along jeopardizes their stability rating.
+ continue;
+ }
+ if (runBatch.tests.size() >= getBatchSizeLimitForInstability(leadingInstability)) {
+ // batch size is limited.
+ break;
+ }
+ runBatch.tests.add(test);
+ }
+
+ return runBatch;
+ }
+
+ private int getBatchNumPendingCases(TestBatch batch) {
+ int numPending = 0;
+ for (TestIdentifier test : batch.tests) {
+ if (mInstanceListerner.isPendingTestInstance(test, batch.config)) {
+ ++numPending;
+ }
+ }
+ return numPending;
+ }
+
+ private int getBatchSizeLimitForInstability(int batchInstabilityRating) {
+ // reduce group size exponentially down to one
+ return Math.max(1, TESTCASE_BATCH_LIMIT / (1 << batchInstabilityRating));
+ }
+
+ private int getTestInstabilityRating(TestIdentifier testId) {
+ if (mTestInstabilityRatings.containsKey(testId)) {
+ return mTestInstabilityRatings.get(testId);
+ } else {
+ return 0;
+ }
+ }
+
+ private void recordTestInstability(TestIdentifier testId) {
+ mTestInstabilityRatings.put(testId, getTestInstabilityRating(testId) + 1);
+ }
+
+ private void clearTestInstability(TestIdentifier testId) {
+ mTestInstabilityRatings.put(testId, 0);
+ }
+
+ /**
+ * Executes all tests on the device.
+ */
+ private void runTests() throws DeviceNotAvailableException, CapabilityQueryFailureException {
+ for (;;) {
+ TestBatch batch = selectRunBatch(mRemainingTests, null);
+
+ if (batch == null) {
+ break;
+ }
+
+ runTestRunBatch(batch);
+ }
+ }
+
+ /**
+ * Runs a TestBatch by either faking it or executing it on a device.
+ */
+ private void runTestRunBatch(TestBatch batch) throws DeviceNotAvailableException,
+ CapabilityQueryFailureException {
+ // prepare instance listener
+ mInstanceListerner.setCurrentConfig(batch.config);
+ for (TestIdentifier test : batch.tests) {
+ mInstanceListerner.setTestInstances(test, getTestRunConfigs(test));
+ }
+
+ // execute only if config is executable, else fake results
+ if (isSupportedRunConfiguration(batch.config)) {
+ executeTestRunBatch(batch);
+ } else {
+ fakePassTestRunBatch(batch);
+ }
+ }
+
+ private boolean isSupportedRunConfiguration(BatchRunConfiguration runConfig)
+ throws DeviceNotAvailableException, CapabilityQueryFailureException {
+ // orientation support
+ if (!BatchRunConfiguration.ROTATION_UNSPECIFIED.equals(runConfig.getRotation())) {
+ final Set<String> features = getDeviceFeatures(mDevice);
+
+ if (isPortraitClassRotation(runConfig.getRotation()) &&
+ !features.contains(FEATURE_PORTRAIT)) {
+ return false;
+ }
+ if (isLandscapeClassRotation(runConfig.getRotation()) &&
+ !features.contains(FEATURE_LANDSCAPE)) {
+ return false;
+ }
+ }
+
+ if (isOpenGlEsPackage()) {
+ // renderability support for OpenGL ES tests
+ return isSupportedGlesRenderConfig(runConfig);
+ } else {
+ return true;
+ }
+ }
+
+ private static final class AdbComLinkOpenError extends Exception {
+ public AdbComLinkOpenError(String description, Throwable inner) {
+ super(description, inner);
+ }
+ }
+
+ private static final class AdbComLinkKilledError extends Exception {
+ public AdbComLinkKilledError(String description, Throwable inner) {
+ super(description, inner);
+ }
+ }
+
+ /**
+ * Executes a given command in adb shell
+ *
+ * @throws AdbComLinkOpenError if connection cannot be established.
+ * @throws AdbComLinkKilledError if established connection is killed prematurely.
+ */
+ private void executeShellCommandAndReadOutput(final String command,
+ final IShellOutputReceiver receiver)
+ throws AdbComLinkOpenError, AdbComLinkKilledError {
+ try {
+ mDevice.getIDevice().executeShellCommand(command, receiver,
+ UNRESPOSIVE_CMD_TIMEOUT_MS, TimeUnit.MILLISECONDS);
+ } catch (TimeoutException ex) {
+ // Opening connection timed out
+ throw new AdbComLinkOpenError("opening connection timed out", ex);
+ } catch (AdbCommandRejectedException ex) {
+ // Command rejected
+ throw new AdbComLinkOpenError("command rejected", ex);
+ } catch (IOException ex) {
+ // shell command channel killed
+ throw new AdbComLinkKilledError("command link killed", ex);
+ } catch (ShellCommandUnresponsiveException ex) {
+ // shell command halted
+ throw new AdbComLinkKilledError("command link hung", ex);
+ }
+ }
+
+ /**
+ * Executes given test batch on a device
+ */
+ private void executeTestRunBatch(TestBatch batch) throws DeviceNotAvailableException {
+ // attempt full run once
+ executeTestRunBatchRun(batch);
+
+ // split remaining tests to two sub batches and execute both. This will terminate
+ // since executeTestRunBatchRun will always progress for a batch of size 1.
+ final ArrayList<TestIdentifier> pendingTests = new ArrayList<>();
+
+ for (TestIdentifier test : batch.tests) {
+ if (mInstanceListerner.isPendingTestInstance(test, batch.config)) {
+ pendingTests.add(test);
+ }
+ }
+
+ final int divisorNdx = pendingTests.size() / 2;
+ final List<TestIdentifier> headList = pendingTests.subList(0, divisorNdx);
+ final List<TestIdentifier> tailList = pendingTests.subList(divisorNdx, pendingTests.size());
+
+ // head
+ for (;;) {
+ TestBatch subBatch = selectRunBatch(headList, batch.config);
+
+ if (subBatch == null) {
+ break;
+ }
+
+ executeTestRunBatch(subBatch);
+ }
+
+ // tail
+ for (;;) {
+ TestBatch subBatch = selectRunBatch(tailList, batch.config);
+
+ if (subBatch == null) {
+ break;
+ }
+
+ executeTestRunBatch(subBatch);
+ }
+
+ if (getBatchNumPendingCases(batch) != 0) {
+ throw new AssertionError("executeTestRunBatch postcondition failed");
+ }
+ }
+
+ /**
+ * Runs one execution pass over the given batch.
+ *
+ * Tries to run the batch. Always makes progress (executes instances or modifies stability
+ * scores).
+ */
+ private void executeTestRunBatchRun(TestBatch batch) throws DeviceNotAvailableException {
+ if (getBatchNumPendingCases(batch) != batch.tests.size()) {
+ throw new AssertionError("executeTestRunBatchRun precondition failed");
+ }
+
+ checkInterrupted(); // throws if interrupted
+
+ final String testCases = generateTestCaseTrie(batch.tests);
+
+ mDevice.executeShellCommand("rm " + CASE_LIST_FILE_NAME);
+ mDevice.executeShellCommand("rm " + LOG_FILE_NAME);
+ mDevice.pushString(testCases + "\n", CASE_LIST_FILE_NAME);
+
+ final String instrumentationName =
+ "com.drawelements.deqp/com.drawelements.deqp.testercore.DeqpInstrumentation";
+
+ final StringBuilder deqpCmdLine = new StringBuilder();
+ deqpCmdLine.append("--deqp-caselist-file=");
+ deqpCmdLine.append(CASE_LIST_FILE_NAME);
+ deqpCmdLine.append(" ");
+ deqpCmdLine.append(getRunConfigDisplayCmdLine(batch.config));
+
+ // If we are not logging data, do not bother outputting the images from the test exe.
+ if (!mLogData) {
+ deqpCmdLine.append(" --deqp-log-images=disable");
+ }
+
+ deqpCmdLine.append(" --deqp-watchdog=enable");
+
+ final String command = String.format(
+ "am instrument %s -w -e deqpLogFileName \"%s\" -e deqpCmdLine \"%s\""
+ + " -e deqpLogData \"%s\" %s",
+ AbiUtils.createAbiFlag(mAbi.getName()), LOG_FILE_NAME, deqpCmdLine.toString(),
+ mLogData, instrumentationName);
+
+ final int numRemainingInstancesBefore = getNumRemainingInstances();
+ final InstrumentationParser parser = new InstrumentationParser(mInstanceListerner);
+ Throwable interruptingError = null;
+
+ try {
+ executeShellCommandAndReadOutput(command, parser);
+ } catch (Throwable ex) {
+ interruptingError = ex;
+ } finally {
+ parser.flush();
+ }
+
+ final boolean progressedSinceLastCall = mInstanceListerner.getCurrentTestId() != null ||
+ getNumRemainingInstances() < numRemainingInstancesBefore;
+
+ if (progressedSinceLastCall) {
+ mDeviceRecovery.onExecutionProgressed();
+ }
+
+ // interrupted, try to recover
+ if (interruptingError != null) {
+ if (interruptingError instanceof AdbComLinkOpenError) {
+ mDeviceRecovery.recoverConnectionRefused();
+ } else if (interruptingError instanceof AdbComLinkKilledError) {
+ mDeviceRecovery.recoverComLinkKilled();
+ } else if (interruptingError instanceof RunInterruptedException) {
+ // external run interruption request. Terminate immediately.
+ throw (RunInterruptedException)interruptingError;
+ } else {
+ CLog.e(interruptingError);
+ throw new RuntimeException(interruptingError);
+ }
+
+ // recoverXXX did not throw => recovery succeeded
+ } else if (!parser.wasSuccessful()) {
+ mDeviceRecovery.recoverComLinkKilled();
+ // recoverXXX did not throw => recovery succeeded
+ }
+
+ // Progress guarantees.
+ if (batch.tests.size() == 1) {
+ final TestIdentifier onlyTest = batch.tests.iterator().next();
+ final boolean wasTestExecuted =
+ !mInstanceListerner.isPendingTestInstance(onlyTest, batch.config) &&
+ mInstanceListerner.getCurrentTestId() == null;
+ final boolean wasLinkFailure = !parser.wasSuccessful() || interruptingError != null;
+
+ // Link failures can be caused by external events, require at least two observations
+ // until bailing.
+ if (!wasTestExecuted && (!wasLinkFailure || getTestInstabilityRating(onlyTest) > 0)) {
+ recordTestInstability(onlyTest);
+ // If we cannot finish the test, mark the case as a crash.
+ //
+ // If we couldn't even start the test, fail the test instance as non-executable.
+ // This is required so that a consistently crashing or non-existent tests will
+ // not cause futile (non-terminating) re-execution attempts.
+ if (mInstanceListerner.getCurrentTestId() != null) {
+ mInstanceListerner.abortTest(onlyTest, INCOMPLETE_LOG_MESSAGE);
+ } else {
+ mInstanceListerner.abortTest(onlyTest, NOT_EXECUTABLE_LOG_MESSAGE);
+ }
+ } else if (wasTestExecuted) {
+ clearTestInstability(onlyTest);
+ }
+ }
+ else
+ {
+ // Analyze results to update test stability ratings. If there is no interrupting test
+ // logged, increase instability rating of all remaining tests. If there is a
+ // interrupting test logged, increase only its instability rating.
+ //
+ // A successful run of tests clears instability rating.
+ if (mInstanceListerner.getCurrentTestId() == null) {
+ for (TestIdentifier test : batch.tests) {
+ if (mInstanceListerner.isPendingTestInstance(test, batch.config)) {
+ recordTestInstability(test);
+ } else {
+ clearTestInstability(test);
+ }
+ }
+ } else {
+ recordTestInstability(mInstanceListerner.getCurrentTestId());
+ for (TestIdentifier test : batch.tests) {
+ // \note: isPendingTestInstance is false for getCurrentTestId. Current ID is
+ // considered 'running' and will be restored to 'pending' in endBatch().
+ if (!test.equals(mInstanceListerner.getCurrentTestId()) &&
+ !mInstanceListerner.isPendingTestInstance(test, batch.config)) {
+ clearTestInstability(test);
+ }
+ }
+ }
+ }
+
+ mInstanceListerner.endBatch();
+ }
+
+ private static String getRunConfigDisplayCmdLine(BatchRunConfiguration runConfig) {
+ final StringBuilder deqpCmdLine = new StringBuilder();
+ if (!runConfig.getGlConfig().isEmpty()) {
+ deqpCmdLine.append("--deqp-gl-config-name=");
+ deqpCmdLine.append(runConfig.getGlConfig());
+ }
+ if (!runConfig.getRotation().isEmpty()) {
+ if (deqpCmdLine.length() != 0) {
+ deqpCmdLine.append(" ");
+ }
+ deqpCmdLine.append("--deqp-screen-rotation=");
+ deqpCmdLine.append(runConfig.getRotation());
+ }
+ if (!runConfig.getSurfaceType().isEmpty()) {
+ if (deqpCmdLine.length() != 0) {
+ deqpCmdLine.append(" ");
+ }
+ deqpCmdLine.append("--deqp-surface-type=");
+ deqpCmdLine.append(runConfig.getSurfaceType());
+ }
+ return deqpCmdLine.toString();
+ }
+
+ private int getNumRemainingInstances() {
+ int retVal = 0;
+ for (TestIdentifier testId : mRemainingTests) {
+ // If case is in current working set, sum only not yet executed instances.
+ // If case is not in current working set, sum all instances (since they are not yet
+ // executed).
+ if (mInstanceListerner.mPendingResults.containsKey(testId)) {
+ retVal += mInstanceListerner.mPendingResults.get(testId).remainingConfigs.size();
+ } else {
+ retVal += mTestInstances.get(testId).size();
+ }
+ }
+ return retVal;
+ }
+
+ /**
+ * Checks if this execution has been marked as interrupted and throws if it has.
+ */
+ private void checkInterrupted() throws RunInterruptedException {
+ // Work around the API. RunUtil::checkInterrupted is private but we can call it indirectly
+ // by sleeping a value <= 0.
+ mRunUtil.sleep(0);
+ }
+
+ /**
+ * Pass given batch tests without running it
+ */
+ private void fakePassTestRunBatch(TestBatch batch) {
+ for (TestIdentifier test : batch.tests) {
+ CLog.d("Skipping test '%s' invocation in config '%s'", test.toString(),
+ batch.config.getId());
+ mInstanceListerner.skipTest(test);
+ }
+ }
+
+ /**
+ * Pass all remaining tests without running them
+ */
+ private void fakePassTests(ITestInvocationListener listener) {
+ Map <String, String> emptyMap = Collections.emptyMap();
+ for (TestIdentifier test : mRemainingTests) {
+ CLog.d("Skipping test '%s', Opengl ES version not supported", test.toString());
+ listener.testStarted(test);
+ listener.testEnded(test, emptyMap);
+ }
+ mRemainingTests.clear();
+ }
+
+ /**
+ * Check if device supports OpenGL ES version.
+ */
+ private static boolean isSupportedGles(ITestDevice device, int requiredMajorVersion,
+ int requiredMinorVersion) throws DeviceNotAvailableException {
+ String roOpenglesVersion = device.getProperty("ro.opengles.version");
+
+ if (roOpenglesVersion == null)
+ return false;
+
+ int intValue = Integer.parseInt(roOpenglesVersion);
+
+ int majorVersion = ((intValue & 0xffff0000) >> 16);
+ int minorVersion = (intValue & 0xffff);
+
+ return (majorVersion > requiredMajorVersion)
+ || (majorVersion == requiredMajorVersion && minorVersion >= requiredMinorVersion);
+ }
+
+ /**
+ * Query if rendertarget is supported
+ */
+ private boolean isSupportedGlesRenderConfig(BatchRunConfiguration runConfig)
+ throws DeviceNotAvailableException, CapabilityQueryFailureException {
+ // query if configuration is supported
+ final StringBuilder configCommandLine =
+ new StringBuilder(getRunConfigDisplayCmdLine(runConfig));
+ if (configCommandLine.length() != 0) {
+ configCommandLine.append(" ");
+ }
+ configCommandLine.append("--deqp-gl-major-version=");
+ configCommandLine.append(getGlesMajorVersion());
+ configCommandLine.append(" --deqp-gl-minor-version=");
+ configCommandLine.append(getGlesMinorVersion());
+
+ final String commandLine = configCommandLine.toString();
+
+ // check for cached result first
+ if (mConfigQuerySupportCache.containsKey(commandLine)) {
+ return mConfigQuerySupportCache.get(commandLine);
+ }
+
+ final boolean supported = queryIsSupportedConfigCommandLine(commandLine);
+ mConfigQuerySupportCache.put(commandLine, supported);
+ return supported;
+ }
+
+ private boolean queryIsSupportedConfigCommandLine(String deqpCommandLine)
+ throws DeviceNotAvailableException, CapabilityQueryFailureException {
+ final String instrumentationName =
+ "com.drawelements.deqp/com.drawelements.deqp.platformutil.DeqpPlatformCapabilityQueryInstrumentation";
+ final String command = String.format(
+ "am instrument %s -w -e deqpQueryType renderConfigSupported -e deqpCmdLine \"%s\""
+ + " %s",
+ AbiUtils.createAbiFlag(mAbi.getName()), deqpCommandLine, instrumentationName);
+
+ final PlatformQueryInstrumentationParser parser = new PlatformQueryInstrumentationParser();
+ mDevice.executeShellCommand(command, parser);
+ parser.flush();
+
+ if (parser.wasSuccessful() && parser.getResultCode() == 0 &&
+ parser.getResultMap().containsKey("Supported")) {
+ if ("Yes".equals(parser.getResultMap().get("Supported"))) {
+ return true;
+ } else if ("No".equals(parser.getResultMap().get("Supported"))) {
+ return false;
+ } else {
+ CLog.e("Capability query did not return a result");
+ throw new CapabilityQueryFailureException();
+ }
+ } else if (parser.wasSuccessful()) {
+ CLog.e("Failed to run capability query. Code: %d, Result: %s",
+ parser.getResultCode(), parser.getResultMap().toString());
+ throw new CapabilityQueryFailureException();
+ } else {
+ CLog.e("Failed to run capability query");
+ throw new CapabilityQueryFailureException();
+ }
+ }
+
+ /**
+ * Return feature set supported by the device
+ */
+ private Set<String> getDeviceFeatures(ITestDevice device)
+ throws DeviceNotAvailableException, CapabilityQueryFailureException {
+ if (mDeviceFeatures == null) {
+ mDeviceFeatures = queryDeviceFeatures(device);
+ }
+ return mDeviceFeatures;
+ }
+
+ /**
+ * Query feature set supported by the device
+ */
+ private static Set<String> queryDeviceFeatures(ITestDevice device)
+ throws DeviceNotAvailableException, CapabilityQueryFailureException {
+ // NOTE: Almost identical code in BaseDevicePolicyTest#hasDeviceFeatures
+ // TODO: Move this logic to ITestDevice.
+ String command = "pm list features";
+ String commandOutput = device.executeShellCommand(command);
+
+ // Extract the id of the new user.
+ HashSet<String> availableFeatures = new HashSet<>();
+ for (String feature: commandOutput.split("\\s+")) {
+ // Each line in the output of the command has the format "feature:{FEATURE_VALUE}".
+ String[] tokens = feature.split(":");
+ if (tokens.length < 2 || !"feature".equals(tokens[0])) {
+ CLog.e("Failed parse features. Unexpect format on line \"%s\"", tokens[0]);
+ throw new CapabilityQueryFailureException();
+ }
+ availableFeatures.add(tokens[1]);
+ }
+ return availableFeatures;
+ }
+
+ private boolean isPortraitClassRotation(String rotation) {
+ return BatchRunConfiguration.ROTATION_PORTRAIT.equals(rotation) ||
+ BatchRunConfiguration.ROTATION_REVERSE_PORTRAIT.equals(rotation);
+ }
+
+ private boolean isLandscapeClassRotation(String rotation) {
+ return BatchRunConfiguration.ROTATION_LANDSCAPE.equals(rotation) ||
+ BatchRunConfiguration.ROTATION_REVERSE_LANDSCAPE.equals(rotation);
+ }
+
+ /**
+ * Install dEQP OnDevice Package
+ */
+ private void installTestApk() throws DeviceNotAvailableException {
+ try {
+ File apkFile = new File(mBuildHelper.getTestsDir(), DEQP_ONDEVICE_APK);
+ String[] options = {AbiUtils.createAbiFlag(mAbi.getName())};
+ String errorCode = getDevice().installPackage(apkFile, true, options);
+ if (errorCode != null) {
+ CLog.e("Failed to install %s. Reason: %s", DEQP_ONDEVICE_APK, errorCode);
+ }
+ } catch (FileNotFoundException e) {
+ CLog.e("Could not find test apk %s", DEQP_ONDEVICE_APK);
+ }
+ }
+
+ /**
+ * Uninstall dEQP OnDevice Package
+ */
+ private void uninstallTestApk() throws DeviceNotAvailableException {
+ getDevice().uninstallPackage(DEQP_ONDEVICE_PKG);
+ }
+
+ /**
+ * Parse gl nature from package name
+ */
+ private boolean isOpenGlEsPackage() {
+ if ("dEQP-GLES2".equals(mDeqpPackage) || "dEQP-GLES3".equals(mDeqpPackage) ||
+ "dEQP-GLES31".equals(mDeqpPackage)) {
+ return true;
+ } else if ("dEQP-EGL".equals(mDeqpPackage)) {
+ return false;
+ } else {
+ throw new IllegalStateException("dEQP runner was created with illegal name");
+ }
+ }
+
+ /**
+ * Check GL support (based on package name)
+ */
+ private boolean isSupportedGles() throws DeviceNotAvailableException {
+ return isSupportedGles(mDevice, getGlesMajorVersion(), getGlesMinorVersion());
+ }
+
+ /**
+ * Get GL major version (based on package name)
+ */
+ private int getGlesMajorVersion() throws DeviceNotAvailableException {
+ if ("dEQP-GLES2".equals(mDeqpPackage)) {
+ return 2;
+ } else if ("dEQP-GLES3".equals(mDeqpPackage)) {
+ return 3;
+ } else if ("dEQP-GLES31".equals(mDeqpPackage)) {
+ return 3;
+ } else {
+ throw new IllegalStateException("getGlesMajorVersion called for non gles pkg");
+ }
+ }
+
+ /**
+ * Get GL minor version (based on package name)
+ */
+ private int getGlesMinorVersion() throws DeviceNotAvailableException {
+ if ("dEQP-GLES2".equals(mDeqpPackage)) {
+ return 0;
+ } else if ("dEQP-GLES3".equals(mDeqpPackage)) {
+ return 0;
+ } else if ("dEQP-GLES31".equals(mDeqpPackage)) {
+ return 1;
+ } else {
+ throw new IllegalStateException("getGlesMinorVersion called for non gles pkg");
+ }
+ }
+
+ private static List<Pattern> buildPatternList(List<String> filters) {
+ List<Pattern> patterns = new ArrayList(filters.size());
+ for (String filter : filters) {
+ patterns.add(Pattern.compile(filter.replace(".","\\.").replace("*",".*")));
+ }
+ return patterns;
+ }
+
+ private static boolean matchesAny(TestIdentifier test, List<Pattern> patterns) {
+ for (Pattern pattern : patterns) {
+ if (pattern.matcher(test.toString()).matches()) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /**
+ * Filter tests.
+ *
+ * '*' is 0 or more characters. '.' is interpreted verbatim.
+ *
+ * Optimized for small number of filters against many tests.
+ *
+ */
+ private static void filterTests(Map<TestIdentifier, Set<BatchRunConfiguration>> tests,
+ List<String> includeFilters,
+ List<String> excludeFilters) {
+ // We could filter faster by building the test case tree.
+ // Let's see if this is fast enough.
+ List<Pattern> includes = buildPatternList(includeFilters);
+ List<Pattern> excludes = buildPatternList(excludeFilters);
+
+ List<TestIdentifier> testList = new ArrayList(tests.keySet());
+ for (TestIdentifier test : testList) {
+ // Remove test if it does not match includes or matches
+ // excludes.
+ // Empty include filter includes everything.
+ if (includes.isEmpty() || matchesAny(test, includes)) {
+ if (!matchesAny(test, excludes)) {
+ continue;
+ }
+ }
+ tests.remove(test);
+ }
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void run(ITestInvocationListener listener) throws DeviceNotAvailableException {
+ final Map<String, String> emptyMap = Collections.emptyMap();
+ final boolean isSupportedApi = !isOpenGlEsPackage() || isSupportedGles();
+
+ try {
+ Reader reader = mCaselistReader;
+ if (reader == null) {
+ File testlist = new File(mBuildHelper.getTestsDir(), mCaselistFile);
+ if (!testlist.isFile()) {
+ throw new FileNotFoundException();
+ }
+ reader = new FileReader(testlist);
+ }
+ mTestInstances = generateTestInstances(reader, mConfigName, mScreenRotation, mSurfaceType);
+ mCaselistReader = null;
+ reader.close();
+ }
+ catch (FileNotFoundException e) {
+ throw new RuntimeException("Cannot read deqp test list file: " + mCaselistFile);
+ }
+ catch (IOException e) {
+ CLog.w("Failed to close test list reader.");
+ }
+ if (!mIncludeFilters.isEmpty() || !mExcludeFilters.isEmpty()) {
+ filterTests(mTestInstances, mIncludeFilters, mExcludeFilters);
+ }
+ mRemainingTests = new LinkedList<>(mTestInstances.keySet());
+
+ listener.testRunStarted(getId(), mRemainingTests.size());
+
+ try {
+ if (isSupportedApi) {
+ // Make sure there is no pre-existing package form earlier interrupted test run.
+ uninstallTestApk();
+ installTestApk();
+
+ mInstanceListerner.setSink(listener);
+ mDeviceRecovery.setDevice(mDevice);
+ runTests();
+
+ uninstallTestApk();
+ } else {
+ // Pass all tests if OpenGL ES version is not supported
+ fakePassTests(listener);
+ }
+ } catch (CapabilityQueryFailureException ex) {
+ // Platform is not behaving correctly, for example crashing when trying to create
+ // a window. Instead of silenty failing, signal failure by leaving the rest of the
+ // test cases in "NotExecuted" state
+ uninstallTestApk();
+ }
+
+ listener.testRunEnded(0, emptyMap);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void addIncludeFilter(String filter) {
+ mIncludeFilters.add(filter);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void addAllIncludeFilters(List<String> filters) {
+ mIncludeFilters.addAll(filters);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void addExcludeFilter(String filter) {
+ mExcludeFilters.add(filter);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void addAllExcludeFilters(List<String> filters) {
+ mExcludeFilters.addAll(filters);
+ }
+}
diff --git a/android/cts/runner/tests/Android.mk b/android/cts/runner/tests/Android.mk
new file mode 100644
index 0000000..b4a3df2
--- /dev/null
+++ b/android/cts/runner/tests/Android.mk
@@ -0,0 +1,27 @@
+# Copyright (C) 2010 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+LOCAL_PATH := $(call my-dir)
+
+include $(CLEAR_VARS)
+
+# Only compile source java files in this lib.
+LOCAL_SRC_FILES := $(call all-java-files-under, src)
+
+LOCAL_MODULE := CtsDeqpRunnerTests
+LOCAL_MODULE_TAGS := optional
+LOCAL_JAVA_LIBRARIES := cts-tradefed_v2 compatibility-host-util tradefed-prebuilt CtsDeqp
+LOCAL_STATIC_JAVA_LIBRARIES := easymock
+
+include $(BUILD_HOST_JAVA_LIBRARY)
diff --git a/android/cts/runner/tests/run_tests.sh b/android/cts/runner/tests/run_tests.sh
new file mode 100755
index 0000000..7293894
--- /dev/null
+++ b/android/cts/runner/tests/run_tests.sh
@@ -0,0 +1,62 @@
+#!/bin/bash
+
+# Copyright (C) 2015 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Helper script for running unit tests for compatibility libraries
+
+checkFile() {
+ if [ ! -f "$1" ]; then
+ echo "Unable to locate $1"
+ exit
+ fi;
+}
+
+# check if in Android build env
+if [ ! -z ${ANDROID_BUILD_TOP} ]; then
+ HOST=`uname`
+ if [ "$HOST" == "Linux" ]; then
+ OS="linux-x86"
+ elif [ "$HOST" == "Darwin" ]; then
+ OS="darwin-x86"
+ else
+ echo "Unrecognized OS"
+ exit
+ fi;
+fi;
+
+JAR_DIR=${ANDROID_HOST_OUT}/framework
+TF_CONSOLE=com.android.tradefed.command.Console
+
+############### Run the cts tests ###############
+JARS="
+ compatibility-host-util\
+ cts-tradefed_v2\
+ ddmlib-prebuilt\
+ hosttestlib\
+ CtsDeqp\
+ CtsDeqpRunnerTests\
+ tradefed-prebuilt"
+JAR_PATH=
+for JAR in $JARS; do
+ checkFile ${JAR_DIR}/${JAR}.jar
+ JAR_PATH=${JAR_PATH}:${JAR_DIR}/${JAR}.jar
+done
+
+TEST_CLASSES="
+ com.drawelements.deqp.runner.DeqpTestRunnerTest"
+
+for CLASS in ${TEST_CLASSES}; do
+ java $RDBG_FLAG -cp ${JAR_PATH} ${TF_CONSOLE} run singleCommand host -n --class ${CLASS} "$@"
+done
diff --git a/android/cts/runner/tests/src/com/drawelements/deqp/runner/DeqpTestRunnerTest.java b/android/cts/runner/tests/src/com/drawelements/deqp/runner/DeqpTestRunnerTest.java
new file mode 100644
index 0000000..a84eef5
--- /dev/null
+++ b/android/cts/runner/tests/src/com/drawelements/deqp/runner/DeqpTestRunnerTest.java
@@ -0,0 +1,1920 @@
+/*
+ * Copyright (C) 2014 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.drawelements.deqp.runner;
+
+import com.android.compatibility.common.tradefed.build.CompatibilityBuildHelper;
+import com.android.compatibility.common.tradefed.testtype.Abi;
+import com.android.compatibility.common.util.AbiUtils;
+import com.android.ddmlib.IDevice;
+import com.android.ddmlib.IShellOutputReceiver;
+import com.android.ddmlib.ShellCommandUnresponsiveException;
+import com.android.ddmlib.testrunner.TestIdentifier;
+import com.android.tradefed.build.IFolderBuildInfo;
+import com.android.tradefed.config.ConfigurationException;
+import com.android.tradefed.config.OptionSetter;
+import com.android.tradefed.device.DeviceNotAvailableException;
+import com.android.tradefed.device.ITestDevice;
+import com.android.tradefed.result.ITestInvocationListener;
+import com.android.tradefed.testtype.IAbi;
+import com.android.tradefed.util.IRunUtil;
+import com.android.tradefed.util.RunInterruptedException;
+
+import junit.framework.TestCase;
+
+import org.easymock.EasyMock;
+import org.easymock.IAnswer;
+import org.easymock.IMocksControl;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.StringReader;
+import java.io.StringWriter;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Unit tests for {@link DeqpTestRunner}.
+ */
+public class DeqpTestRunnerTest extends TestCase {
+ private static final String NAME = "dEQP-GLES3";
+ private static final IAbi ABI = new Abi("armeabi-v7a", "32");
+ private static final String CASE_LIST_FILE_NAME = "/sdcard/dEQP-TestCaseList.txt";
+ private static final String LOG_FILE_NAME = "/sdcard/TestLog.qpa";
+ private static final String INSTRUMENTATION_NAME =
+ "com.drawelements.deqp/com.drawelements.deqp.testercore.DeqpInstrumentation";
+ private static final String QUERY_INSTRUMENTATION_NAME =
+ "com.drawelements.deqp/com.drawelements.deqp.platformutil.DeqpPlatformCapabilityQueryInstrumentation";
+ private static final String DEQP_ONDEVICE_APK = "com.drawelements.deqp.apk";
+ private static final String DEQP_ONDEVICE_PKG = "com.drawelements.deqp";
+ private static final String ONLY_LANDSCAPE_FEATURES =
+ "feature:"+DeqpTestRunner.FEATURE_LANDSCAPE;
+ private static final String ALL_FEATURES =
+ ONLY_LANDSCAPE_FEATURES + "\nfeature:"+DeqpTestRunner.FEATURE_PORTRAIT;
+ private static List<Map<String,String>> DEFAULT_INSTANCE_ARGS;
+
+ static {
+ DEFAULT_INSTANCE_ARGS = new ArrayList<>(1);
+ DEFAULT_INSTANCE_ARGS.add(new HashMap<String,String>());
+ DEFAULT_INSTANCE_ARGS.iterator().next().put("glconfig", "rgba8888d24s8");
+ DEFAULT_INSTANCE_ARGS.iterator().next().put("rotation", "unspecified");
+ DEFAULT_INSTANCE_ARGS.iterator().next().put("surfacetype", "window");
+ }
+
+ private static class StubRecovery implements DeqpTestRunner.IRecovery {
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void setSleepProvider(DeqpTestRunner.ISleepProvider sleepProvider) {
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void setDevice(ITestDevice device) {
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void onExecutionProgressed() {
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void recoverConnectionRefused() throws DeviceNotAvailableException {
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void recoverComLinkKilled() throws DeviceNotAvailableException {
+ }
+ };
+
+ public static class BuildHelperMock extends CompatibilityBuildHelper {
+ public BuildHelperMock(IFolderBuildInfo buildInfo) {
+ super(buildInfo);
+ }
+ @Override
+ public File getTestsDir() throws FileNotFoundException {
+ return new File("logs");
+ }
+ }
+
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected void setUp() throws Exception {
+ super.setUp();
+ }
+
+ private static DeqpTestRunner buildGlesTestRunner(int majorVersion,
+ int minorVersion,
+ Collection<TestIdentifier> tests) throws ConfigurationException, FileNotFoundException {
+ StringWriter testlist = new StringWriter();
+ for (TestIdentifier test : tests) {
+ testlist.write(test.getClassName() + "." + test.getTestName() + "\n");
+ }
+ return buildGlesTestRunner(majorVersion, minorVersion, testlist.toString());
+ }
+
+ private static DeqpTestRunner buildGlesTestRunner(int majorVersion,
+ int minorVersion,
+ String testlist) throws ConfigurationException, FileNotFoundException {
+ DeqpTestRunner runner = new DeqpTestRunner();
+ OptionSetter setter = new OptionSetter(runner);
+
+ String deqpPackage = "dEQP-GLES" + Integer.toString(majorVersion)
+ + (minorVersion > 0 ? Integer.toString(minorVersion) : "");
+
+ setter.setOptionValue("deqp-package", deqpPackage);
+ setter.setOptionValue("deqp-gl-config-name", "rgba8888d24s8");
+ setter.setOptionValue("deqp-caselist-file", "dummyfile.txt");
+ setter.setOptionValue("deqp-screen-rotation", "unspecified");
+ setter.setOptionValue("deqp-surface-type", "window");
+
+ runner.setCaselistReader(new StringReader(testlist));
+
+ runner.setAbi(ABI);
+
+ IFolderBuildInfo mockIFolderBuildInfo = EasyMock.createMock(IFolderBuildInfo.class);
+ EasyMock.replay(mockIFolderBuildInfo);
+ CompatibilityBuildHelper mockHelper = new BuildHelperMock(mockIFolderBuildInfo);
+ runner.setBuildHelper(mockHelper);
+
+ return runner;
+ }
+
+ private static String getTestId(DeqpTestRunner runner) {
+ return AbiUtils.createId(ABI.getName(), runner.getPackageName());
+ }
+
+ /**
+ * Test version of OpenGL ES.
+ */
+ private void testGlesVersion(int requiredMajorVersion, int requiredMinorVersion, int majorVersion, int minorVersion) throws Exception {
+ final TestIdentifier testId = new TestIdentifier("dEQP-GLES"
+ + Integer.toString(requiredMajorVersion) + Integer.toString(requiredMinorVersion)
+ + ".info", "version");
+
+ final String testPath = "dEQP-GLES"
+ + Integer.toString(requiredMajorVersion) + Integer.toString(requiredMinorVersion)
+ +".info.version";
+
+ final String testTrie = "{dEQP-GLES"
+ + Integer.toString(requiredMajorVersion) + Integer.toString(requiredMinorVersion)
+ + "{info{version}}}";
+
+ final String resultCode = "Pass";
+
+ /* MultiLineReceiver expects "\r\n" line ending. */
+ final String output = "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=" + testPath + "\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=" + resultCode + "\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Detail" + resultCode + "\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_CODE: 0\r\n";
+
+ ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class);
+ ITestInvocationListener mockListener
+ = EasyMock.createStrictMock(ITestInvocationListener.class);
+ IDevice mockIDevice = EasyMock.createMock(IDevice.class);
+ Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>();
+ tests.add(testId);
+
+ DeqpTestRunner deqpTest = buildGlesTestRunner(requiredMajorVersion, requiredMinorVersion, tests);
+
+ int version = (majorVersion << 16) | minorVersion;
+ EasyMock.expect(mockDevice.getProperty("ro.opengles.version"))
+ .andReturn(Integer.toString(version)).atLeastOnce();
+
+ if (majorVersion > requiredMajorVersion
+ || (majorVersion == requiredMajorVersion && minorVersion >= requiredMinorVersion)) {
+
+ EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG)))
+ .andReturn("").once();
+ EasyMock.expect(mockDevice.installPackage(EasyMock.<File>anyObject(),
+ EasyMock.eq(true),
+ EasyMock.eq(AbiUtils.createAbiFlag(ABI.getName()))))
+ .andReturn(null).once();
+
+ expectRenderConfigQuery(mockDevice, requiredMajorVersion,
+ requiredMinorVersion);
+
+ String commandLine = String.format(
+ "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 "
+ + "--deqp-screen-rotation=unspecified "
+ + "--deqp-surface-type=window "
+ + "--deqp-log-images=disable "
+ + "--deqp-watchdog=enable",
+ CASE_LIST_FILE_NAME);
+
+ runInstrumentationLineAndAnswer(mockDevice, mockIDevice, testTrie, commandLine,
+ output);
+
+ EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG)))
+ .andReturn("").once();
+ }
+
+ mockListener.testRunStarted(getTestId(deqpTest), 1);
+ EasyMock.expectLastCall().once();
+
+ mockListener.testStarted(EasyMock.eq(testId));
+ EasyMock.expectLastCall().once();
+
+ mockListener.testEnded(EasyMock.eq(testId), EasyMock.<Map<String, String>>notNull());
+ EasyMock.expectLastCall().once();
+
+ mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<Map<String, String>>notNull());
+ EasyMock.expectLastCall().once();
+
+ EasyMock.replay(mockDevice, mockIDevice);
+ EasyMock.replay(mockListener);
+
+ deqpTest.setDevice(mockDevice);
+ deqpTest.run(mockListener);
+
+ EasyMock.verify(mockListener);
+ EasyMock.verify(mockDevice, mockIDevice);
+ }
+
+ private void expectRenderConfigQuery(ITestDevice mockDevice, int majorVersion,
+ int minorVersion) throws Exception {
+ expectRenderConfigQuery(mockDevice,
+ String.format("--deqp-gl-config-name=rgba8888d24s8 "
+ + "--deqp-screen-rotation=unspecified "
+ + "--deqp-surface-type=window "
+ + "--deqp-gl-major-version=%d "
+ + "--deqp-gl-minor-version=%d", majorVersion, minorVersion));
+ }
+
+ private void expectRenderConfigQuery(ITestDevice mockDevice, String commandLine)
+ throws Exception {
+ expectRenderConfigQueryAndReturn(mockDevice, commandLine, "Yes");
+ }
+
+ private void expectRenderConfigQueryAndReturn(ITestDevice mockDevice, String commandLine,
+ String output) throws Exception {
+ final String queryOutput = "INSTRUMENTATION_RESULT: Supported=" + output + "\r\n"
+ + "INSTRUMENTATION_CODE: 0\r\n";
+ final String command = String.format(
+ "am instrument %s -w -e deqpQueryType renderConfigSupported -e deqpCmdLine "
+ + "\"%s\" %s",
+ AbiUtils.createAbiFlag(ABI.getName()), commandLine,
+ QUERY_INSTRUMENTATION_NAME);
+
+ mockDevice.executeShellCommand(EasyMock.eq(command),
+ EasyMock.<IShellOutputReceiver>notNull());
+
+ EasyMock.expectLastCall().andAnswer(new IAnswer<Object>() {
+ @Override
+ public Object answer() {
+ IShellOutputReceiver receiver
+ = (IShellOutputReceiver)EasyMock.getCurrentArguments()[1];
+
+ receiver.addOutput(queryOutput.getBytes(), 0, queryOutput.length());
+ receiver.flush();
+
+ return null;
+ }
+ });
+ }
+
+ /**
+ * Test that result code produces correctly pass or fail.
+ */
+ private void testResultCode(final String resultCode, boolean pass) throws Exception {
+ final TestIdentifier testId = new TestIdentifier("dEQP-GLES3.info", "version");
+ final String testPath = "dEQP-GLES3.info.version";
+ final String testTrie = "{dEQP-GLES3{info{version}}}";
+
+ /* MultiLineReceiver expects "\r\n" line ending. */
+ final String output = "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=" + testPath + "\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=" + resultCode + "\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Detail" + resultCode + "\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_CODE: 0\r\n";
+
+ ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class);
+ ITestInvocationListener mockListener
+ = EasyMock.createStrictMock(ITestInvocationListener.class);
+ IDevice mockIDevice = EasyMock.createMock(IDevice.class);
+
+ Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>();
+ tests.add(testId);
+
+ DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests);
+
+ int version = 3 << 16;
+ EasyMock.expect(mockDevice.getProperty("ro.opengles.version"))
+ .andReturn(Integer.toString(version)).atLeastOnce();
+
+ EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).andReturn("")
+ .once();
+
+ EasyMock.expect(mockDevice.installPackage(EasyMock.<File>anyObject(),
+ EasyMock.eq(true), EasyMock.eq(AbiUtils.createAbiFlag(ABI.getName()))))
+ .andReturn(null).once();
+
+ expectRenderConfigQuery(mockDevice, 3, 0);
+
+ String commandLine = String.format(
+ "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 "
+ + "--deqp-screen-rotation=unspecified "
+ + "--deqp-surface-type=window "
+ + "--deqp-log-images=disable "
+ + "--deqp-watchdog=enable",
+ CASE_LIST_FILE_NAME);
+
+ runInstrumentationLineAndAnswer(mockDevice, mockIDevice, testTrie, commandLine, output);
+
+ mockListener.testRunStarted(getTestId(deqpTest), 1);
+ EasyMock.expectLastCall().once();
+
+ mockListener.testStarted(EasyMock.eq(testId));
+ EasyMock.expectLastCall().once();
+
+ if (!pass) {
+ mockListener.testFailed(testId,
+ "=== with config {glformat=rgba8888d24s8,rotation=unspecified,surfacetype=window} ===\n"
+ + resultCode + ": Detail" + resultCode);
+
+ EasyMock.expectLastCall().once();
+ }
+
+ mockListener.testEnded(EasyMock.eq(testId), EasyMock.<Map<String, String>>notNull());
+ EasyMock.expectLastCall().once();
+
+ mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<Map<String, String>>notNull());
+ EasyMock.expectLastCall().once();
+
+ EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).andReturn("")
+ .once();
+
+ EasyMock.replay(mockDevice, mockIDevice);
+ EasyMock.replay(mockListener);
+
+ deqpTest.setDevice(mockDevice);
+ deqpTest.run(mockListener);
+
+ EasyMock.verify(mockListener);
+ EasyMock.verify(mockDevice, mockIDevice);
+ }
+
+ /**
+ * Test running multiple test cases.
+ */
+ public void testRun_multipleTests() throws Exception {
+ /* MultiLineReceiver expects "\r\n" line ending. */
+ final String output = "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.vendor\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.renderer\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.version\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.shading_language_version\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.extensions\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.render_target\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_CODE: 0\r\n";
+
+ final TestIdentifier[] testIds = {
+ new TestIdentifier("dEQP-GLES3.info", "vendor"),
+ new TestIdentifier("dEQP-GLES3.info", "renderer"),
+ new TestIdentifier("dEQP-GLES3.info", "version"),
+ new TestIdentifier("dEQP-GLES3.info", "shading_language_version"),
+ new TestIdentifier("dEQP-GLES3.info", "extensions"),
+ new TestIdentifier("dEQP-GLES3.info", "render_target")
+ };
+
+ final String[] testPaths = {
+ "dEQP-GLES3.info.vendor",
+ "dEQP-GLES3.info.renderer",
+ "dEQP-GLES3.info.version",
+ "dEQP-GLES3.info.shading_language_version",
+ "dEQP-GLES3.info.extensions",
+ "dEQP-GLES3.info.render_target"
+ };
+
+ final String testTrie
+ = "{dEQP-GLES3{info{vendor,renderer,version,shading_language_version,extensions,render_target}}}";
+
+ ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class);
+ ITestInvocationListener mockListener
+ = EasyMock.createStrictMock(ITestInvocationListener.class);
+ IDevice mockIDevice = EasyMock.createMock(IDevice.class);
+
+ Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>();
+
+ for (TestIdentifier id : testIds) {
+ tests.add(id);
+ }
+
+ DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests);
+
+ int version = 3 << 16;
+ EasyMock.expect(mockDevice.getProperty("ro.opengles.version"))
+ .andReturn(Integer.toString(version)).atLeastOnce();
+
+ EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).andReturn("")
+ .once();
+ EasyMock.expect(mockDevice.installPackage(EasyMock.<File>anyObject(),
+ EasyMock.eq(true), EasyMock.eq(AbiUtils.createAbiFlag(ABI.getName()))))
+ .andReturn(null).once();
+
+ expectRenderConfigQuery(mockDevice, 3, 0);
+
+ String commandLine = String.format(
+ "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 "
+ + "--deqp-screen-rotation=unspecified "
+ + "--deqp-surface-type=window "
+ + "--deqp-log-images=disable "
+ + "--deqp-watchdog=enable",
+ CASE_LIST_FILE_NAME);
+
+ runInstrumentationLineAndAnswer(mockDevice, mockIDevice, testTrie, commandLine, output);
+
+ mockListener.testRunStarted(getTestId(deqpTest), testPaths.length);
+ EasyMock.expectLastCall().once();
+
+ for (int i = 0; i < testPaths.length; i++) {
+ mockListener.testStarted(EasyMock.eq(testIds[i]));
+ EasyMock.expectLastCall().once();
+
+ mockListener.testEnded(EasyMock.eq(testIds[i]),
+ EasyMock.<Map<String, String>>notNull());
+
+ EasyMock.expectLastCall().once();
+ }
+
+ mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<Map<String, String>>notNull());
+ EasyMock.expectLastCall().once();
+
+ EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).andReturn("")
+ .once();
+
+ EasyMock.replay(mockDevice, mockIDevice);
+ EasyMock.replay(mockListener);
+
+ deqpTest.setDevice(mockDevice);
+ deqpTest.run(mockListener);
+
+ EasyMock.verify(mockListener);
+ EasyMock.verify(mockDevice, mockIDevice);
+ }
+
+ private void testFiltering(List<String> includes,
+ List<String> excludes,
+ List<TestIdentifier> fullTestList,
+ String expectedTrie,
+ List<TestIdentifier> expectedTests) throws Exception{
+
+ /* MultiLineReceiver expects "\r\n" line ending. */
+ final String outputHeader = "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n";
+
+ final String outputEnd = "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_CODE: 0\r\n";
+
+ StringWriter output = new StringWriter();
+ output.write(outputHeader);
+ for (TestIdentifier test : expectedTests) {
+ output.write("INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n");
+ output.write("INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=");
+ output.write(test.getClassName());
+ output.write(".");
+ output.write(test.getTestName());
+ output.write("\r\n");
+ output.write("INSTRUMENTATION_STATUS_CODE: 0\r\n");
+ output.write("INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n");
+ output.write("INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n");
+ output.write("INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n");
+ output.write("INSTRUMENTATION_STATUS_CODE: 0\r\n");
+ output.write("INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n");
+ output.write("INSTRUMENTATION_STATUS_CODE: 0\r\n");
+ }
+ output.write(outputEnd);
+
+ ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class);
+ ITestInvocationListener mockListener
+ = EasyMock.createStrictMock(ITestInvocationListener.class);
+ IDevice mockIDevice = EasyMock.createMock(IDevice.class);
+
+ DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, fullTestList);
+ if (includes != null) {
+ deqpTest.addAllIncludeFilters(includes);
+ }
+ if (excludes != null) {
+ deqpTest.addAllExcludeFilters(excludes);
+ }
+
+ int version = 3 << 16;
+ EasyMock.expect(mockDevice.getProperty("ro.opengles.version"))
+ .andReturn(Integer.toString(version)).atLeastOnce();
+
+ EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).andReturn("")
+ .once();
+ EasyMock.expect(mockDevice.installPackage(EasyMock.<File>anyObject(),
+ EasyMock.eq(true), EasyMock.eq(AbiUtils.createAbiFlag(ABI.getName()))))
+ .andReturn(null).once();
+
+ mockListener.testRunStarted(getTestId(deqpTest), expectedTests.size());
+ EasyMock.expectLastCall().once();
+
+ if (expectedTests.size() > 0)
+ {
+ expectRenderConfigQuery(mockDevice, 3, 0);
+
+ String commandLine = String.format(
+ "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 "
+ + "--deqp-screen-rotation=unspecified "
+ + "--deqp-surface-type=window "
+ + "--deqp-log-images=disable "
+ + "--deqp-watchdog=enable",
+ CASE_LIST_FILE_NAME);
+
+ runInstrumentationLineAndAnswer(mockDevice, mockIDevice, expectedTrie, commandLine, output.toString());
+
+ for (int i = 0; i < expectedTests.size(); i++) {
+ mockListener.testStarted(EasyMock.eq(expectedTests.get(i)));
+ EasyMock.expectLastCall().once();
+
+ mockListener.testEnded(EasyMock.eq(expectedTests.get(i)),
+ EasyMock.<Map<String, String>>notNull());
+
+ EasyMock.expectLastCall().once();
+ }
+ }
+
+ mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<Map<String, String>>notNull());
+ EasyMock.expectLastCall().once();
+
+ EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).andReturn("")
+ .once();
+
+ EasyMock.replay(mockDevice, mockIDevice);
+ EasyMock.replay(mockListener);
+
+ deqpTest.setDevice(mockDevice);
+ deqpTest.run(mockListener);
+
+ EasyMock.verify(mockListener);
+ EasyMock.verify(mockDevice, mockIDevice);
+ output.close();
+ }
+
+ public void testRun_trivialIncludeFilter() throws Exception {
+ final TestIdentifier[] testIds = {
+ new TestIdentifier("dEQP-GLES3.missing", "no"),
+ new TestIdentifier("dEQP-GLES3.missing", "nope"),
+ new TestIdentifier("dEQP-GLES3.missing", "donotwant"),
+ new TestIdentifier("dEQP-GLES3.pick_me", "yes"),
+ new TestIdentifier("dEQP-GLES3.pick_me", "ok"),
+ new TestIdentifier("dEQP-GLES3.pick_me", "accepted"),
+ };
+
+ List<TestIdentifier> allTests = new ArrayList<TestIdentifier>();
+ for (TestIdentifier id : testIds) {
+ allTests.add(id);
+ }
+
+ List<TestIdentifier> activeTests = new ArrayList<TestIdentifier>();
+ activeTests.add(testIds[3]);
+ activeTests.add(testIds[4]);
+ activeTests.add(testIds[5]);
+
+ String expectedTrie = "{dEQP-GLES3{pick_me{yes,ok,accepted}}}";
+
+ ArrayList<String> includes = new ArrayList();
+ includes.add("dEQP-GLES3.pick_me#*");
+ testFiltering(includes, null, allTests, expectedTrie, activeTests);
+ }
+
+ public void testRun_trivialExcludeFilter() throws Exception {
+ final TestIdentifier[] testIds = {
+ new TestIdentifier("dEQP-GLES3.missing", "no"),
+ new TestIdentifier("dEQP-GLES3.missing", "nope"),
+ new TestIdentifier("dEQP-GLES3.missing", "donotwant"),
+ new TestIdentifier("dEQP-GLES3.pick_me", "yes"),
+ new TestIdentifier("dEQP-GLES3.pick_me", "ok"),
+ new TestIdentifier("dEQP-GLES3.pick_me", "accepted"),
+ };
+
+ List<TestIdentifier> allTests = new ArrayList<TestIdentifier>();
+ for (TestIdentifier id : testIds) {
+ allTests.add(id);
+ }
+
+ List<TestIdentifier> activeTests = new ArrayList<TestIdentifier>();
+ activeTests.add(testIds[3]);
+ activeTests.add(testIds[4]);
+ activeTests.add(testIds[5]);
+
+ String expectedTrie = "{dEQP-GLES3{pick_me{yes,ok,accepted}}}";
+
+ ArrayList<String> excludes = new ArrayList();
+ excludes.add("dEQP-GLES3.missing#*");
+ testFiltering(null, excludes, allTests, expectedTrie, activeTests);
+ }
+
+ public void testRun_includeAndExcludeFilter() throws Exception {
+ final TestIdentifier[] testIds = {
+ new TestIdentifier("dEQP-GLES3.group1", "foo"),
+ new TestIdentifier("dEQP-GLES3.group1", "nope"),
+ new TestIdentifier("dEQP-GLES3.group1", "donotwant"),
+ new TestIdentifier("dEQP-GLES3.group2", "foo"),
+ new TestIdentifier("dEQP-GLES3.group2", "yes"),
+ new TestIdentifier("dEQP-GLES3.group2", "thoushallnotpass"),
+ };
+
+ List<TestIdentifier> allTests = new ArrayList<TestIdentifier>();
+ for (TestIdentifier id : testIds) {
+ allTests.add(id);
+ }
+
+ List<TestIdentifier> activeTests = new ArrayList<TestIdentifier>();
+ activeTests.add(testIds[4]);
+
+ String expectedTrie = "{dEQP-GLES3{group2{yes}}}";
+
+ ArrayList<String> includes = new ArrayList();
+ includes.add("dEQP-GLES3.group2#*");
+
+ ArrayList<String> excludes = new ArrayList();
+ excludes.add("*foo");
+ excludes.add("*thoushallnotpass");
+ testFiltering(includes, excludes, allTests, expectedTrie, activeTests);
+ }
+
+ public void testRun_includeAll() throws Exception {
+ final TestIdentifier[] testIds = {
+ new TestIdentifier("dEQP-GLES3.group1", "mememe"),
+ new TestIdentifier("dEQP-GLES3.group1", "yeah"),
+ new TestIdentifier("dEQP-GLES3.group1", "takeitall"),
+ new TestIdentifier("dEQP-GLES3.group2", "jeba"),
+ new TestIdentifier("dEQP-GLES3.group2", "yes"),
+ new TestIdentifier("dEQP-GLES3.group2", "granted"),
+ };
+
+ List<TestIdentifier> allTests = new ArrayList<TestIdentifier>();
+ for (TestIdentifier id : testIds) {
+ allTests.add(id);
+ }
+
+ String expectedTrie = "{dEQP-GLES3{group1{mememe,yeah,takeitall},group2{jeba,yes,granted}}}";
+
+ ArrayList<String> includes = new ArrayList();
+ includes.add("*");
+
+ testFiltering(includes, null, allTests, expectedTrie, allTests);
+ }
+
+ public void testRun_excludeAll() throws Exception {
+ final TestIdentifier[] testIds = {
+ new TestIdentifier("dEQP-GLES3.group1", "no"),
+ new TestIdentifier("dEQP-GLES3.group1", "nope"),
+ new TestIdentifier("dEQP-GLES3.group1", "nottoday"),
+ new TestIdentifier("dEQP-GLES3.group2", "banned"),
+ new TestIdentifier("dEQP-GLES3.group2", "notrecognized"),
+ new TestIdentifier("dEQP-GLES3.group2", "-2"),
+ };
+
+ List<TestIdentifier> allTests = new ArrayList<TestIdentifier>();
+ for (TestIdentifier id : testIds) {
+ allTests.add(id);
+ }
+
+ String expectedTrie = "";
+
+ ArrayList<String> excludes = new ArrayList();
+ excludes.add("*");
+
+ testFiltering(null, excludes, allTests, expectedTrie, new ArrayList<TestIdentifier>());
+ }
+
+ /**
+ * Test running a unexecutable test.
+ */
+ public void testRun_unexecutableTests() throws Exception {
+ final String instrumentationAnswerNoExecs =
+ "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_CODE: 0\r\n";
+
+ final TestIdentifier[] testIds = {
+ new TestIdentifier("dEQP-GLES3.missing", "no"),
+ new TestIdentifier("dEQP-GLES3.missing", "nope"),
+ new TestIdentifier("dEQP-GLES3.missing", "donotwant"),
+ };
+
+ final String[] testPaths = {
+ "dEQP-GLES3.missing.no",
+ "dEQP-GLES3.missing.nope",
+ "dEQP-GLES3.missing.donotwant",
+ };
+
+ ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class);
+ ITestInvocationListener mockListener
+ = EasyMock.createStrictMock(ITestInvocationListener.class);
+ IDevice mockIDevice = EasyMock.createMock(IDevice.class);
+
+ Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>();
+
+ for (TestIdentifier id : testIds) {
+ tests.add(id);
+ }
+
+ DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests);
+
+ int version = 3 << 16;
+ EasyMock.expect(mockDevice.getProperty("ro.opengles.version"))
+ .andReturn(Integer.toString(version)).atLeastOnce();
+
+ EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).andReturn("")
+ .once();
+ EasyMock.expect(mockDevice.installPackage(EasyMock.<File>anyObject(),
+ EasyMock.eq(true), EasyMock.eq(AbiUtils.createAbiFlag(ABI.getName()))))
+ .andReturn(null).once();
+
+ expectRenderConfigQuery(mockDevice, 3, 0);
+
+ String commandLine = String.format(
+ "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 "
+ + "--deqp-screen-rotation=unspecified "
+ + "--deqp-surface-type=window "
+ + "--deqp-log-images=disable "
+ + "--deqp-watchdog=enable",
+ CASE_LIST_FILE_NAME);
+
+ // first try
+ runInstrumentationLineAndAnswer(mockDevice, mockIDevice,
+ "{dEQP-GLES3{missing{no,nope,donotwant}}}", commandLine, instrumentationAnswerNoExecs);
+
+ // splitting begins
+ runInstrumentationLineAndAnswer(mockDevice, mockIDevice,
+ "{dEQP-GLES3{missing{no}}}", commandLine, instrumentationAnswerNoExecs);
+ runInstrumentationLineAndAnswer(mockDevice, mockIDevice,
+ "{dEQP-GLES3{missing{nope,donotwant}}}", commandLine, instrumentationAnswerNoExecs);
+ runInstrumentationLineAndAnswer(mockDevice, mockIDevice,
+ "{dEQP-GLES3{missing{nope}}}", commandLine, instrumentationAnswerNoExecs);
+ runInstrumentationLineAndAnswer(mockDevice, mockIDevice,
+ "{dEQP-GLES3{missing{donotwant}}}", commandLine, instrumentationAnswerNoExecs);
+
+ mockListener.testRunStarted(getTestId(deqpTest), testPaths.length);
+ EasyMock.expectLastCall().once();
+
+ for (int i = 0; i < testPaths.length; i++) {
+ mockListener.testStarted(EasyMock.eq(testIds[i]));
+ EasyMock.expectLastCall().once();
+
+ mockListener.testFailed(EasyMock.eq(testIds[i]),
+ EasyMock.eq("=== with config {glformat=rgba8888d24s8,rotation=unspecified,surfacetype=window} ===\n"
+ + "Abort: Test cannot be executed"));
+ EasyMock.expectLastCall().once();
+
+ mockListener.testEnded(EasyMock.eq(testIds[i]),
+ EasyMock.<Map<String, String>>notNull());
+ EasyMock.expectLastCall().once();
+ }
+
+ mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<Map<String, String>>notNull());
+ EasyMock.expectLastCall().once();
+
+ EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).andReturn("")
+ .once();
+
+ EasyMock.replay(mockDevice, mockIDevice);
+ EasyMock.replay(mockListener);
+
+ deqpTest.setDevice(mockDevice);
+ deqpTest.run(mockListener);
+
+ EasyMock.verify(mockListener);
+ EasyMock.verify(mockDevice, mockIDevice);
+ }
+
+ /**
+ * Test that test are left unexecuted if pm list query fails
+ */
+ public void testRun_queryPmListFailure()
+ throws Exception {
+ final TestIdentifier testId = new TestIdentifier("dEQP-GLES3.orientation", "test");
+
+ ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class);
+ ITestInvocationListener mockListener
+ = EasyMock.createStrictMock(ITestInvocationListener.class);
+ Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>();
+ tests.add(testId);
+
+ DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests);
+ OptionSetter setter = new OptionSetter(deqpTest);
+ // Note: If the rotation is the default unspecified, features are not queried at all
+ setter.setOptionValue("deqp-screen-rotation", "90");
+
+ deqpTest.setDevice(mockDevice);
+
+ int version = 3 << 16;
+ EasyMock.expect(mockDevice.getProperty("ro.opengles.version"))
+ .andReturn(Integer.toString(version)).atLeastOnce();
+
+ EasyMock.expect(mockDevice.executeShellCommand("pm list features"))
+ .andReturn("not a valid format");
+
+ EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).
+ andReturn("").once();
+
+ EasyMock.expect(mockDevice.installPackage(EasyMock.<File>anyObject(),
+ EasyMock.eq(true),
+ EasyMock.eq(AbiUtils.createAbiFlag(ABI.getName())))).andReturn(null)
+ .once();
+
+ EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG)))
+ .andReturn("").once();
+
+ mockListener.testRunStarted(getTestId(deqpTest), 1);
+ EasyMock.expectLastCall().once();
+
+ mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<Map<String, String>>notNull());
+ EasyMock.expectLastCall().once();
+
+ EasyMock.replay(mockDevice);
+ EasyMock.replay(mockListener);
+ deqpTest.run(mockListener);
+ EasyMock.verify(mockListener);
+ EasyMock.verify(mockDevice);
+ }
+
+ /**
+ * Test that test are left unexecuted if renderablity query fails
+ */
+ public void testRun_queryRenderabilityFailure()
+ throws Exception {
+ final TestIdentifier testId = new TestIdentifier("dEQP-GLES3.orientation", "test");
+
+ ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class);
+ ITestInvocationListener mockListener
+ = EasyMock.createStrictMock(ITestInvocationListener.class);
+
+ Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>();
+ tests.add(testId);
+
+ DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests);
+
+ deqpTest.setDevice(mockDevice);
+
+ int version = 3 << 16;
+ EasyMock.expect(mockDevice.getProperty("ro.opengles.version"))
+ .andReturn(Integer.toString(version)).atLeastOnce();
+
+ EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).
+ andReturn("").once();
+
+ EasyMock.expect(mockDevice.installPackage(EasyMock.<File>anyObject(),
+ EasyMock.eq(true),
+ EasyMock.eq(AbiUtils.createAbiFlag(ABI.getName())))).andReturn(null)
+ .once();
+
+ expectRenderConfigQueryAndReturn(mockDevice,
+ "--deqp-gl-config-name=rgba8888d24s8 "
+ + "--deqp-screen-rotation=unspecified "
+ + "--deqp-surface-type=window "
+ + "--deqp-gl-major-version=3 "
+ + "--deqp-gl-minor-version=0", "Maybe?");
+
+ EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG)))
+ .andReturn("").once();
+
+ mockListener.testRunStarted(getTestId(deqpTest), 1);
+ EasyMock.expectLastCall().once();
+
+ mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<Map<String, String>>notNull());
+ EasyMock.expectLastCall().once();
+
+ EasyMock.replay(mockDevice);
+ EasyMock.replay(mockListener);
+ deqpTest.run(mockListener);
+ EasyMock.verify(mockListener);
+ EasyMock.verify(mockDevice);
+ }
+
+ /**
+ * Test that orientation is supplied to runner correctly
+ */
+ private void testOrientation(final String rotation, final String featureString)
+ throws Exception {
+ final TestIdentifier testId = new TestIdentifier("dEQP-GLES3.orientation", "test");
+ final String testPath = "dEQP-GLES3.orientation.test";
+ final String testTrie = "{dEQP-GLES3{orientation{test}}}";
+ final String output = "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=" + testPath + "\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_CODE: 0\r\n";
+
+ ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class);
+ ITestInvocationListener mockListener
+ = EasyMock.createStrictMock(ITestInvocationListener.class);
+ IDevice mockIDevice = EasyMock.createMock(IDevice.class);
+
+ Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>();
+ tests.add(testId);
+
+ DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests);
+ OptionSetter setter = new OptionSetter(deqpTest);
+ setter.setOptionValue("deqp-screen-rotation", rotation);
+
+ deqpTest.setDevice(mockDevice);
+
+ int version = 3 << 16;
+ EasyMock.expect(mockDevice.getProperty("ro.opengles.version"))
+ .andReturn(Integer.toString(version)).atLeastOnce();
+
+ if (!rotation.equals(DeqpTestRunner.BatchRunConfiguration.ROTATION_UNSPECIFIED)) {
+ EasyMock.expect(mockDevice.executeShellCommand("pm list features"))
+ .andReturn(featureString);
+ }
+
+ final boolean isPortraitOrientation =
+ rotation.equals(DeqpTestRunner.BatchRunConfiguration.ROTATION_PORTRAIT) ||
+ rotation.equals(DeqpTestRunner.BatchRunConfiguration.ROTATION_REVERSE_PORTRAIT);
+ final boolean isLandscapeOrientation =
+ rotation.equals(DeqpTestRunner.BatchRunConfiguration.ROTATION_LANDSCAPE) ||
+ rotation.equals(DeqpTestRunner.BatchRunConfiguration.ROTATION_REVERSE_LANDSCAPE);
+ final boolean executable =
+ rotation.equals(DeqpTestRunner.BatchRunConfiguration.ROTATION_UNSPECIFIED) ||
+ (isPortraitOrientation &&
+ featureString.contains(DeqpTestRunner.FEATURE_PORTRAIT)) ||
+ (isLandscapeOrientation &&
+ featureString.contains(DeqpTestRunner.FEATURE_LANDSCAPE));
+
+ EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).
+ andReturn("").once();
+
+ EasyMock.expect(mockDevice.installPackage(EasyMock.<File>anyObject(),
+ EasyMock.eq(true),
+ EasyMock.eq(AbiUtils.createAbiFlag(ABI.getName())))).andReturn(null)
+ .once();
+
+ if (executable) {
+ expectRenderConfigQuery(mockDevice, String.format(
+ "--deqp-gl-config-name=rgba8888d24s8 --deqp-screen-rotation=%s "
+ + "--deqp-surface-type=window --deqp-gl-major-version=3 "
+ + "--deqp-gl-minor-version=0", rotation));
+
+ String commandLine = String.format(
+ "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 "
+ + "--deqp-screen-rotation=%s "
+ + "--deqp-surface-type=window "
+ + "--deqp-log-images=disable "
+ + "--deqp-watchdog=enable",
+ CASE_LIST_FILE_NAME, rotation);
+
+ runInstrumentationLineAndAnswer(mockDevice, mockIDevice, testTrie, commandLine,
+ output);
+ }
+
+ EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG)))
+ .andReturn("").once();
+
+ mockListener.testRunStarted(getTestId(deqpTest), 1);
+ EasyMock.expectLastCall().once();
+
+ mockListener.testStarted(EasyMock.eq(testId));
+ EasyMock.expectLastCall().once();
+
+ mockListener.testEnded(EasyMock.eq(testId), EasyMock.<Map<String, String>>notNull());
+ EasyMock.expectLastCall().once();
+
+ mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<Map<String, String>>notNull());
+ EasyMock.expectLastCall().once();
+
+ EasyMock.replay(mockDevice, mockIDevice);
+ EasyMock.replay(mockListener);
+ deqpTest.run(mockListener);
+ EasyMock.verify(mockListener);
+ EasyMock.verify(mockDevice, mockIDevice);
+ }
+
+ /**
+ * Test OpeGL ES3 tests on device with OpenGL ES2.
+ */
+ public void testRun_require30DeviceVersion20() throws Exception {
+ testGlesVersion(3, 0, 2, 0);
+ }
+
+ /**
+ * Test OpeGL ES3.1 tests on device with OpenGL ES2.
+ */
+ public void testRun_require31DeviceVersion20() throws Exception {
+ testGlesVersion(3, 1, 2, 0);
+ }
+
+ /**
+ * Test OpeGL ES3 tests on device with OpenGL ES3.
+ */
+ public void testRun_require30DeviceVersion30() throws Exception {
+ testGlesVersion(3, 0, 3, 0);
+ }
+
+ /**
+ * Test OpeGL ES3.1 tests on device with OpenGL ES3.
+ */
+ public void testRun_require31DeviceVersion30() throws Exception {
+ testGlesVersion(3, 1, 3, 0);
+ }
+
+ /**
+ * Test OpeGL ES3 tests on device with OpenGL ES3.1.
+ */
+ public void testRun_require30DeviceVersion31() throws Exception {
+ testGlesVersion(3, 0, 3, 1);
+ }
+
+ /**
+ * Test OpeGL ES3.1 tests on device with OpenGL ES3.1.
+ */
+ public void testRun_require31DeviceVersion31() throws Exception {
+ testGlesVersion(3, 1, 3, 1);
+ }
+
+ /**
+ * Test dEQP Pass result code.
+ */
+ public void testRun_resultPass() throws Exception {
+ testResultCode("Pass", true);
+ }
+
+ /**
+ * Test dEQP Fail result code.
+ */
+ public void testRun_resultFail() throws Exception {
+ testResultCode("Fail", false);
+ }
+
+ /**
+ * Test dEQP NotSupported result code.
+ */
+ public void testRun_resultNotSupported() throws Exception {
+ testResultCode("NotSupported", true);
+ }
+
+ /**
+ * Test dEQP QualityWarning result code.
+ */
+ public void testRun_resultQualityWarning() throws Exception {
+ testResultCode("QualityWarning", true);
+ }
+
+ /**
+ * Test dEQP CompatibilityWarning result code.
+ */
+ public void testRun_resultCompatibilityWarning() throws Exception {
+ testResultCode("CompatibilityWarning", true);
+ }
+
+ /**
+ * Test dEQP ResourceError result code.
+ */
+ public void testRun_resultResourceError() throws Exception {
+ testResultCode("ResourceError", false);
+ }
+
+ /**
+ * Test dEQP InternalError result code.
+ */
+ public void testRun_resultInternalError() throws Exception {
+ testResultCode("InternalError", false);
+ }
+
+ /**
+ * Test dEQP Crash result code.
+ */
+ public void testRun_resultCrash() throws Exception {
+ testResultCode("Crash", false);
+ }
+
+ /**
+ * Test dEQP Timeout result code.
+ */
+ public void testRun_resultTimeout() throws Exception {
+ testResultCode("Timeout", false);
+ }
+ /**
+ * Test dEQP Orientation
+ */
+ public void testRun_orientationLandscape() throws Exception {
+ testOrientation("90", ALL_FEATURES);
+ }
+
+ /**
+ * Test dEQP Orientation
+ */
+ public void testRun_orientationPortrait() throws Exception {
+ testOrientation("0", ALL_FEATURES);
+ }
+
+ /**
+ * Test dEQP Orientation
+ */
+ public void testRun_orientationReverseLandscape() throws Exception {
+ testOrientation("270", ALL_FEATURES);
+ }
+
+ /**
+ * Test dEQP Orientation
+ */
+ public void testRun_orientationReversePortrait() throws Exception {
+ testOrientation("180", ALL_FEATURES);
+ }
+
+ /**
+ * Test dEQP Orientation
+ */
+ public void testRun_orientationUnspecified() throws Exception {
+ testOrientation("unspecified", ALL_FEATURES);
+ }
+
+ /**
+ * Test dEQP Orientation with limited features
+ */
+ public void testRun_orientationUnspecifiedLimitedFeatures() throws Exception {
+ testOrientation("unspecified", ONLY_LANDSCAPE_FEATURES);
+ }
+
+ /**
+ * Test dEQP Orientation with limited features
+ */
+ public void testRun_orientationLandscapeLimitedFeatures() throws Exception {
+ testOrientation("90", ONLY_LANDSCAPE_FEATURES);
+ }
+
+ /**
+ * Test dEQP Orientation with limited features
+ */
+ public void testRun_orientationPortraitLimitedFeatures() throws Exception {
+ testOrientation("0", ONLY_LANDSCAPE_FEATURES);
+ }
+
+ /**
+ * Test dEQP unsupported pixel format
+ */
+ public void testRun_unsupportedPixelFormat() throws Exception {
+ final String pixelFormat = "rgba5658d16m4";
+ final TestIdentifier testId = new TestIdentifier("dEQP-GLES3.pixelformat", "test");
+ final String testPath = "dEQP-GLES3.pixelformat.test";
+ final String testTrie = "{dEQP-GLES3{pixelformat{test}}}";
+ final String output = "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=" + testPath + "\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_CODE: 0\r\n";
+
+ ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class);
+ ITestInvocationListener mockListener
+ = EasyMock.createStrictMock(ITestInvocationListener.class);
+
+ Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>();
+ tests.add(testId);
+
+ DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests);
+ OptionSetter setter = new OptionSetter(deqpTest);
+ setter.setOptionValue("deqp-gl-config-name", pixelFormat);
+
+ deqpTest.setDevice(mockDevice);
+
+ int version = 3 << 16;
+ EasyMock.expect(mockDevice.getProperty("ro.opengles.version"))
+ .andReturn(Integer.toString(version)).atLeastOnce();
+
+ EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).
+ andReturn("").once();
+
+ EasyMock.expect(mockDevice.installPackage(EasyMock.<File>anyObject(),
+ EasyMock.eq(true),
+ EasyMock.eq(AbiUtils.createAbiFlag(ABI.getName())))).andReturn(null)
+ .once();
+
+ expectRenderConfigQueryAndReturn(mockDevice, String.format(
+ "--deqp-gl-config-name=%s --deqp-screen-rotation=unspecified "
+ + "--deqp-surface-type=window "
+ + "--deqp-gl-major-version=3 "
+ + "--deqp-gl-minor-version=0", pixelFormat), "No");
+
+ EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG)))
+ .andReturn("").once();
+
+ mockListener.testRunStarted(getTestId(deqpTest), 1);
+ EasyMock.expectLastCall().once();
+
+ mockListener.testStarted(EasyMock.eq(testId));
+ EasyMock.expectLastCall().once();
+
+ mockListener.testEnded(EasyMock.eq(testId), EasyMock.<Map<String, String>>notNull());
+ EasyMock.expectLastCall().once();
+
+ mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<Map<String, String>>notNull());
+ EasyMock.expectLastCall().once();
+
+ EasyMock.replay(mockDevice);
+ EasyMock.replay(mockListener);
+ deqpTest.run(mockListener);
+ EasyMock.verify(mockListener);
+ EasyMock.verify(mockDevice);
+ }
+
+ public static interface RecoverableTestDevice extends ITestDevice {
+ public void recoverDevice() throws DeviceNotAvailableException;
+ }
+
+ private static enum RecoveryEvent {
+ PROGRESS,
+ FAIL_CONNECTION_REFUSED,
+ FAIL_LINK_KILLED,
+ };
+
+ private void runRecoveryWithPattern(DeqpTestRunner.Recovery recovery, RecoveryEvent[] events)
+ throws DeviceNotAvailableException {
+ for (RecoveryEvent event : events) {
+ switch (event) {
+ case PROGRESS:
+ recovery.onExecutionProgressed();
+ break;
+ case FAIL_CONNECTION_REFUSED:
+ recovery.recoverConnectionRefused();
+ break;
+ case FAIL_LINK_KILLED:
+ recovery.recoverComLinkKilled();
+ break;
+ }
+ }
+ }
+
+ private void setRecoveryExpectationWait(DeqpTestRunner.ISleepProvider mockSleepProvider) {
+ mockSleepProvider.sleep(EasyMock.gt(0));
+ EasyMock.expectLastCall().once();
+ }
+
+ private void setRecoveryExpectationKillProcess(RecoverableTestDevice mockDevice,
+ DeqpTestRunner.ISleepProvider mockSleepProvider) throws DeviceNotAvailableException {
+ EasyMock.expect(mockDevice.executeShellCommand(EasyMock.contains("ps"))).
+ andReturn("root 1234 com.drawelement.deqp").once();
+
+ EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("kill -9 1234"))).
+ andReturn("").once();
+
+ // Recovery checks if kill failed
+ mockSleepProvider.sleep(EasyMock.gt(0));
+ EasyMock.expectLastCall().once();
+ EasyMock.expect(mockDevice.executeShellCommand(EasyMock.contains("ps"))).
+ andReturn("").once();
+ }
+
+ private void setRecoveryExpectationRecovery(RecoverableTestDevice mockDevice)
+ throws DeviceNotAvailableException {
+ mockDevice.recoverDevice();
+ EasyMock.expectLastCall().once();
+ }
+
+ private void setRecoveryExpectationReboot(RecoverableTestDevice mockDevice)
+ throws DeviceNotAvailableException {
+ mockDevice.reboot();
+ EasyMock.expectLastCall().once();
+ }
+
+ private int setRecoveryExpectationOfAConnFailure(RecoverableTestDevice mockDevice,
+ DeqpTestRunner.ISleepProvider mockSleepProvider, int numConsecutiveErrors)
+ throws DeviceNotAvailableException {
+ switch (numConsecutiveErrors) {
+ case 0:
+ case 1:
+ setRecoveryExpectationRecovery(mockDevice);
+ return 2;
+ case 2:
+ setRecoveryExpectationReboot(mockDevice);
+ return 3;
+ default:
+ return 4;
+ }
+ }
+
+ private int setRecoveryExpectationOfAComKilled(RecoverableTestDevice mockDevice,
+ DeqpTestRunner.ISleepProvider mockSleepProvider, int numConsecutiveErrors)
+ throws DeviceNotAvailableException {
+ switch (numConsecutiveErrors) {
+ case 0:
+ setRecoveryExpectationWait(mockSleepProvider);
+ setRecoveryExpectationKillProcess(mockDevice, mockSleepProvider);
+ return 1;
+ case 1:
+ setRecoveryExpectationRecovery(mockDevice);
+ setRecoveryExpectationKillProcess(mockDevice, mockSleepProvider);
+ return 2;
+ case 2:
+ setRecoveryExpectationReboot(mockDevice);
+ return 3;
+ default:
+ return 4;
+ }
+ }
+
+ private void setRecoveryExpectationsOfAPattern(RecoverableTestDevice mockDevice,
+ DeqpTestRunner.ISleepProvider mockSleepProvider, RecoveryEvent[] events)
+ throws DeviceNotAvailableException {
+ int numConsecutiveErrors = 0;
+ for (RecoveryEvent event : events) {
+ switch (event) {
+ case PROGRESS:
+ numConsecutiveErrors = 0;
+ break;
+ case FAIL_CONNECTION_REFUSED:
+ numConsecutiveErrors = setRecoveryExpectationOfAConnFailure(mockDevice,
+ mockSleepProvider, numConsecutiveErrors);
+ break;
+ case FAIL_LINK_KILLED:
+ numConsecutiveErrors = setRecoveryExpectationOfAComKilled(mockDevice,
+ mockSleepProvider, numConsecutiveErrors);
+ break;
+ }
+ }
+ }
+
+ /**
+ * Test dEQP runner recovery state machine.
+ */
+ private void testRecoveryWithPattern(boolean expectSuccess, RecoveryEvent...pattern)
+ throws Exception {
+ DeqpTestRunner.Recovery recovery = new DeqpTestRunner.Recovery();
+ IMocksControl orderedControl = EasyMock.createStrictControl();
+ RecoverableTestDevice mockDevice = orderedControl.createMock(RecoverableTestDevice.class);
+ DeqpTestRunner.ISleepProvider mockSleepProvider =
+ orderedControl.createMock(DeqpTestRunner.ISleepProvider.class);
+
+ setRecoveryExpectationsOfAPattern(mockDevice, mockSleepProvider, pattern);
+
+ orderedControl.replay();
+
+ recovery.setDevice(mockDevice);
+ recovery.setSleepProvider(mockSleepProvider);
+ try {
+ runRecoveryWithPattern(recovery, pattern);
+ if (!expectSuccess) {
+ fail("Expected DeviceNotAvailableException");
+ }
+ } catch (DeviceNotAvailableException ex) {
+ if (expectSuccess) {
+ fail("Did not expect DeviceNotAvailableException");
+ }
+ }
+
+ orderedControl.verify();
+ }
+
+ // basic patterns
+
+ public void testRecovery_NoEvents() throws Exception {
+ testRecoveryWithPattern(true);
+ }
+
+ public void testRecovery_AllOk() throws Exception {
+ testRecoveryWithPattern(true, RecoveryEvent.PROGRESS, RecoveryEvent.PROGRESS);
+ }
+
+ // conn fail patterns
+
+ public void testRecovery_OneConnectionFailureBegin() throws Exception {
+ testRecoveryWithPattern(true, RecoveryEvent.FAIL_CONNECTION_REFUSED,
+ RecoveryEvent.PROGRESS);
+ }
+
+ public void testRecovery_TwoConnectionFailuresBegin() throws Exception {
+ testRecoveryWithPattern(true, RecoveryEvent.FAIL_CONNECTION_REFUSED,
+ RecoveryEvent.FAIL_CONNECTION_REFUSED, RecoveryEvent.PROGRESS);
+ }
+
+ public void testRecovery_ThreeConnectionFailuresBegin() throws Exception {
+ testRecoveryWithPattern(false, RecoveryEvent.FAIL_CONNECTION_REFUSED,
+ RecoveryEvent.FAIL_CONNECTION_REFUSED, RecoveryEvent.FAIL_CONNECTION_REFUSED);
+ }
+
+ public void testRecovery_OneConnectionFailureMid() throws Exception {
+ testRecoveryWithPattern(true, RecoveryEvent.PROGRESS,
+ RecoveryEvent.FAIL_CONNECTION_REFUSED, RecoveryEvent.PROGRESS);
+ }
+
+ public void testRecovery_TwoConnectionFailuresMid() throws Exception {
+ testRecoveryWithPattern(true, RecoveryEvent.PROGRESS,
+ RecoveryEvent.FAIL_CONNECTION_REFUSED, RecoveryEvent.FAIL_CONNECTION_REFUSED,
+ RecoveryEvent.PROGRESS);
+ }
+
+ public void testRecovery_ThreeConnectionFailuresMid() throws Exception {
+ testRecoveryWithPattern(false, RecoveryEvent.PROGRESS,
+ RecoveryEvent.FAIL_CONNECTION_REFUSED, RecoveryEvent.FAIL_CONNECTION_REFUSED,
+ RecoveryEvent.FAIL_CONNECTION_REFUSED);
+ }
+
+ // link fail patterns
+
+ public void testRecovery_OneLinkFailureBegin() throws Exception {
+ testRecoveryWithPattern(true, RecoveryEvent.FAIL_LINK_KILLED,
+ RecoveryEvent.PROGRESS);
+ }
+
+ public void testRecovery_TwoLinkFailuresBegin() throws Exception {
+ testRecoveryWithPattern(true, RecoveryEvent.FAIL_LINK_KILLED,
+ RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.PROGRESS);
+ }
+
+ public void testRecovery_ThreeLinkFailuresBegin() throws Exception {
+ testRecoveryWithPattern(true, RecoveryEvent.FAIL_LINK_KILLED,
+ RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.FAIL_LINK_KILLED,
+ RecoveryEvent.PROGRESS);
+ }
+
+ public void testRecovery_FourLinkFailuresBegin() throws Exception {
+ testRecoveryWithPattern(false, RecoveryEvent.FAIL_LINK_KILLED,
+ RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.FAIL_LINK_KILLED,
+ RecoveryEvent.FAIL_LINK_KILLED);
+ }
+
+ public void testRecovery_OneLinkFailureMid() throws Exception {
+ testRecoveryWithPattern(true, RecoveryEvent.PROGRESS,
+ RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.PROGRESS);
+ }
+
+ public void testRecovery_TwoLinkFailuresMid() throws Exception {
+ testRecoveryWithPattern(true, RecoveryEvent.PROGRESS,
+ RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.FAIL_LINK_KILLED,
+ RecoveryEvent.PROGRESS);
+ }
+
+ public void testRecovery_ThreeLinkFailuresMid() throws Exception {
+ testRecoveryWithPattern(true, RecoveryEvent.PROGRESS,
+ RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.FAIL_LINK_KILLED,
+ RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.PROGRESS);
+ }
+
+ public void testRecovery_FourLinkFailuresMid() throws Exception {
+ testRecoveryWithPattern(false, RecoveryEvent.PROGRESS, RecoveryEvent.FAIL_LINK_KILLED,
+ RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.FAIL_LINK_KILLED,
+ RecoveryEvent.FAIL_LINK_KILLED);
+ }
+
+ // mixed patterns
+
+ public void testRecovery_MixedFailuresProgressBetween() throws Exception {
+ testRecoveryWithPattern(true,
+ RecoveryEvent.PROGRESS, RecoveryEvent.FAIL_LINK_KILLED,
+ RecoveryEvent.PROGRESS, RecoveryEvent.FAIL_CONNECTION_REFUSED,
+ RecoveryEvent.PROGRESS, RecoveryEvent.FAIL_LINK_KILLED,
+ RecoveryEvent.PROGRESS, RecoveryEvent.FAIL_CONNECTION_REFUSED,
+ RecoveryEvent.PROGRESS);
+ }
+
+ public void testRecovery_MixedFailuresNoProgressBetween() throws Exception {
+ testRecoveryWithPattern(true,
+ RecoveryEvent.PROGRESS, RecoveryEvent.FAIL_LINK_KILLED,
+ RecoveryEvent.FAIL_CONNECTION_REFUSED, RecoveryEvent.FAIL_LINK_KILLED,
+ RecoveryEvent.PROGRESS);
+ }
+
+ /**
+ * Test recovery if process cannot be killed
+ */
+ public void testRecovery_unkillableProcess () throws Exception {
+ DeqpTestRunner.Recovery recovery = new DeqpTestRunner.Recovery();
+ IMocksControl orderedControl = EasyMock.createStrictControl();
+ RecoverableTestDevice mockDevice = orderedControl.createMock(RecoverableTestDevice.class);
+ DeqpTestRunner.ISleepProvider mockSleepProvider =
+ orderedControl.createMock(DeqpTestRunner.ISleepProvider.class);
+
+ // recovery attempts to kill the process after a timeout
+ mockSleepProvider.sleep(EasyMock.gt(0));
+ EasyMock.expect(mockDevice.executeShellCommand(EasyMock.contains("ps"))).
+ andReturn("root 1234 com.drawelement.deqp").once();
+ EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("kill -9 1234"))).
+ andReturn("").once();
+
+ // Recovery checks if kill failed
+ mockSleepProvider.sleep(EasyMock.gt(0));
+ EasyMock.expectLastCall().once();
+ EasyMock.expect(mockDevice.executeShellCommand(EasyMock.contains("ps"))).
+ andReturn("root 1234 com.drawelement.deqp").once();
+
+ // Recovery resets the connection
+ mockDevice.recoverDevice();
+ EasyMock.expectLastCall().once();
+
+ // and attempts to kill the process again
+ EasyMock.expect(mockDevice.executeShellCommand(EasyMock.contains("ps"))).
+ andReturn("root 1234 com.drawelement.deqp").once();
+ EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("kill -9 1234"))).
+ andReturn("").once();
+
+ // Recovery checks if kill failed
+ mockSleepProvider.sleep(EasyMock.gt(0));
+ EasyMock.expectLastCall().once();
+ EasyMock.expect(mockDevice.executeShellCommand(EasyMock.contains("ps"))).
+ andReturn("root 1234 com.drawelement.deqp").once();
+
+ // recovery reboots the device
+ mockDevice.reboot();
+ EasyMock.expectLastCall().once();
+
+ orderedControl.replay();
+ recovery.setDevice(mockDevice);
+ recovery.setSleepProvider(mockSleepProvider);
+ recovery.recoverComLinkKilled();
+ orderedControl.verify();
+ }
+
+ /**
+ * Test external interruption before batch run.
+ */
+ public void testInterrupt_killBeforeBatch() throws Exception {
+ final TestIdentifier testId = new TestIdentifier("dEQP-GLES3.interrupt", "test");
+
+ Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>();
+ tests.add(testId);
+
+ ITestInvocationListener mockListener
+ = EasyMock.createStrictMock(ITestInvocationListener.class);
+ ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class);
+ IDevice mockIDevice = EasyMock.createMock(IDevice.class);
+ IRunUtil mockRunUtil = EasyMock.createMock(IRunUtil.class);
+
+ DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests);
+
+ deqpTest.setDevice(mockDevice);
+ deqpTest.setRunUtil(mockRunUtil);
+
+ int version = 3 << 16;
+ EasyMock.expect(mockDevice.getProperty("ro.opengles.version"))
+ .andReturn(Integer.toString(version)).atLeastOnce();
+
+ EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).
+ andReturn("").once();
+
+ EasyMock.expect(mockDevice.installPackage(EasyMock.<File>anyObject(),
+ EasyMock.eq(true),
+ EasyMock.eq(AbiUtils.createAbiFlag(ABI.getName())))).andReturn(null)
+ .once();
+
+ expectRenderConfigQuery(mockDevice,
+ "--deqp-gl-config-name=rgba8888d24s8 --deqp-screen-rotation=unspecified "
+ + "--deqp-surface-type=window --deqp-gl-major-version=3 "
+ + "--deqp-gl-minor-version=0");
+
+ mockRunUtil.sleep(0);
+ EasyMock.expectLastCall().andThrow(new RunInterruptedException());
+
+ mockListener.testRunStarted(getTestId(deqpTest), 1);
+ EasyMock.expectLastCall().once();
+
+ EasyMock.replay(mockDevice, mockIDevice);
+ EasyMock.replay(mockListener);
+ EasyMock.replay(mockRunUtil);
+ try {
+ deqpTest.run(mockListener);
+ fail("expected RunInterruptedException");
+ } catch (RunInterruptedException ex) {
+ // expected
+ }
+ EasyMock.verify(mockRunUtil);
+ EasyMock.verify(mockListener);
+ EasyMock.verify(mockDevice, mockIDevice);
+ }
+
+ /**
+ * Test external interruption in testFailed().
+ */
+ public void testInterrupt_killReportTestFailed() throws Exception {
+ final TestIdentifier testId = new TestIdentifier("dEQP-GLES3.interrupt", "test");
+ final String testPath = "dEQP-GLES3.interrupt.test";
+ final String testTrie = "{dEQP-GLES3{interrupt{test}}}";
+ final String output = "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=" + testPath + "\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Fail\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Fail\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_CODE: 0\r\n";
+
+ Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>();
+ tests.add(testId);
+
+ ITestInvocationListener mockListener
+ = EasyMock.createStrictMock(ITestInvocationListener.class);
+ ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class);
+ IDevice mockIDevice = EasyMock.createMock(IDevice.class);
+ IRunUtil mockRunUtil = EasyMock.createMock(IRunUtil.class);
+
+ DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests);
+
+ deqpTest.setDevice(mockDevice);
+ deqpTest.setRunUtil(mockRunUtil);
+
+ int version = 3 << 16;
+ EasyMock.expect(mockDevice.getProperty("ro.opengles.version"))
+ .andReturn(Integer.toString(version)).atLeastOnce();
+
+ EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).
+ andReturn("").once();
+
+ EasyMock.expect(mockDevice.installPackage(EasyMock.<File>anyObject(),
+ EasyMock.eq(true),
+ EasyMock.eq(AbiUtils.createAbiFlag(ABI.getName())))).andReturn(null)
+ .once();
+
+ expectRenderConfigQuery(mockDevice,
+ "--deqp-gl-config-name=rgba8888d24s8 --deqp-screen-rotation=unspecified "
+ + "--deqp-surface-type=window --deqp-gl-major-version=3 "
+ + "--deqp-gl-minor-version=0");
+
+ mockRunUtil.sleep(0);
+ EasyMock.expectLastCall().once();
+
+ String commandLine = String.format(
+ "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 "
+ + "--deqp-screen-rotation=unspecified "
+ + "--deqp-surface-type=window "
+ + "--deqp-log-images=disable "
+ + "--deqp-watchdog=enable",
+ CASE_LIST_FILE_NAME);
+
+ runInstrumentationLineAndAnswer(mockDevice, mockIDevice, testTrie, commandLine,
+ output);
+
+ mockListener.testRunStarted(getTestId(deqpTest), 1);
+ EasyMock.expectLastCall().once();
+
+ mockListener.testStarted(EasyMock.eq(testId));
+ EasyMock.expectLastCall().once();
+
+ mockListener.testFailed(EasyMock.eq(testId), EasyMock.<String>notNull());
+ EasyMock.expectLastCall().andThrow(new RunInterruptedException());
+
+ EasyMock.replay(mockDevice, mockIDevice);
+ EasyMock.replay(mockListener);
+ EasyMock.replay(mockRunUtil);
+ try {
+ deqpTest.run(mockListener);
+ fail("expected RunInterruptedException");
+ } catch (RunInterruptedException ex) {
+ // expected
+ }
+ EasyMock.verify(mockRunUtil);
+ EasyMock.verify(mockListener);
+ EasyMock.verify(mockDevice, mockIDevice);
+ }
+
+ private void runInstrumentationLineAndAnswer(ITestDevice mockDevice, IDevice mockIDevice,
+ final String testTrie, final String cmd, final String output) throws Exception {
+ EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("rm " + CASE_LIST_FILE_NAME)))
+ .andReturn("").once();
+
+ EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("rm " + LOG_FILE_NAME)))
+ .andReturn("").once();
+
+ EasyMock.expect(mockDevice.pushString(testTrie + "\n", CASE_LIST_FILE_NAME))
+ .andReturn(true).once();
+
+ String command = String.format(
+ "am instrument %s -w -e deqpLogFileName \"%s\" -e deqpCmdLine \"%s\" "
+ + "-e deqpLogData \"%s\" %s",
+ AbiUtils.createAbiFlag(ABI.getName()), LOG_FILE_NAME, cmd, false,
+ INSTRUMENTATION_NAME);
+
+ EasyMock.expect(mockDevice.getIDevice()).andReturn(mockIDevice);
+ mockIDevice.executeShellCommand(EasyMock.eq(command),
+ EasyMock.<IShellOutputReceiver>notNull(), EasyMock.anyLong(),
+ EasyMock.isA(TimeUnit.class));
+
+ EasyMock.expectLastCall().andAnswer(new IAnswer<Object>() {
+ @Override
+ public Object answer() {
+ IShellOutputReceiver receiver
+ = (IShellOutputReceiver)EasyMock.getCurrentArguments()[1];
+
+ receiver.addOutput(output.getBytes(), 0, output.length());
+ receiver.flush();
+
+ return null;
+ }
+ });
+ }
+}
diff --git a/android/package/Android.mk b/android/package/Android.mk
index d4015a3..555a4ef 100644
--- a/android/package/Android.mk
+++ b/android/package/Android.mk
@@ -1,8 +1,30 @@
+# Copyright (C) 2014-2015 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
LOCAL_PATH := $(call my-dir)
+
include $(CLEAR_VARS)
+# don't include this package in any target ??????
+LOCAL_MODULE_TAGS := optional
+# and when built explicitly put it in the data partition
+LOCAL_MODULE_PATH := $(TARGET_OUT_DATA_APPS)
+
LOCAL_MODULE_TAGS := tests
+LOCAL_COMPATIBILITY_SUITE := cts_v2
+
LOCAL_SRC_FILES := $(call all-java-files-under,src)
LOCAL_JNI_SHARED_LIBRARIES := libdeqp
@@ -10,4 +32,7 @@
LOCAL_PACKAGE_NAME := com.drawelements.deqp
LOCAL_MULTILIB := both
+# We could go down all the way to API-13 for 32bit. 22 is required for 64bit ARM.
+LOCAL_SDK_VERSION := 22
+
include $(BUILD_PACKAGE)
diff --git a/scripts/build_android_mustpass.py b/scripts/build_android_mustpass.py
index 285f9d1..c991ced 100644
--- a/scripts/build_android_mustpass.py
+++ b/scripts/build_android_mustpass.py
@@ -31,6 +31,27 @@
import xml.dom.minidom as minidom
CTS_DATA_DIR = os.path.join(DEQP_DIR, "android", "cts")
+APK_NAME = "com.drawelements.deqp.apk"
+
+COPYRIGHT_DECLARATION = """
+ Copyright (C) 2015 The Android Open Source Project
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ """
+
+GENERATED_FILE_WARNING = """
+ This file has been automatically generated. Edit with caution.
+ """
class Configuration:
def __init__ (self, name, glconfig, rotation, surfacetype, filters):
@@ -248,6 +269,8 @@
return Filter(Filter.TYPE_EXCLUDE, filename)
def prettifyXML (doc):
+ doc.insert(0, ElementTree.Comment(COPYRIGHT_DECLARATION))
+ doc.insert(1, ElementTree.Comment(GENERATED_FILE_WARNING))
uglyString = ElementTree.tostring(doc, 'utf-8')
reparsed = minidom.parseString(uglyString)
return reparsed.toprettyxml(indent='\t', encoding='utf-8')
@@ -315,6 +338,32 @@
return mustpassElem
+def addOptionElement (parent, optionName, optionValue):
+ ElementTree.SubElement(parent, "option", name=optionName, value=optionValue)
+
+def genAndroidTestXml (mustpass):
+ INSTALLER_CLASS = "com.android.compatibility.common.tradefed.targetprep.ApkInstaller"
+ RUNNER_CLASS = "com.drawelements.deqp.runner.DeqpTestRunner"
+ configElement = ElementTree.Element("configuration")
+ preparerElement = ElementTree.SubElement(configElement, "target_preparer")
+ preparerElement.set("class", INSTALLER_CLASS)
+ addOptionElement(preparerElement, "cleanup-apks", "true")
+ addOptionElement(preparerElement, "test-file-name", APK_NAME)
+
+ for package in mustpass.packages:
+ for config in package.configurations:
+ testElement = ElementTree.SubElement(configElement, "test")
+ testElement.set("class", RUNNER_CLASS)
+ addOptionElement(testElement, "deqp-package", package.module.name)
+ addOptionElement(testElement, "deqp-caselist-file", getCaseListFileName(package,config))
+ # \todo [2015-10-16 kalle]: Replace with just command line? - requires simplifications in the runner/tests as well.
+ addOptionElement(testElement, "deqp-gl-config-name", config.glconfig)
+ addOptionElement(testElement, "deqp-surface-type", config.surfacetype)
+ addOptionElement(testElement, "deqp-screen-rotation", config.rotation)
+
+ return configElement
+
+
def genMustpass (mustpass, moduleCaseLists):
print "Generating mustpass '%s'" % mustpass.version
@@ -343,6 +392,7 @@
for case in matchingByConfig[config]:
testCaseMap[case].configurations.append(config)
+ # NOTE: CTS v2 does not need package XML files. Remove when transition is complete.
packageXml = genCTSPackageXML(package, root)
xmlFilename = os.path.join(CTS_DATA_DIR, mustpass.version, getCTSPackageName(package) + ".xml")
@@ -355,6 +405,14 @@
print " Writing spec: " + specFilename
writeFile(specFilename, prettifyXML(specXML))
+ # TODO: Which is the best selector mechanism?
+ if (mustpass.version == "mnc"):
+ androidTestXML = genAndroidTestXml(mustpass)
+ androidTestFilename = os.path.join(CTS_DATA_DIR, "AndroidTest.xml")
+
+ print " Writing AndroidTest.xml: " + androidTestFilename
+ writeFile(androidTestFilename, prettifyXML(androidTestXML))
+
print "Done!"
def genMustpassLists (mustpassLists, generator, buildCfg):