Merge "CTS v2 integration for deqp"
am: 6f483cc3fa
* commit '6f483cc3fa820e58ed9f83c83bdf8d213293b3ad':
CTS v2 integration for deqp
diff --git a/Android.mk b/Android.mk
index e4c6997..038f891 100644
--- a/Android.mk
+++ b/Android.mk
@@ -746,4 +746,8 @@
include $(BUILD_SHARED_LIBRARY)
-include $(LOCAL_PATH)/android/package/Android.mk
+
+# Build the test APKs using their own makefiles
+# include $(call all-makefiles-under,$(LOCAL_PATH)/android)
+
+include $(LOCAL_PATH)/android/package/Android.mk $(LOCAL_PATH)/android/cts/Android.mk
diff --git a/android/cts/Android.mk b/android/cts/Android.mk
new file mode 100644
index 0000000..b3f167a
--- /dev/null
+++ b/android/cts/Android.mk
@@ -0,0 +1,40 @@
+# Copyright (C) 2015 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+LOCAL_PATH := $(call my-dir)
+
+include $(CLEAR_VARS)
+
+LOCAL_MODULE := CtsDeqpTestCases
+
+LOCAL_MODULE_TAGS := optional
+
+# Tag this module as a cts_v2 test artifact
+LOCAL_COMPATIBILITY_SUITE := cts_v2
+
+LOCAL_SDK_VERSION := 22
+
+LOCAL_SRC_FILES := $(call all-java-files-under, runner/src)
+LOCAL_JAVA_LIBRARIES := cts-tradefed_v2 compatibility-host-util tradefed-prebuilt
+
+DEQP_CASELISTS:=$(sort $(patsubst mnc/%,%, \
+ $(shell cd $(LOCAL_PATH) ; \
+ find -L mnc -maxdepth 1 -name "*.txt") \
+ ))
+LOCAL_COMPATIBILITY_SUPPORT_FILES := $(foreach file, $(DEQP_CASELISTS), ./mnc/$(file):$(file))
+
+
+include $(BUILD_HOST_JAVA_LIBRARY)
+
+include $(call all-makefiles-under,$(LOCAL_PATH))
diff --git a/android/cts/AndroidTest.xml b/android/cts/AndroidTest.xml
new file mode 100644
index 0000000..c61f982
--- /dev/null
+++ b/android/cts/AndroidTest.xml
@@ -0,0 +1,137 @@
+<?xml version="1.0" encoding="utf-8"?>
+<configuration>
+ <!--
+ Copyright (C) 2015 The Android Open Source Project
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+ <!--
+ This file has been automatically generated. Edit with caution.
+ -->
+ <target_preparer class="com.android.compatibility.common.tradefed.targetprep.ApkInstaller">
+ <option name="cleanup-apks" value="true"/>
+ <option name="test-file-name" value="com.drawelements.deqp.apk"/>
+ </target_preparer>
+ <test class="com.drawelements.deqp.runner.DeqpTestRunner">
+ <option name="deqp-package" value="dEQP-EGL"/>
+ <option name="deqp-caselist-file" value="egl-master.txt"/>
+ <option name="deqp-gl-config-name" value="rgba8888d24s8ms0"/>
+ <option name="deqp-surface-type" value="window"/>
+ <option name="deqp-screen-rotation" value="unspecified"/>
+ </test>
+ <test class="com.drawelements.deqp.runner.DeqpTestRunner">
+ <option name="deqp-package" value="dEQP-GLES2"/>
+ <option name="deqp-caselist-file" value="gles2-master.txt"/>
+ <option name="deqp-gl-config-name" value="rgba8888d24s8ms0"/>
+ <option name="deqp-surface-type" value="window"/>
+ <option name="deqp-screen-rotation" value="unspecified"/>
+ </test>
+ <test class="com.drawelements.deqp.runner.DeqpTestRunner">
+ <option name="deqp-package" value="dEQP-GLES3"/>
+ <option name="deqp-caselist-file" value="gles3-master.txt"/>
+ <option name="deqp-gl-config-name" value="rgba8888d24s8ms0"/>
+ <option name="deqp-surface-type" value="window"/>
+ <option name="deqp-screen-rotation" value="unspecified"/>
+ </test>
+ <test class="com.drawelements.deqp.runner.DeqpTestRunner">
+ <option name="deqp-package" value="dEQP-GLES3"/>
+ <option name="deqp-caselist-file" value="gles3-rotate-portrait.txt"/>
+ <option name="deqp-gl-config-name" value="rgba8888d24s8ms0"/>
+ <option name="deqp-surface-type" value="window"/>
+ <option name="deqp-screen-rotation" value="0"/>
+ </test>
+ <test class="com.drawelements.deqp.runner.DeqpTestRunner">
+ <option name="deqp-package" value="dEQP-GLES3"/>
+ <option name="deqp-caselist-file" value="gles3-rotate-landscape.txt"/>
+ <option name="deqp-gl-config-name" value="rgba8888d24s8ms0"/>
+ <option name="deqp-surface-type" value="window"/>
+ <option name="deqp-screen-rotation" value="90"/>
+ </test>
+ <test class="com.drawelements.deqp.runner.DeqpTestRunner">
+ <option name="deqp-package" value="dEQP-GLES3"/>
+ <option name="deqp-caselist-file" value="gles3-rotate-reverse-portrait.txt"/>
+ <option name="deqp-gl-config-name" value="rgba8888d24s8ms0"/>
+ <option name="deqp-surface-type" value="window"/>
+ <option name="deqp-screen-rotation" value="180"/>
+ </test>
+ <test class="com.drawelements.deqp.runner.DeqpTestRunner">
+ <option name="deqp-package" value="dEQP-GLES3"/>
+ <option name="deqp-caselist-file" value="gles3-rotate-reverse-landscape.txt"/>
+ <option name="deqp-gl-config-name" value="rgba8888d24s8ms0"/>
+ <option name="deqp-surface-type" value="window"/>
+ <option name="deqp-screen-rotation" value="270"/>
+ </test>
+ <test class="com.drawelements.deqp.runner.DeqpTestRunner">
+ <option name="deqp-package" value="dEQP-GLES3"/>
+ <option name="deqp-caselist-file" value="gles3-multisample.txt"/>
+ <option name="deqp-gl-config-name" value="rgba8888d24s8ms4"/>
+ <option name="deqp-surface-type" value="window"/>
+ <option name="deqp-screen-rotation" value="unspecified"/>
+ </test>
+ <test class="com.drawelements.deqp.runner.DeqpTestRunner">
+ <option name="deqp-package" value="dEQP-GLES3"/>
+ <option name="deqp-caselist-file" value="gles3-565-no-depth-no-stencil.txt"/>
+ <option name="deqp-gl-config-name" value="rgb565d0s0ms0"/>
+ <option name="deqp-surface-type" value="window"/>
+ <option name="deqp-screen-rotation" value="unspecified"/>
+ </test>
+ <test class="com.drawelements.deqp.runner.DeqpTestRunner">
+ <option name="deqp-package" value="dEQP-GLES31"/>
+ <option name="deqp-caselist-file" value="gles31-master.txt"/>
+ <option name="deqp-gl-config-name" value="rgba8888d24s8ms0"/>
+ <option name="deqp-surface-type" value="window"/>
+ <option name="deqp-screen-rotation" value="unspecified"/>
+ </test>
+ <test class="com.drawelements.deqp.runner.DeqpTestRunner">
+ <option name="deqp-package" value="dEQP-GLES31"/>
+ <option name="deqp-caselist-file" value="gles31-rotate-portrait.txt"/>
+ <option name="deqp-gl-config-name" value="rgba8888d24s8ms0"/>
+ <option name="deqp-surface-type" value="window"/>
+ <option name="deqp-screen-rotation" value="0"/>
+ </test>
+ <test class="com.drawelements.deqp.runner.DeqpTestRunner">
+ <option name="deqp-package" value="dEQP-GLES31"/>
+ <option name="deqp-caselist-file" value="gles31-rotate-landscape.txt"/>
+ <option name="deqp-gl-config-name" value="rgba8888d24s8ms0"/>
+ <option name="deqp-surface-type" value="window"/>
+ <option name="deqp-screen-rotation" value="90"/>
+ </test>
+ <test class="com.drawelements.deqp.runner.DeqpTestRunner">
+ <option name="deqp-package" value="dEQP-GLES31"/>
+ <option name="deqp-caselist-file" value="gles31-rotate-reverse-portrait.txt"/>
+ <option name="deqp-gl-config-name" value="rgba8888d24s8ms0"/>
+ <option name="deqp-surface-type" value="window"/>
+ <option name="deqp-screen-rotation" value="180"/>
+ </test>
+ <test class="com.drawelements.deqp.runner.DeqpTestRunner">
+ <option name="deqp-package" value="dEQP-GLES31"/>
+ <option name="deqp-caselist-file" value="gles31-rotate-reverse-landscape.txt"/>
+ <option name="deqp-gl-config-name" value="rgba8888d24s8ms0"/>
+ <option name="deqp-surface-type" value="window"/>
+ <option name="deqp-screen-rotation" value="270"/>
+ </test>
+ <test class="com.drawelements.deqp.runner.DeqpTestRunner">
+ <option name="deqp-package" value="dEQP-GLES31"/>
+ <option name="deqp-caselist-file" value="gles31-multisample.txt"/>
+ <option name="deqp-gl-config-name" value="rgba8888d24s8ms4"/>
+ <option name="deqp-surface-type" value="window"/>
+ <option name="deqp-screen-rotation" value="unspecified"/>
+ </test>
+ <test class="com.drawelements.deqp.runner.DeqpTestRunner">
+ <option name="deqp-package" value="dEQP-GLES31"/>
+ <option name="deqp-caselist-file" value="gles31-565-no-depth-no-stencil.txt"/>
+ <option name="deqp-gl-config-name" value="rgb565d0s0ms0"/>
+ <option name="deqp-surface-type" value="window"/>
+ <option name="deqp-screen-rotation" value="unspecified"/>
+ </test>
+</configuration>
diff --git a/android/cts/lmp-mr1/com.drawelements.deqp.gles3.xml b/android/cts/lmp-mr1/com.drawelements.deqp.gles3.xml
index be78487..b41f4e4 100644
--- a/android/cts/lmp-mr1/com.drawelements.deqp.gles3.xml
+++ b/android/cts/lmp-mr1/com.drawelements.deqp.gles3.xml
@@ -1,5 +1,23 @@
<?xml version="1.0" encoding="utf-8"?>
<TestPackage appPackageName="com.drawelements.deqp.gles3" deqp:glesVersion="196608" name="dEQP-GLES3" testType="deqpTest" xmlns:deqp="http://drawelements.com/deqp">
+ <!--
+ Copyright (C) 2015 The Android Open Source Project
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+ <!--
+ This file has been automatically generated. Edit with caution.
+ -->
<TestSuite name="dEQP-GLES3">
<TestCase name="info">
<Test name="vendor">
diff --git a/android/cts/lmp-mr1/com.drawelements.deqp.gles31.xml b/android/cts/lmp-mr1/com.drawelements.deqp.gles31.xml
index 3af235b..985279b 100644
--- a/android/cts/lmp-mr1/com.drawelements.deqp.gles31.xml
+++ b/android/cts/lmp-mr1/com.drawelements.deqp.gles31.xml
@@ -1,5 +1,23 @@
<?xml version="1.0" encoding="utf-8"?>
<TestPackage appPackageName="com.drawelements.deqp.gles31" deqp:glesVersion="196609" name="dEQP-GLES31" testType="deqpTest" xmlns:deqp="http://drawelements.com/deqp">
+ <!--
+ Copyright (C) 2015 The Android Open Source Project
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+ <!--
+ This file has been automatically generated. Edit with caution.
+ -->
<TestSuite name="dEQP-GLES31">
<TestCase name="info">
<Test name="vendor">
diff --git a/android/cts/lmp-mr1/mustpass.xml b/android/cts/lmp-mr1/mustpass.xml
index bae28d3..76e34cc 100644
--- a/android/cts/lmp-mr1/mustpass.xml
+++ b/android/cts/lmp-mr1/mustpass.xml
@@ -1,5 +1,23 @@
<?xml version="1.0" encoding="utf-8"?>
<Mustpass version="lmp-mr1">
+ <!--
+ Copyright (C) 2015 The Android Open Source Project
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+ <!--
+ This file has been automatically generated. Edit with caution.
+ -->
<TestPackage name="dEQP-GLES3">
<Configuration caseListFile="gles3-master.txt" commandLine="--deqp-gl-config-name=rgba8888d24s8ms0 --deqp-screen-rotation=unspecified --deqp-surface-type=window --deqp-watchdog=enable" name="master"/>
</TestPackage>
diff --git a/android/cts/lmp/com.drawelements.deqp.gles3.xml b/android/cts/lmp/com.drawelements.deqp.gles3.xml
index b068e3b..c417472 100644
--- a/android/cts/lmp/com.drawelements.deqp.gles3.xml
+++ b/android/cts/lmp/com.drawelements.deqp.gles3.xml
@@ -1,5 +1,23 @@
<?xml version="1.0" encoding="utf-8"?>
<TestPackage appPackageName="com.drawelements.deqp.gles3" deqp:glesVersion="196608" name="dEQP-GLES3" testType="deqpTest" xmlns:deqp="http://drawelements.com/deqp">
+ <!--
+ Copyright (C) 2015 The Android Open Source Project
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+ <!--
+ This file has been automatically generated. Edit with caution.
+ -->
<TestSuite name="dEQP-GLES3">
<TestCase name="info">
<Test name="vendor">
diff --git a/android/cts/lmp/com.drawelements.deqp.gles31.xml b/android/cts/lmp/com.drawelements.deqp.gles31.xml
index dde2070..00c367c 100644
--- a/android/cts/lmp/com.drawelements.deqp.gles31.xml
+++ b/android/cts/lmp/com.drawelements.deqp.gles31.xml
@@ -1,5 +1,23 @@
<?xml version="1.0" encoding="utf-8"?>
<TestPackage appPackageName="com.drawelements.deqp.gles31" deqp:glesVersion="196609" name="dEQP-GLES31" testType="deqpTest" xmlns:deqp="http://drawelements.com/deqp">
+ <!--
+ Copyright (C) 2015 The Android Open Source Project
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+ <!--
+ This file has been automatically generated. Edit with caution.
+ -->
<TestSuite name="dEQP-GLES31">
<TestCase name="info">
<Test name="vendor">
diff --git a/android/cts/lmp/mustpass.xml b/android/cts/lmp/mustpass.xml
index abc191f..e32573f 100644
--- a/android/cts/lmp/mustpass.xml
+++ b/android/cts/lmp/mustpass.xml
@@ -1,5 +1,23 @@
<?xml version="1.0" encoding="utf-8"?>
<Mustpass version="lmp">
+ <!--
+ Copyright (C) 2015 The Android Open Source Project
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+ <!--
+ This file has been automatically generated. Edit with caution.
+ -->
<TestPackage name="dEQP-GLES3">
<Configuration caseListFile="gles3-master.txt" commandLine="--deqp-gl-config-name=rgba8888d24s8ms0 --deqp-screen-rotation=unspecified --deqp-surface-type=window --deqp-watchdog=enable" name="master"/>
</TestPackage>
diff --git a/android/cts/master/com.drawelements.deqp.egl.xml b/android/cts/master/com.drawelements.deqp.egl.xml
index 58cdf61..c801d48 100644
--- a/android/cts/master/com.drawelements.deqp.egl.xml
+++ b/android/cts/master/com.drawelements.deqp.egl.xml
@@ -1,5 +1,23 @@
<?xml version="1.0" encoding="utf-8"?>
<TestPackage appPackageName="com.drawelements.deqp.egl" deqp:glesVersion="131072" name="dEQP-EGL" testType="deqpTest" xmlns:deqp="http://drawelements.com/deqp">
+ <!--
+ Copyright (C) 2015 The Android Open Source Project
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+ <!--
+ This file has been automatically generated. Edit with caution.
+ -->
<TestSuite name="dEQP-EGL">
<TestCase name="info">
<Test name="version">
diff --git a/android/cts/master/com.drawelements.deqp.gles2.xml b/android/cts/master/com.drawelements.deqp.gles2.xml
index adbc036..88e9dc8 100644
--- a/android/cts/master/com.drawelements.deqp.gles2.xml
+++ b/android/cts/master/com.drawelements.deqp.gles2.xml
@@ -1,5 +1,23 @@
<?xml version="1.0" encoding="utf-8"?>
<TestPackage appPackageName="com.drawelements.deqp.gles2" deqp:glesVersion="131072" name="dEQP-GLES2" testType="deqpTest" xmlns:deqp="http://drawelements.com/deqp">
+ <!--
+ Copyright (C) 2015 The Android Open Source Project
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+ <!--
+ This file has been automatically generated. Edit with caution.
+ -->
<TestSuite name="dEQP-GLES2">
<TestCase name="info">
<Test name="vendor">
diff --git a/android/cts/master/com.drawelements.deqp.gles3.xml b/android/cts/master/com.drawelements.deqp.gles3.xml
index e7bef92..e58122e 100644
--- a/android/cts/master/com.drawelements.deqp.gles3.xml
+++ b/android/cts/master/com.drawelements.deqp.gles3.xml
@@ -1,5 +1,23 @@
<?xml version="1.0" encoding="utf-8"?>
<TestPackage appPackageName="com.drawelements.deqp.gles3" deqp:glesVersion="196608" name="dEQP-GLES3" testType="deqpTest" xmlns:deqp="http://drawelements.com/deqp">
+ <!--
+ Copyright (C) 2015 The Android Open Source Project
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+ <!--
+ This file has been automatically generated. Edit with caution.
+ -->
<TestSuite name="dEQP-GLES3">
<TestCase name="info">
<Test name="vendor">
diff --git a/android/cts/master/com.drawelements.deqp.gles31.xml b/android/cts/master/com.drawelements.deqp.gles31.xml
index b56b6b9..df29b69 100644
--- a/android/cts/master/com.drawelements.deqp.gles31.xml
+++ b/android/cts/master/com.drawelements.deqp.gles31.xml
@@ -1,5 +1,23 @@
<?xml version="1.0" encoding="utf-8"?>
<TestPackage appPackageName="com.drawelements.deqp.gles31" deqp:glesVersion="196609" name="dEQP-GLES31" testType="deqpTest" xmlns:deqp="http://drawelements.com/deqp">
+ <!--
+ Copyright (C) 2015 The Android Open Source Project
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+ <!--
+ This file has been automatically generated. Edit with caution.
+ -->
<TestSuite name="dEQP-GLES31">
<TestCase name="info">
<Test name="vendor">
diff --git a/android/cts/master/mustpass.xml b/android/cts/master/mustpass.xml
index f8c924a..3388e4c 100644
--- a/android/cts/master/mustpass.xml
+++ b/android/cts/master/mustpass.xml
@@ -1,5 +1,23 @@
<?xml version="1.0" encoding="utf-8"?>
<Mustpass version="master">
+ <!--
+ Copyright (C) 2015 The Android Open Source Project
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+ <!--
+ This file has been automatically generated. Edit with caution.
+ -->
<TestPackage name="dEQP-EGL">
<Configuration caseListFile="egl-master.txt" commandLine="--deqp-gl-config-name=rgba8888d24s8ms0 --deqp-screen-rotation=unspecified --deqp-surface-type=window --deqp-watchdog=enable" name="master"/>
</TestPackage>
diff --git a/android/cts/mnc/com.drawelements.deqp.egl.xml b/android/cts/mnc/com.drawelements.deqp.egl.xml
index 73784a3..4694698 100644
--- a/android/cts/mnc/com.drawelements.deqp.egl.xml
+++ b/android/cts/mnc/com.drawelements.deqp.egl.xml
@@ -1,5 +1,23 @@
<?xml version="1.0" encoding="utf-8"?>
<TestPackage appPackageName="com.drawelements.deqp.egl" deqp:glesVersion="131072" name="dEQP-EGL" testType="deqpTest" xmlns:deqp="http://drawelements.com/deqp">
+ <!--
+ Copyright (C) 2015 The Android Open Source Project
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+ <!--
+ This file has been automatically generated. Edit with caution.
+ -->
<TestSuite name="dEQP-EGL">
<TestCase name="info">
<Test name="version">
diff --git a/android/cts/mnc/com.drawelements.deqp.gles2.xml b/android/cts/mnc/com.drawelements.deqp.gles2.xml
index adbc036..88e9dc8 100644
--- a/android/cts/mnc/com.drawelements.deqp.gles2.xml
+++ b/android/cts/mnc/com.drawelements.deqp.gles2.xml
@@ -1,5 +1,23 @@
<?xml version="1.0" encoding="utf-8"?>
<TestPackage appPackageName="com.drawelements.deqp.gles2" deqp:glesVersion="131072" name="dEQP-GLES2" testType="deqpTest" xmlns:deqp="http://drawelements.com/deqp">
+ <!--
+ Copyright (C) 2015 The Android Open Source Project
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+ <!--
+ This file has been automatically generated. Edit with caution.
+ -->
<TestSuite name="dEQP-GLES2">
<TestCase name="info">
<Test name="vendor">
diff --git a/android/cts/mnc/com.drawelements.deqp.gles3.xml b/android/cts/mnc/com.drawelements.deqp.gles3.xml
index d8f6c4b..171ca14 100644
--- a/android/cts/mnc/com.drawelements.deqp.gles3.xml
+++ b/android/cts/mnc/com.drawelements.deqp.gles3.xml
@@ -1,5 +1,23 @@
<?xml version="1.0" encoding="utf-8"?>
<TestPackage appPackageName="com.drawelements.deqp.gles3" deqp:glesVersion="196608" name="dEQP-GLES3" testType="deqpTest" xmlns:deqp="http://drawelements.com/deqp">
+ <!--
+ Copyright (C) 2015 The Android Open Source Project
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+ <!--
+ This file has been automatically generated. Edit with caution.
+ -->
<TestSuite name="dEQP-GLES3">
<TestCase name="info">
<Test name="vendor">
diff --git a/android/cts/mnc/com.drawelements.deqp.gles31.xml b/android/cts/mnc/com.drawelements.deqp.gles31.xml
index 5891262..a30add5 100644
--- a/android/cts/mnc/com.drawelements.deqp.gles31.xml
+++ b/android/cts/mnc/com.drawelements.deqp.gles31.xml
@@ -1,5 +1,23 @@
<?xml version="1.0" encoding="utf-8"?>
<TestPackage appPackageName="com.drawelements.deqp.gles31" deqp:glesVersion="196609" name="dEQP-GLES31" testType="deqpTest" xmlns:deqp="http://drawelements.com/deqp">
+ <!--
+ Copyright (C) 2015 The Android Open Source Project
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+ <!--
+ This file has been automatically generated. Edit with caution.
+ -->
<TestSuite name="dEQP-GLES31">
<TestCase name="info">
<Test name="vendor">
diff --git a/android/cts/mnc/mustpass.xml b/android/cts/mnc/mustpass.xml
index dbaf3e8..bd5d337 100644
--- a/android/cts/mnc/mustpass.xml
+++ b/android/cts/mnc/mustpass.xml
@@ -1,5 +1,23 @@
<?xml version="1.0" encoding="utf-8"?>
<Mustpass version="mnc">
+ <!--
+ Copyright (C) 2015 The Android Open Source Project
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+ <!--
+ This file has been automatically generated. Edit with caution.
+ -->
<TestPackage name="dEQP-EGL">
<Configuration caseListFile="egl-master.txt" commandLine="--deqp-gl-config-name=rgba8888d24s8ms0 --deqp-screen-rotation=unspecified --deqp-surface-type=window --deqp-watchdog=enable" name="master"/>
</TestPackage>
diff --git a/android/cts/runner/Android.mk b/android/cts/runner/Android.mk
new file mode 100644
index 0000000..cb61b2a
--- /dev/null
+++ b/android/cts/runner/Android.mk
@@ -0,0 +1,20 @@
+# Copyright (C) 2015 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+LOCAL_PATH := $(call my-dir)
+
+include $(CLEAR_VARS)
+
+# Build all sub-directories
+include $(call all-makefiles-under,$(LOCAL_PATH))
diff --git a/android/cts/runner/src/com/drawelements/deqp/runner/DeqpTestRunner.java b/android/cts/runner/src/com/drawelements/deqp/runner/DeqpTestRunner.java
new file mode 100644
index 0000000..79d8d13
--- /dev/null
+++ b/android/cts/runner/src/com/drawelements/deqp/runner/DeqpTestRunner.java
@@ -0,0 +1,1959 @@
+/*
+ * Copyright (C) 2014 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.drawelements.deqp.runner;
+
+import com.android.compatibility.common.tradefed.build.CompatibilityBuildHelper;
+import com.android.compatibility.common.util.AbiUtils;
+import com.android.ddmlib.AdbCommandRejectedException;
+import com.android.ddmlib.IShellOutputReceiver;
+import com.android.ddmlib.MultiLineReceiver;
+import com.android.ddmlib.ShellCommandUnresponsiveException;
+import com.android.ddmlib.TimeoutException;
+import com.android.ddmlib.testrunner.TestIdentifier;
+import com.android.tradefed.build.IBuildInfo;
+import com.android.tradefed.build.IFolderBuildInfo;
+import com.android.tradefed.config.Option;
+import com.android.tradefed.config.OptionClass;
+import com.android.tradefed.device.DeviceNotAvailableException;
+import com.android.tradefed.device.ITestDevice;
+import com.android.tradefed.log.LogUtil.CLog;
+import com.android.tradefed.result.ByteArrayInputStreamSource;
+import com.android.tradefed.result.ITestInvocationListener;
+import com.android.tradefed.result.LogDataType;
+import com.android.tradefed.testtype.IAbi;
+import com.android.tradefed.testtype.IBuildReceiver;
+import com.android.tradefed.testtype.IAbiReceiver;
+import com.android.tradefed.testtype.IDeviceTest;
+import com.android.tradefed.testtype.IRemoteTest;
+import com.android.tradefed.testtype.ITestFilterReceiver;
+import com.android.tradefed.util.IRunUtil;
+import com.android.tradefed.util.RunInterruptedException;
+import com.android.tradefed.util.RunUtil;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileReader;
+import java.io.IOException;
+import java.io.Reader;
+import java.lang.reflect.Method;
+import java.lang.reflect.InvocationTargetException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.regex.Pattern;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Test runner for dEQP tests
+ *
+ * Supports running drawElements Quality Program tests found under external/deqp.
+ */
+@OptionClass(alias="deqp-test-runner")
+public class DeqpTestRunner implements IBuildReceiver, IDeviceTest, IRemoteTest, ITestFilterReceiver, IAbiReceiver {
+
+ private static final String DEQP_ONDEVICE_APK = "com.drawelements.deqp.apk";
+ private static final String DEQP_ONDEVICE_PKG = "com.drawelements.deqp";
+ private static final String INCOMPLETE_LOG_MESSAGE = "Crash: Incomplete test log";
+ private static final String SKIPPED_INSTANCE_LOG_MESSAGE = "Configuration skipped";
+ private static final String NOT_EXECUTABLE_LOG_MESSAGE = "Abort: Test cannot be executed";
+ private static final String CASE_LIST_FILE_NAME = "/sdcard/dEQP-TestCaseList.txt";
+ private static final String LOG_FILE_NAME = "/sdcard/TestLog.qpa";
+ public static final String FEATURE_LANDSCAPE = "android.hardware.screen.landscape";
+ public static final String FEATURE_PORTRAIT = "android.hardware.screen.portrait";
+
+ private static final int TESTCASE_BATCH_LIMIT = 1000;
+ private static final BatchRunConfiguration DEFAULT_CONFIG =
+ new BatchRunConfiguration("rgba8888d24s8", "unspecified", "window");
+
+ private static final int UNRESPOSIVE_CMD_TIMEOUT_MS = 60000; // one minute
+
+ @Option(name="deqp-package", description="Name of the deqp module used. Determines GLES version.", importance=Option.Importance.ALWAYS)
+ private String mDeqpPackage;
+ @Option(name="deqp-gl-config-name", description="GL render target config. See deqp documentation for syntax. ", importance=Option.Importance.ALWAYS)
+ private String mConfigName;
+ @Option(name="deqp-caselist-file", description="File listing the names of the cases to be run.", importance=Option.Importance.ALWAYS)
+ private String mCaselistFile;
+ @Option(name="deqp-screen-rotation", description="Screen orientation. Defaults to 'unspecified'", importance=Option.Importance.NEVER)
+ private String mScreenRotation = "unspecified";
+ @Option(name="deqp-surface-type", description="Surface type ('window', 'pbuffer', 'fbo'). Defaults to 'window'", importance=Option.Importance.NEVER)
+ private String mSurfaceType = "window";
+
+ @Option(name = "include-filter",
+ description="Test include filter. '*' is zero or more letters. '.' has no special meaning.")
+ private List<String> mIncludeFilters = new ArrayList<>();
+ @Option(name = "exclude-filter",
+ description="Test exclude filter. '*' is zero or more letters. '.' has no special meaning.")
+ private List<String> mExcludeFilters = new ArrayList<>();
+ private Collection<TestIdentifier> mRemainingTests = null;
+ private Map<TestIdentifier, Set<BatchRunConfiguration>> mTestInstances = null;
+ private final TestInstanceResultListener mInstanceListerner = new TestInstanceResultListener();
+ private final Map<TestIdentifier, Integer> mTestInstabilityRatings = new HashMap<>();
+ private IAbi mAbi;
+ private CompatibilityBuildHelper mBuildHelper;
+ private boolean mLogData = false;
+ private ITestDevice mDevice;
+ private Set<String> mDeviceFeatures;
+ private Map<String, Boolean> mConfigQuerySupportCache = new HashMap<>();
+ private IRunUtil mRunUtil = RunUtil.getDefault();
+ // When set will override the mCaselistFile for testing purposes.
+ private Reader mCaselistReader = null;
+
+ private IRecovery mDeviceRecovery = new Recovery();
+ {
+ mDeviceRecovery.setSleepProvider(new SleepProvider());
+ }
+
+ /**
+ * @param abi the ABI to run the test on
+ */
+ @Override
+ public void setAbi(IAbi abi) {
+ mAbi = abi;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void setBuild(IBuildInfo buildInfo) {
+ setBuildHelper(new CompatibilityBuildHelper((IFolderBuildInfo)buildInfo));
+ }
+
+ /**
+ * Exposed for better mockability during testing. In real use, always flows from
+ * setBuild() called by the framework
+ */
+ public void setBuildHelper(CompatibilityBuildHelper helper) {
+ mBuildHelper = helper;
+ }
+
+ /**
+ * Enable or disable raw dEQP test log collection.
+ */
+ public void setCollectLogs(boolean logData) {
+ mLogData = logData;
+ }
+
+ /**
+ * Get the deqp-package option contents.
+ */
+ public String getPackageName() {
+ return mDeqpPackage;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void setDevice(ITestDevice device) {
+ mDevice = device;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public ITestDevice getDevice() {
+ return mDevice;
+ }
+
+ /**
+ * Set recovery handler.
+ *
+ * Exposed for unit testing.
+ */
+ public void setRecovery(IRecovery deviceRecovery) {
+ mDeviceRecovery = deviceRecovery;
+ }
+
+ /**
+ * Set IRunUtil.
+ *
+ * Exposed for unit testing.
+ */
+ public void setRunUtil(IRunUtil runUtil) {
+ mRunUtil = runUtil;
+ }
+
+ /**
+ * Exposed for unit testing
+ */
+ public void setCaselistReader(Reader caselistReader) {
+ mCaselistReader = caselistReader;
+ }
+
+ private static final class CapabilityQueryFailureException extends Exception {
+ }
+
+ /**
+ * Test configuration of dEPQ test instance execution.
+ * Exposed for unit testing
+ */
+ public static final class BatchRunConfiguration {
+ public static final String ROTATION_UNSPECIFIED = "unspecified";
+ public static final String ROTATION_PORTRAIT = "0";
+ public static final String ROTATION_LANDSCAPE = "90";
+ public static final String ROTATION_REVERSE_PORTRAIT = "180";
+ public static final String ROTATION_REVERSE_LANDSCAPE = "270";
+
+ private final String mGlConfig;
+ private final String mRotation;
+ private final String mSurfaceType;
+
+ public BatchRunConfiguration(String glConfig, String rotation, String surfaceType) {
+ mGlConfig = glConfig;
+ mRotation = rotation;
+ mSurfaceType = surfaceType;
+ }
+
+ /**
+ * Get string that uniquely identifies this config
+ */
+ public String getId() {
+ return String.format("{glformat=%s,rotation=%s,surfacetype=%s}",
+ mGlConfig, mRotation, mSurfaceType);
+ }
+
+ /**
+ * Get the GL config used in this configuration.
+ */
+ public String getGlConfig() {
+ return mGlConfig;
+ }
+
+ /**
+ * Get the screen rotation used in this configuration.
+ */
+ public String getRotation() {
+ return mRotation;
+ }
+
+ /**
+ * Get the surface type used in this configuration.
+ */
+ public String getSurfaceType() {
+ return mSurfaceType;
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == null) {
+ return false;
+ } else if (!(other instanceof BatchRunConfiguration)) {
+ return false;
+ } else {
+ return getId().equals(((BatchRunConfiguration)other).getId());
+ }
+ }
+
+ @Override
+ public int hashCode() {
+ return getId().hashCode();
+ }
+ }
+
+ /**
+ * dEQP test instance listerer and invocation result forwarded
+ */
+ private class TestInstanceResultListener {
+ private ITestInvocationListener mSink;
+ private BatchRunConfiguration mRunConfig;
+
+ private TestIdentifier mCurrentTestId;
+ private boolean mGotTestResult;
+ private String mCurrentTestLog;
+
+ private class PendingResult {
+ boolean allInstancesPassed;
+ Map<BatchRunConfiguration, String> testLogs;
+ Map<BatchRunConfiguration, String> errorMessages;
+ Set<BatchRunConfiguration> remainingConfigs;
+ }
+
+ private final Map<TestIdentifier, PendingResult> mPendingResults = new HashMap<>();
+
+ public void setSink(ITestInvocationListener sink) {
+ mSink = sink;
+ }
+
+ public void setCurrentConfig(BatchRunConfiguration runConfig) {
+ mRunConfig = runConfig;
+ }
+
+ /**
+ * Get currently processed test id, or null if not currently processing a test case
+ */
+ public TestIdentifier getCurrentTestId() {
+ return mCurrentTestId;
+ }
+
+ /**
+ * Forward result to sink
+ */
+ private void forwardFinalizedPendingResult(TestIdentifier testId) {
+ if (mRemainingTests.contains(testId)) {
+ final PendingResult result = mPendingResults.get(testId);
+
+ mPendingResults.remove(testId);
+ mRemainingTests.remove(testId);
+
+ // Forward results to the sink
+ mSink.testStarted(testId);
+
+ // Test Log
+ if (mLogData) {
+ for (Map.Entry<BatchRunConfiguration, String> entry :
+ result.testLogs.entrySet()) {
+ final ByteArrayInputStreamSource source
+ = new ByteArrayInputStreamSource(entry.getValue().getBytes());
+
+ mSink.testLog(testId.getClassName() + "." + testId.getTestName() + "@"
+ + entry.getKey().getId(), LogDataType.XML, source);
+
+ source.cancel();
+ }
+ }
+
+ // Error message
+ if (!result.allInstancesPassed) {
+ final StringBuilder errorLog = new StringBuilder();
+
+ for (Map.Entry<BatchRunConfiguration, String> entry :
+ result.errorMessages.entrySet()) {
+ if (errorLog.length() > 0) {
+ errorLog.append('\n');
+ }
+ errorLog.append(String.format("=== with config %s ===\n",
+ entry.getKey().getId()));
+ errorLog.append(entry.getValue());
+ }
+
+ mSink.testFailed(testId, errorLog.toString());
+ }
+
+ final Map<String, String> emptyMap = Collections.emptyMap();
+ mSink.testEnded(testId, emptyMap);
+ }
+ }
+
+ /**
+ * Declare existence of a test and instances
+ */
+ public void setTestInstances(TestIdentifier testId, Set<BatchRunConfiguration> configs) {
+ // Test instances cannot change at runtime, ignore if we have already set this
+ if (!mPendingResults.containsKey(testId)) {
+ final PendingResult pendingResult = new PendingResult();
+ pendingResult.allInstancesPassed = true;
+ pendingResult.testLogs = new LinkedHashMap<>();
+ pendingResult.errorMessages = new LinkedHashMap<>();
+ pendingResult.remainingConfigs = new HashSet<>(configs); // avoid mutating argument
+ mPendingResults.put(testId, pendingResult);
+ }
+ }
+
+ /**
+ * Query if test instance has not yet been executed
+ */
+ public boolean isPendingTestInstance(TestIdentifier testId,
+ BatchRunConfiguration config) {
+ final PendingResult result = mPendingResults.get(testId);
+ if (result == null) {
+ // test is not in the current working batch of the runner, i.e. it cannot be
+ // "partially" completed.
+ if (!mRemainingTests.contains(testId)) {
+ // The test has been fully executed. Not pending.
+ return false;
+ } else {
+ // Test has not yet been executed. Check if such instance exists
+ return mTestInstances.get(testId).contains(config);
+ }
+ } else {
+ // could be partially completed, check this particular config
+ return result.remainingConfigs.contains(config);
+ }
+ }
+
+ /**
+ * Fake execution of an instance with current config
+ */
+ public void skipTest(TestIdentifier testId) {
+ final PendingResult result = mPendingResults.get(testId);
+
+ result.errorMessages.put(mRunConfig, SKIPPED_INSTANCE_LOG_MESSAGE);
+ result.remainingConfigs.remove(mRunConfig);
+
+ // Pending result finished, report result
+ if (result.remainingConfigs.isEmpty()) {
+ forwardFinalizedPendingResult(testId);
+ }
+ }
+
+ /**
+ * Fake failure of an instance with current config
+ */
+ public void abortTest(TestIdentifier testId, String errorMessage) {
+ final PendingResult result = mPendingResults.get(testId);
+
+ // Mark as executed
+ result.allInstancesPassed = false;
+ result.errorMessages.put(mRunConfig, errorMessage);
+ result.remainingConfigs.remove(mRunConfig);
+
+ // Pending result finished, report result
+ if (result.remainingConfigs.isEmpty()) {
+ forwardFinalizedPendingResult(testId);
+ }
+
+ if (testId.equals(mCurrentTestId)) {
+ mCurrentTestId = null;
+ }
+ }
+
+ /**
+ * Handles beginning of dEQP session.
+ */
+ private void handleBeginSession(Map<String, String> values) {
+ // ignore
+ }
+
+ /**
+ * Handles end of dEQP session.
+ */
+ private void handleEndSession(Map<String, String> values) {
+ // ignore
+ }
+
+ /**
+ * Handles beginning of dEQP testcase.
+ */
+ private void handleBeginTestCase(Map<String, String> values) {
+ mCurrentTestId = pathToIdentifier(values.get("dEQP-BeginTestCase-TestCasePath"));
+ mCurrentTestLog = "";
+ mGotTestResult = false;
+
+ // mark instance as started
+ if (mPendingResults.get(mCurrentTestId) != null) {
+ mPendingResults.get(mCurrentTestId).remainingConfigs.remove(mRunConfig);
+ } else {
+ CLog.w("Got unexpected start of %s", mCurrentTestId);
+ }
+ }
+
+ /**
+ * Handles end of dEQP testcase.
+ */
+ private void handleEndTestCase(Map<String, String> values) {
+ final PendingResult result = mPendingResults.get(mCurrentTestId);
+
+ if (result != null) {
+ if (!mGotTestResult) {
+ result.allInstancesPassed = false;
+ result.errorMessages.put(mRunConfig, INCOMPLETE_LOG_MESSAGE);
+ }
+
+ if (mLogData && mCurrentTestLog != null && mCurrentTestLog.length() > 0) {
+ result.testLogs.put(mRunConfig, mCurrentTestLog);
+ }
+
+ // Pending result finished, report result
+ if (result.remainingConfigs.isEmpty()) {
+ forwardFinalizedPendingResult(mCurrentTestId);
+ }
+ } else {
+ CLog.w("Got unexpected end of %s", mCurrentTestId);
+ }
+ mCurrentTestId = null;
+ }
+
+ /**
+ * Handles dEQP testcase result.
+ */
+ private void handleTestCaseResult(Map<String, String> values) {
+ String code = values.get("dEQP-TestCaseResult-Code");
+ String details = values.get("dEQP-TestCaseResult-Details");
+
+ if (mPendingResults.get(mCurrentTestId) == null) {
+ CLog.w("Got unexpected result for %s", mCurrentTestId);
+ mGotTestResult = true;
+ return;
+ }
+
+ if (code.compareTo("Pass") == 0) {
+ mGotTestResult = true;
+ } else if (code.compareTo("NotSupported") == 0) {
+ mGotTestResult = true;
+ } else if (code.compareTo("QualityWarning") == 0) {
+ mGotTestResult = true;
+ } else if (code.compareTo("CompatibilityWarning") == 0) {
+ mGotTestResult = true;
+ } else if (code.compareTo("Fail") == 0 || code.compareTo("ResourceError") == 0
+ || code.compareTo("InternalError") == 0 || code.compareTo("Crash") == 0
+ || code.compareTo("Timeout") == 0) {
+ mPendingResults.get(mCurrentTestId).allInstancesPassed = false;
+ mPendingResults.get(mCurrentTestId)
+ .errorMessages.put(mRunConfig, code + ": " + details);
+ mGotTestResult = true;
+ } else {
+ String codeError = "Unknown result code: " + code;
+ mPendingResults.get(mCurrentTestId).allInstancesPassed = false;
+ mPendingResults.get(mCurrentTestId)
+ .errorMessages.put(mRunConfig, codeError + ": " + details);
+ mGotTestResult = true;
+ }
+ }
+
+ /**
+ * Handles terminated dEQP testcase.
+ */
+ private void handleTestCaseTerminate(Map<String, String> values) {
+ final PendingResult result = mPendingResults.get(mCurrentTestId);
+
+ if (result != null) {
+ String reason = values.get("dEQP-TerminateTestCase-Reason");
+ mPendingResults.get(mCurrentTestId).allInstancesPassed = false;
+ mPendingResults.get(mCurrentTestId)
+ .errorMessages.put(mRunConfig, "Terminated: " + reason);
+
+ // Pending result finished, report result
+ if (result.remainingConfigs.isEmpty()) {
+ forwardFinalizedPendingResult(mCurrentTestId);
+ }
+ } else {
+ CLog.w("Got unexpected termination of %s", mCurrentTestId);
+ }
+
+ mCurrentTestId = null;
+ mGotTestResult = true;
+ }
+
+ /**
+ * Handles dEQP testlog data.
+ */
+ private void handleTestLogData(Map<String, String> values) {
+ mCurrentTestLog = mCurrentTestLog + values.get("dEQP-TestLogData-Log");
+ }
+
+ /**
+ * Handles new instrumentation status message.
+ */
+ public void handleStatus(Map<String, String> values) {
+ String eventType = values.get("dEQP-EventType");
+
+ if (eventType == null) {
+ return;
+ }
+
+ if (eventType.compareTo("BeginSession") == 0) {
+ handleBeginSession(values);
+ } else if (eventType.compareTo("EndSession") == 0) {
+ handleEndSession(values);
+ } else if (eventType.compareTo("BeginTestCase") == 0) {
+ handleBeginTestCase(values);
+ } else if (eventType.compareTo("EndTestCase") == 0) {
+ handleEndTestCase(values);
+ } else if (eventType.compareTo("TestCaseResult") == 0) {
+ handleTestCaseResult(values);
+ } else if (eventType.compareTo("TerminateTestCase") == 0) {
+ handleTestCaseTerminate(values);
+ } else if (eventType.compareTo("TestLogData") == 0) {
+ handleTestLogData(values);
+ }
+ }
+
+ /**
+ * Signal listener that batch ended and forget incomplete results.
+ */
+ public void endBatch() {
+ // end open test if when stream ends
+ if (mCurrentTestId != null) {
+ // Current instance was removed from remainingConfigs when case
+ // started. Mark current instance as pending.
+ if (mPendingResults.get(mCurrentTestId) != null) {
+ mPendingResults.get(mCurrentTestId).remainingConfigs.add(mRunConfig);
+ } else {
+ CLog.w("Got unexpected internal state of %s", mCurrentTestId);
+ }
+ }
+ mCurrentTestId = null;
+ }
+ }
+
+ /**
+ * dEQP instrumentation parser
+ */
+ private static class InstrumentationParser extends MultiLineReceiver {
+ private TestInstanceResultListener mListener;
+
+ private Map<String, String> mValues;
+ private String mCurrentName;
+ private String mCurrentValue;
+ private int mResultCode;
+ private boolean mGotExitValue = false;
+
+
+ public InstrumentationParser(TestInstanceResultListener listener) {
+ mListener = listener;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void processNewLines(String[] lines) {
+ for (String line : lines) {
+ if (mValues == null) mValues = new HashMap<String, String>();
+
+ if (line.startsWith("INSTRUMENTATION_STATUS_CODE: ")) {
+ if (mCurrentName != null) {
+ mValues.put(mCurrentName, mCurrentValue);
+
+ mCurrentName = null;
+ mCurrentValue = null;
+ }
+
+ mListener.handleStatus(mValues);
+ mValues = null;
+ } else if (line.startsWith("INSTRUMENTATION_STATUS: dEQP-")) {
+ if (mCurrentName != null) {
+ mValues.put(mCurrentName, mCurrentValue);
+
+ mCurrentValue = null;
+ mCurrentName = null;
+ }
+
+ String prefix = "INSTRUMENTATION_STATUS: ";
+ int nameBegin = prefix.length();
+ int nameEnd = line.indexOf('=');
+ int valueBegin = nameEnd + 1;
+
+ mCurrentName = line.substring(nameBegin, nameEnd);
+ mCurrentValue = line.substring(valueBegin);
+ } else if (line.startsWith("INSTRUMENTATION_CODE: ")) {
+ try {
+ mResultCode = Integer.parseInt(line.substring(22));
+ mGotExitValue = true;
+ } catch (NumberFormatException ex) {
+ CLog.w("Instrumentation code format unexpected");
+ }
+ } else if (mCurrentValue != null) {
+ mCurrentValue = mCurrentValue + line;
+ }
+ }
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void done() {
+ if (mCurrentName != null) {
+ mValues.put(mCurrentName, mCurrentValue);
+
+ mCurrentName = null;
+ mCurrentValue = null;
+ }
+
+ if (mValues != null) {
+ mListener.handleStatus(mValues);
+ mValues = null;
+ }
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public boolean isCancelled() {
+ return false;
+ }
+
+ /**
+ * Returns whether target instrumentation exited normally.
+ */
+ public boolean wasSuccessful() {
+ return mGotExitValue;
+ }
+
+ /**
+ * Returns Instrumentation return code
+ */
+ public int getResultCode() {
+ return mResultCode;
+ }
+ }
+
+ /**
+ * dEQP platfom query instrumentation parser
+ */
+ private static class PlatformQueryInstrumentationParser extends MultiLineReceiver {
+ private Map<String,String> mResultMap = new LinkedHashMap<>();
+ private int mResultCode;
+ private boolean mGotExitValue = false;
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void processNewLines(String[] lines) {
+ for (String line : lines) {
+ if (line.startsWith("INSTRUMENTATION_RESULT: ")) {
+ final String parts[] = line.substring(24).split("=",2);
+ if (parts.length == 2) {
+ mResultMap.put(parts[0], parts[1]);
+ } else {
+ CLog.w("Instrumentation status format unexpected");
+ }
+ } else if (line.startsWith("INSTRUMENTATION_CODE: ")) {
+ try {
+ mResultCode = Integer.parseInt(line.substring(22));
+ mGotExitValue = true;
+ } catch (NumberFormatException ex) {
+ CLog.w("Instrumentation code format unexpected");
+ }
+ }
+ }
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public boolean isCancelled() {
+ return false;
+ }
+
+ /**
+ * Returns whether target instrumentation exited normally.
+ */
+ public boolean wasSuccessful() {
+ return mGotExitValue;
+ }
+
+ /**
+ * Returns Instrumentation return code
+ */
+ public int getResultCode() {
+ return mResultCode;
+ }
+
+ public Map<String,String> getResultMap() {
+ return mResultMap;
+ }
+ }
+
+ /**
+ * Interface for sleeping.
+ *
+ * Exposed for unit testing
+ */
+ public static interface ISleepProvider {
+ public void sleep(int milliseconds);
+ }
+
+ private static class SleepProvider implements ISleepProvider {
+ public void sleep(int milliseconds) {
+ try {
+ Thread.sleep(milliseconds);
+ } catch (InterruptedException ex) {
+ }
+ }
+ }
+
+ /**
+ * Interface for failure recovery.
+ *
+ * Exposed for unit testing
+ */
+ public static interface IRecovery {
+ /**
+ * Sets the sleep provider IRecovery works on
+ */
+ public void setSleepProvider(ISleepProvider sleepProvider);
+
+ /**
+ * Sets the device IRecovery works on
+ */
+ public void setDevice(ITestDevice device);
+
+ /**
+ * Informs Recovery that test execution has progressed since the last recovery
+ */
+ public void onExecutionProgressed();
+
+ /**
+ * Tries to recover device after failed refused connection.
+ *
+ * @throws DeviceNotAvailableException if recovery did not succeed
+ */
+ public void recoverConnectionRefused() throws DeviceNotAvailableException;
+
+ /**
+ * Tries to recover device after abnormal execution termination or link failure.
+ *
+ * @param progressedSinceLastCall true if test execution has progressed since last call
+ * @throws DeviceNotAvailableException if recovery did not succeed
+ */
+ public void recoverComLinkKilled() throws DeviceNotAvailableException;
+ };
+
+ /**
+ * State machine for execution failure recovery.
+ *
+ * Exposed for unit testing
+ */
+ public static class Recovery implements IRecovery {
+ private int RETRY_COOLDOWN_MS = 6000; // 6 seconds
+ private int PROCESS_KILL_WAIT_MS = 1000; // 1 second
+
+ private static enum MachineState {
+ WAIT, // recover by waiting
+ RECOVER, // recover by calling recover()
+ REBOOT, // recover by rebooting
+ FAIL, // cannot recover
+ };
+
+ private MachineState mState = MachineState.WAIT;
+ private ITestDevice mDevice;
+ private ISleepProvider mSleepProvider;
+
+ private static class ProcessKillFailureException extends Exception {
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ public void setSleepProvider(ISleepProvider sleepProvider) {
+ mSleepProvider = sleepProvider;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void setDevice(ITestDevice device) {
+ mDevice = device;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void onExecutionProgressed() {
+ mState = MachineState.WAIT;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void recoverConnectionRefused() throws DeviceNotAvailableException {
+ switch (mState) {
+ case WAIT: // not a valid stratedy for connection refusal, fallthrough
+ case RECOVER:
+ // First failure, just try to recover
+ CLog.w("ADB connection failed, trying to recover");
+ mState = MachineState.REBOOT; // the next step is to reboot
+
+ try {
+ recoverDevice();
+ } catch (DeviceNotAvailableException ex) {
+ // chain forward
+ recoverConnectionRefused();
+ }
+ break;
+
+ case REBOOT:
+ // Second failure in a row, try to reboot
+ CLog.w("ADB connection failed after recovery, rebooting device");
+ mState = MachineState.FAIL; // the next step is to fail
+
+ try {
+ rebootDevice();
+ } catch (DeviceNotAvailableException ex) {
+ // chain forward
+ recoverConnectionRefused();
+ }
+ break;
+
+ case FAIL:
+ // Third failure in a row, just fail
+ CLog.w("Cannot recover ADB connection");
+ throw new DeviceNotAvailableException("failed to connect after reboot");
+ }
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void recoverComLinkKilled() throws DeviceNotAvailableException {
+ switch (mState) {
+ case WAIT:
+ // First failure, just try to wait and try again
+ CLog.w("ADB link failed, retrying after a cooldown period");
+ mState = MachineState.RECOVER; // the next step is to recover the device
+
+ waitCooldown();
+
+ // even if the link to deqp on-device process was killed, the process might
+ // still be alive. Locate and terminate such unwanted processes.
+ try {
+ killDeqpProcess();
+ } catch (DeviceNotAvailableException ex) {
+ // chain forward
+ recoverComLinkKilled();
+ } catch (ProcessKillFailureException ex) {
+ // chain forward
+ recoverComLinkKilled();
+ }
+ break;
+
+ case RECOVER:
+ // Second failure, just try to recover
+ CLog.w("ADB link failed, trying to recover");
+ mState = MachineState.REBOOT; // the next step is to reboot
+
+ try {
+ recoverDevice();
+ killDeqpProcess();
+ } catch (DeviceNotAvailableException ex) {
+ // chain forward
+ recoverComLinkKilled();
+ } catch (ProcessKillFailureException ex) {
+ // chain forward
+ recoverComLinkKilled();
+ }
+ break;
+
+ case REBOOT:
+ // Third failure in a row, try to reboot
+ CLog.w("ADB link failed after recovery, rebooting device");
+ mState = MachineState.FAIL; // the next step is to fail
+
+ try {
+ rebootDevice();
+ } catch (DeviceNotAvailableException ex) {
+ // chain forward
+ recoverComLinkKilled();
+ }
+ break;
+
+ case FAIL:
+ // Fourth failure in a row, just fail
+ CLog.w("Cannot recover ADB connection");
+ throw new DeviceNotAvailableException("link killed after reboot");
+ }
+ }
+
+ private void waitCooldown() {
+ mSleepProvider.sleep(RETRY_COOLDOWN_MS);
+ }
+
+ private Iterable<Integer> getDeqpProcessPids() throws DeviceNotAvailableException {
+ final List<Integer> pids = new ArrayList<Integer>(2);
+ final String processes = mDevice.executeShellCommand("ps | grep com.drawelements");
+ final String[] lines = processes.split("(\\r|\\n)+");
+ for (String line : lines) {
+ final String[] fields = line.split("\\s+");
+ if (fields.length < 2) {
+ continue;
+ }
+
+ try {
+ final int processId = Integer.parseInt(fields[1], 10);
+ pids.add(processId);
+ } catch (NumberFormatException ex) {
+ continue;
+ }
+ }
+ return pids;
+ }
+
+ private void killDeqpProcess() throws DeviceNotAvailableException,
+ ProcessKillFailureException {
+ for (Integer processId : getDeqpProcessPids()) {
+ mDevice.executeShellCommand(String.format("kill -9 %d", processId));
+ }
+
+ mSleepProvider.sleep(PROCESS_KILL_WAIT_MS);
+
+ // check that processes actually died
+ if (getDeqpProcessPids().iterator().hasNext()) {
+ // a process is still alive, killing failed
+ throw new ProcessKillFailureException();
+ }
+ }
+
+ public void recoverDevice() throws DeviceNotAvailableException {
+ // Work around the API. We need to call recoverDevice() on the test device and
+ // we know that mDevice is a TestDevice. However even though the recoverDevice()
+ // method is public suggesting it should be publicly accessible, the class itself
+ // and its super-interface (IManagedTestDevice) are package-private.
+ final Method recoverDeviceMethod;
+ try {
+ recoverDeviceMethod = mDevice.getClass().getMethod("recoverDevice");
+ recoverDeviceMethod.setAccessible(true);
+ } catch (NoSuchMethodException ex) {
+ throw new AssertionError("Test device must have recoverDevice()");
+ }
+
+ try {
+ recoverDeviceMethod.invoke(mDevice);
+ } catch (InvocationTargetException ex) {
+ if (ex.getCause() instanceof DeviceNotAvailableException) {
+ throw (DeviceNotAvailableException)ex.getCause();
+ } else if (ex.getCause() instanceof RuntimeException) {
+ throw (RuntimeException)ex.getCause();
+ } else {
+ throw new AssertionError("unexpected throw", ex);
+ }
+ } catch (IllegalAccessException ex) {
+ throw new AssertionError("unexpected throw", ex);
+ }
+ }
+
+ private void rebootDevice() throws DeviceNotAvailableException {
+ mDevice.reboot();
+ }
+ }
+
+ private static Map<TestIdentifier, Set<BatchRunConfiguration>> generateTestInstances(
+ Reader testlist, String configName, String screenRotation, String surfaceType) throws FileNotFoundException {
+ // Note: This is specifically a LinkedHashMap to guarantee that tests are iterated
+ // in the insertion order.
+ final Map<TestIdentifier, Set<BatchRunConfiguration>> instances = new LinkedHashMap<>();
+ try {
+ BufferedReader testlistReader = new BufferedReader(testlist);
+ String testName;
+ while ((testName = testlistReader.readLine()) != null) {
+ // Test name -> testId -> only one config -> done.
+ final Set<BatchRunConfiguration> testInstanceSet = new LinkedHashSet<>();
+ BatchRunConfiguration config = new BatchRunConfiguration(configName, screenRotation, surfaceType);
+ testInstanceSet.add(config);
+ TestIdentifier test = pathToIdentifier(testName);
+ instances.put(test, testInstanceSet);
+ }
+ testlistReader.close();
+ }
+ catch (IOException e)
+ {
+ throw new RuntimeException("Failure while reading the test case list for deqp: " + e.getMessage());
+ }
+
+ return instances;
+ }
+
+ private Set<BatchRunConfiguration> getTestRunConfigs (TestIdentifier testId) {
+ return mTestInstances.get(testId);
+ }
+
+ /**
+ * Converts dEQP testcase path to TestIdentifier.
+ */
+ private static TestIdentifier pathToIdentifier(String testPath) {
+ int indexOfLastDot = testPath.lastIndexOf('.');
+ String className = testPath.substring(0, indexOfLastDot);
+ String testName = testPath.substring(indexOfLastDot+1);
+
+ return new TestIdentifier(className, testName);
+ }
+
+ // \todo [2015-10-16 kalle] How unique should this be?
+ private String getId() {
+ return AbiUtils.createId(mAbi.getName(), mDeqpPackage);
+ }
+
+ /**
+ * Generates tescase trie from dEQP testcase paths. Used to define which testcases to execute.
+ */
+ private static String generateTestCaseTrieFromPaths(Collection<String> tests) {
+ String result = "{";
+ boolean first = true;
+
+ // Add testcases to results
+ for (Iterator<String> iter = tests.iterator(); iter.hasNext();) {
+ String test = iter.next();
+ String[] components = test.split("\\.");
+
+ if (components.length == 1) {
+ if (!first) {
+ result = result + ",";
+ }
+ first = false;
+
+ result += components[0];
+ iter.remove();
+ }
+ }
+
+ if (!tests.isEmpty()) {
+ HashMap<String, ArrayList<String> > testGroups = new HashMap<>();
+
+ // Collect all sub testgroups
+ for (String test : tests) {
+ String[] components = test.split("\\.");
+ ArrayList<String> testGroup = testGroups.get(components[0]);
+
+ if (testGroup == null) {
+ testGroup = new ArrayList<String>();
+ testGroups.put(components[0], testGroup);
+ }
+
+ testGroup.add(test.substring(components[0].length()+1));
+ }
+
+ for (String testGroup : testGroups.keySet()) {
+ if (!first) {
+ result = result + ",";
+ }
+
+ first = false;
+ result = result + testGroup
+ + generateTestCaseTrieFromPaths(testGroups.get(testGroup));
+ }
+ }
+
+ return result + "}";
+ }
+
+ /**
+ * Generates testcase trie from TestIdentifiers.
+ */
+ private static String generateTestCaseTrie(Collection<TestIdentifier> tests) {
+ ArrayList<String> testPaths = new ArrayList<String>();
+
+ for (TestIdentifier test : tests) {
+ testPaths.add(test.getClassName() + "." + test.getTestName());
+ }
+
+ return generateTestCaseTrieFromPaths(testPaths);
+ }
+
+ private static class TestBatch {
+ public BatchRunConfiguration config;
+ public List<TestIdentifier> tests;
+ }
+
+ /**
+ * Creates a TestBatch from the given tests or null if not tests remaining.
+ *
+ * @param pool List of tests to select from
+ * @param requiredConfig Select only instances with pending requiredConfig, or null to select
+ * any run configuration.
+ */
+ private TestBatch selectRunBatch(Collection<TestIdentifier> pool,
+ BatchRunConfiguration requiredConfig) {
+ // select one test (leading test) that is going to be executed and then pack along as many
+ // other compatible instances as possible.
+
+ TestIdentifier leadingTest = null;
+ for (TestIdentifier test : pool) {
+ if (!mRemainingTests.contains(test)) {
+ continue;
+ }
+ if (requiredConfig != null &&
+ !mInstanceListerner.isPendingTestInstance(test, requiredConfig)) {
+ continue;
+ }
+ leadingTest = test;
+ break;
+ }
+
+ // no remaining tests?
+ if (leadingTest == null) {
+ return null;
+ }
+
+ BatchRunConfiguration leadingTestConfig = null;
+ if (requiredConfig != null) {
+ leadingTestConfig = requiredConfig;
+ } else {
+ for (BatchRunConfiguration runConfig : getTestRunConfigs(leadingTest)) {
+ if (mInstanceListerner.isPendingTestInstance(leadingTest, runConfig)) {
+ leadingTestConfig = runConfig;
+ break;
+ }
+ }
+ }
+
+ // test pending <=> test has a pending config
+ if (leadingTestConfig == null) {
+ throw new AssertionError("search postcondition failed");
+ }
+
+ final int leadingInstability = getTestInstabilityRating(leadingTest);
+
+ final TestBatch runBatch = new TestBatch();
+ runBatch.config = leadingTestConfig;
+ runBatch.tests = new ArrayList<>();
+ runBatch.tests.add(leadingTest);
+
+ for (TestIdentifier test : pool) {
+ if (test == leadingTest) {
+ // do not re-select the leading tests
+ continue;
+ }
+ if (!mInstanceListerner.isPendingTestInstance(test, leadingTestConfig)) {
+ // select only compatible
+ continue;
+ }
+ if (getTestInstabilityRating(test) != leadingInstability) {
+ // pack along only cases in the same stability category. Packing more dangerous
+ // tests along jeopardizes the stability of this run. Packing more stable tests
+ // along jeopardizes their stability rating.
+ continue;
+ }
+ if (runBatch.tests.size() >= getBatchSizeLimitForInstability(leadingInstability)) {
+ // batch size is limited.
+ break;
+ }
+ runBatch.tests.add(test);
+ }
+
+ return runBatch;
+ }
+
+ private int getBatchNumPendingCases(TestBatch batch) {
+ int numPending = 0;
+ for (TestIdentifier test : batch.tests) {
+ if (mInstanceListerner.isPendingTestInstance(test, batch.config)) {
+ ++numPending;
+ }
+ }
+ return numPending;
+ }
+
+ private int getBatchSizeLimitForInstability(int batchInstabilityRating) {
+ // reduce group size exponentially down to one
+ return Math.max(1, TESTCASE_BATCH_LIMIT / (1 << batchInstabilityRating));
+ }
+
+ private int getTestInstabilityRating(TestIdentifier testId) {
+ if (mTestInstabilityRatings.containsKey(testId)) {
+ return mTestInstabilityRatings.get(testId);
+ } else {
+ return 0;
+ }
+ }
+
+ private void recordTestInstability(TestIdentifier testId) {
+ mTestInstabilityRatings.put(testId, getTestInstabilityRating(testId) + 1);
+ }
+
+ private void clearTestInstability(TestIdentifier testId) {
+ mTestInstabilityRatings.put(testId, 0);
+ }
+
+ /**
+ * Executes all tests on the device.
+ */
+ private void runTests() throws DeviceNotAvailableException, CapabilityQueryFailureException {
+ for (;;) {
+ TestBatch batch = selectRunBatch(mRemainingTests, null);
+
+ if (batch == null) {
+ break;
+ }
+
+ runTestRunBatch(batch);
+ }
+ }
+
+ /**
+ * Runs a TestBatch by either faking it or executing it on a device.
+ */
+ private void runTestRunBatch(TestBatch batch) throws DeviceNotAvailableException,
+ CapabilityQueryFailureException {
+ // prepare instance listener
+ mInstanceListerner.setCurrentConfig(batch.config);
+ for (TestIdentifier test : batch.tests) {
+ mInstanceListerner.setTestInstances(test, getTestRunConfigs(test));
+ }
+
+ // execute only if config is executable, else fake results
+ if (isSupportedRunConfiguration(batch.config)) {
+ executeTestRunBatch(batch);
+ } else {
+ fakePassTestRunBatch(batch);
+ }
+ }
+
+ private boolean isSupportedRunConfiguration(BatchRunConfiguration runConfig)
+ throws DeviceNotAvailableException, CapabilityQueryFailureException {
+ // orientation support
+ if (!BatchRunConfiguration.ROTATION_UNSPECIFIED.equals(runConfig.getRotation())) {
+ final Set<String> features = getDeviceFeatures(mDevice);
+
+ if (isPortraitClassRotation(runConfig.getRotation()) &&
+ !features.contains(FEATURE_PORTRAIT)) {
+ return false;
+ }
+ if (isLandscapeClassRotation(runConfig.getRotation()) &&
+ !features.contains(FEATURE_LANDSCAPE)) {
+ return false;
+ }
+ }
+
+ if (isOpenGlEsPackage()) {
+ // renderability support for OpenGL ES tests
+ return isSupportedGlesRenderConfig(runConfig);
+ } else {
+ return true;
+ }
+ }
+
+ private static final class AdbComLinkOpenError extends Exception {
+ public AdbComLinkOpenError(String description, Throwable inner) {
+ super(description, inner);
+ }
+ }
+
+ private static final class AdbComLinkKilledError extends Exception {
+ public AdbComLinkKilledError(String description, Throwable inner) {
+ super(description, inner);
+ }
+ }
+
+ /**
+ * Executes a given command in adb shell
+ *
+ * @throws AdbComLinkOpenError if connection cannot be established.
+ * @throws AdbComLinkKilledError if established connection is killed prematurely.
+ */
+ private void executeShellCommandAndReadOutput(final String command,
+ final IShellOutputReceiver receiver)
+ throws AdbComLinkOpenError, AdbComLinkKilledError {
+ try {
+ mDevice.getIDevice().executeShellCommand(command, receiver,
+ UNRESPOSIVE_CMD_TIMEOUT_MS, TimeUnit.MILLISECONDS);
+ } catch (TimeoutException ex) {
+ // Opening connection timed out
+ throw new AdbComLinkOpenError("opening connection timed out", ex);
+ } catch (AdbCommandRejectedException ex) {
+ // Command rejected
+ throw new AdbComLinkOpenError("command rejected", ex);
+ } catch (IOException ex) {
+ // shell command channel killed
+ throw new AdbComLinkKilledError("command link killed", ex);
+ } catch (ShellCommandUnresponsiveException ex) {
+ // shell command halted
+ throw new AdbComLinkKilledError("command link hung", ex);
+ }
+ }
+
+ /**
+ * Executes given test batch on a device
+ */
+ private void executeTestRunBatch(TestBatch batch) throws DeviceNotAvailableException {
+ // attempt full run once
+ executeTestRunBatchRun(batch);
+
+ // split remaining tests to two sub batches and execute both. This will terminate
+ // since executeTestRunBatchRun will always progress for a batch of size 1.
+ final ArrayList<TestIdentifier> pendingTests = new ArrayList<>();
+
+ for (TestIdentifier test : batch.tests) {
+ if (mInstanceListerner.isPendingTestInstance(test, batch.config)) {
+ pendingTests.add(test);
+ }
+ }
+
+ final int divisorNdx = pendingTests.size() / 2;
+ final List<TestIdentifier> headList = pendingTests.subList(0, divisorNdx);
+ final List<TestIdentifier> tailList = pendingTests.subList(divisorNdx, pendingTests.size());
+
+ // head
+ for (;;) {
+ TestBatch subBatch = selectRunBatch(headList, batch.config);
+
+ if (subBatch == null) {
+ break;
+ }
+
+ executeTestRunBatch(subBatch);
+ }
+
+ // tail
+ for (;;) {
+ TestBatch subBatch = selectRunBatch(tailList, batch.config);
+
+ if (subBatch == null) {
+ break;
+ }
+
+ executeTestRunBatch(subBatch);
+ }
+
+ if (getBatchNumPendingCases(batch) != 0) {
+ throw new AssertionError("executeTestRunBatch postcondition failed");
+ }
+ }
+
+ /**
+ * Runs one execution pass over the given batch.
+ *
+ * Tries to run the batch. Always makes progress (executes instances or modifies stability
+ * scores).
+ */
+ private void executeTestRunBatchRun(TestBatch batch) throws DeviceNotAvailableException {
+ if (getBatchNumPendingCases(batch) != batch.tests.size()) {
+ throw new AssertionError("executeTestRunBatchRun precondition failed");
+ }
+
+ checkInterrupted(); // throws if interrupted
+
+ final String testCases = generateTestCaseTrie(batch.tests);
+
+ mDevice.executeShellCommand("rm " + CASE_LIST_FILE_NAME);
+ mDevice.executeShellCommand("rm " + LOG_FILE_NAME);
+ mDevice.pushString(testCases + "\n", CASE_LIST_FILE_NAME);
+
+ final String instrumentationName =
+ "com.drawelements.deqp/com.drawelements.deqp.testercore.DeqpInstrumentation";
+
+ final StringBuilder deqpCmdLine = new StringBuilder();
+ deqpCmdLine.append("--deqp-caselist-file=");
+ deqpCmdLine.append(CASE_LIST_FILE_NAME);
+ deqpCmdLine.append(" ");
+ deqpCmdLine.append(getRunConfigDisplayCmdLine(batch.config));
+
+ // If we are not logging data, do not bother outputting the images from the test exe.
+ if (!mLogData) {
+ deqpCmdLine.append(" --deqp-log-images=disable");
+ }
+
+ deqpCmdLine.append(" --deqp-watchdog=enable");
+
+ final String command = String.format(
+ "am instrument %s -w -e deqpLogFileName \"%s\" -e deqpCmdLine \"%s\""
+ + " -e deqpLogData \"%s\" %s",
+ AbiUtils.createAbiFlag(mAbi.getName()), LOG_FILE_NAME, deqpCmdLine.toString(),
+ mLogData, instrumentationName);
+
+ final int numRemainingInstancesBefore = getNumRemainingInstances();
+ final InstrumentationParser parser = new InstrumentationParser(mInstanceListerner);
+ Throwable interruptingError = null;
+
+ try {
+ executeShellCommandAndReadOutput(command, parser);
+ } catch (Throwable ex) {
+ interruptingError = ex;
+ } finally {
+ parser.flush();
+ }
+
+ final boolean progressedSinceLastCall = mInstanceListerner.getCurrentTestId() != null ||
+ getNumRemainingInstances() < numRemainingInstancesBefore;
+
+ if (progressedSinceLastCall) {
+ mDeviceRecovery.onExecutionProgressed();
+ }
+
+ // interrupted, try to recover
+ if (interruptingError != null) {
+ if (interruptingError instanceof AdbComLinkOpenError) {
+ mDeviceRecovery.recoverConnectionRefused();
+ } else if (interruptingError instanceof AdbComLinkKilledError) {
+ mDeviceRecovery.recoverComLinkKilled();
+ } else if (interruptingError instanceof RunInterruptedException) {
+ // external run interruption request. Terminate immediately.
+ throw (RunInterruptedException)interruptingError;
+ } else {
+ CLog.e(interruptingError);
+ throw new RuntimeException(interruptingError);
+ }
+
+ // recoverXXX did not throw => recovery succeeded
+ } else if (!parser.wasSuccessful()) {
+ mDeviceRecovery.recoverComLinkKilled();
+ // recoverXXX did not throw => recovery succeeded
+ }
+
+ // Progress guarantees.
+ if (batch.tests.size() == 1) {
+ final TestIdentifier onlyTest = batch.tests.iterator().next();
+ final boolean wasTestExecuted =
+ !mInstanceListerner.isPendingTestInstance(onlyTest, batch.config) &&
+ mInstanceListerner.getCurrentTestId() == null;
+ final boolean wasLinkFailure = !parser.wasSuccessful() || interruptingError != null;
+
+ // Link failures can be caused by external events, require at least two observations
+ // until bailing.
+ if (!wasTestExecuted && (!wasLinkFailure || getTestInstabilityRating(onlyTest) > 0)) {
+ recordTestInstability(onlyTest);
+ // If we cannot finish the test, mark the case as a crash.
+ //
+ // If we couldn't even start the test, fail the test instance as non-executable.
+ // This is required so that a consistently crashing or non-existent tests will
+ // not cause futile (non-terminating) re-execution attempts.
+ if (mInstanceListerner.getCurrentTestId() != null) {
+ mInstanceListerner.abortTest(onlyTest, INCOMPLETE_LOG_MESSAGE);
+ } else {
+ mInstanceListerner.abortTest(onlyTest, NOT_EXECUTABLE_LOG_MESSAGE);
+ }
+ } else if (wasTestExecuted) {
+ clearTestInstability(onlyTest);
+ }
+ }
+ else
+ {
+ // Analyze results to update test stability ratings. If there is no interrupting test
+ // logged, increase instability rating of all remaining tests. If there is a
+ // interrupting test logged, increase only its instability rating.
+ //
+ // A successful run of tests clears instability rating.
+ if (mInstanceListerner.getCurrentTestId() == null) {
+ for (TestIdentifier test : batch.tests) {
+ if (mInstanceListerner.isPendingTestInstance(test, batch.config)) {
+ recordTestInstability(test);
+ } else {
+ clearTestInstability(test);
+ }
+ }
+ } else {
+ recordTestInstability(mInstanceListerner.getCurrentTestId());
+ for (TestIdentifier test : batch.tests) {
+ // \note: isPendingTestInstance is false for getCurrentTestId. Current ID is
+ // considered 'running' and will be restored to 'pending' in endBatch().
+ if (!test.equals(mInstanceListerner.getCurrentTestId()) &&
+ !mInstanceListerner.isPendingTestInstance(test, batch.config)) {
+ clearTestInstability(test);
+ }
+ }
+ }
+ }
+
+ mInstanceListerner.endBatch();
+ }
+
+ private static String getRunConfigDisplayCmdLine(BatchRunConfiguration runConfig) {
+ final StringBuilder deqpCmdLine = new StringBuilder();
+ if (!runConfig.getGlConfig().isEmpty()) {
+ deqpCmdLine.append("--deqp-gl-config-name=");
+ deqpCmdLine.append(runConfig.getGlConfig());
+ }
+ if (!runConfig.getRotation().isEmpty()) {
+ if (deqpCmdLine.length() != 0) {
+ deqpCmdLine.append(" ");
+ }
+ deqpCmdLine.append("--deqp-screen-rotation=");
+ deqpCmdLine.append(runConfig.getRotation());
+ }
+ if (!runConfig.getSurfaceType().isEmpty()) {
+ if (deqpCmdLine.length() != 0) {
+ deqpCmdLine.append(" ");
+ }
+ deqpCmdLine.append("--deqp-surface-type=");
+ deqpCmdLine.append(runConfig.getSurfaceType());
+ }
+ return deqpCmdLine.toString();
+ }
+
+ private int getNumRemainingInstances() {
+ int retVal = 0;
+ for (TestIdentifier testId : mRemainingTests) {
+ // If case is in current working set, sum only not yet executed instances.
+ // If case is not in current working set, sum all instances (since they are not yet
+ // executed).
+ if (mInstanceListerner.mPendingResults.containsKey(testId)) {
+ retVal += mInstanceListerner.mPendingResults.get(testId).remainingConfigs.size();
+ } else {
+ retVal += mTestInstances.get(testId).size();
+ }
+ }
+ return retVal;
+ }
+
+ /**
+ * Checks if this execution has been marked as interrupted and throws if it has.
+ */
+ private void checkInterrupted() throws RunInterruptedException {
+ // Work around the API. RunUtil::checkInterrupted is private but we can call it indirectly
+ // by sleeping a value <= 0.
+ mRunUtil.sleep(0);
+ }
+
+ /**
+ * Pass given batch tests without running it
+ */
+ private void fakePassTestRunBatch(TestBatch batch) {
+ for (TestIdentifier test : batch.tests) {
+ CLog.d("Skipping test '%s' invocation in config '%s'", test.toString(),
+ batch.config.getId());
+ mInstanceListerner.skipTest(test);
+ }
+ }
+
+ /**
+ * Pass all remaining tests without running them
+ */
+ private void fakePassTests(ITestInvocationListener listener) {
+ Map <String, String> emptyMap = Collections.emptyMap();
+ for (TestIdentifier test : mRemainingTests) {
+ CLog.d("Skipping test '%s', Opengl ES version not supported", test.toString());
+ listener.testStarted(test);
+ listener.testEnded(test, emptyMap);
+ }
+ mRemainingTests.clear();
+ }
+
+ /**
+ * Check if device supports OpenGL ES version.
+ */
+ private static boolean isSupportedGles(ITestDevice device, int requiredMajorVersion,
+ int requiredMinorVersion) throws DeviceNotAvailableException {
+ String roOpenglesVersion = device.getProperty("ro.opengles.version");
+
+ if (roOpenglesVersion == null)
+ return false;
+
+ int intValue = Integer.parseInt(roOpenglesVersion);
+
+ int majorVersion = ((intValue & 0xffff0000) >> 16);
+ int minorVersion = (intValue & 0xffff);
+
+ return (majorVersion > requiredMajorVersion)
+ || (majorVersion == requiredMajorVersion && minorVersion >= requiredMinorVersion);
+ }
+
+ /**
+ * Query if rendertarget is supported
+ */
+ private boolean isSupportedGlesRenderConfig(BatchRunConfiguration runConfig)
+ throws DeviceNotAvailableException, CapabilityQueryFailureException {
+ // query if configuration is supported
+ final StringBuilder configCommandLine =
+ new StringBuilder(getRunConfigDisplayCmdLine(runConfig));
+ if (configCommandLine.length() != 0) {
+ configCommandLine.append(" ");
+ }
+ configCommandLine.append("--deqp-gl-major-version=");
+ configCommandLine.append(getGlesMajorVersion());
+ configCommandLine.append(" --deqp-gl-minor-version=");
+ configCommandLine.append(getGlesMinorVersion());
+
+ final String commandLine = configCommandLine.toString();
+
+ // check for cached result first
+ if (mConfigQuerySupportCache.containsKey(commandLine)) {
+ return mConfigQuerySupportCache.get(commandLine);
+ }
+
+ final boolean supported = queryIsSupportedConfigCommandLine(commandLine);
+ mConfigQuerySupportCache.put(commandLine, supported);
+ return supported;
+ }
+
+ private boolean queryIsSupportedConfigCommandLine(String deqpCommandLine)
+ throws DeviceNotAvailableException, CapabilityQueryFailureException {
+ final String instrumentationName =
+ "com.drawelements.deqp/com.drawelements.deqp.platformutil.DeqpPlatformCapabilityQueryInstrumentation";
+ final String command = String.format(
+ "am instrument %s -w -e deqpQueryType renderConfigSupported -e deqpCmdLine \"%s\""
+ + " %s",
+ AbiUtils.createAbiFlag(mAbi.getName()), deqpCommandLine, instrumentationName);
+
+ final PlatformQueryInstrumentationParser parser = new PlatformQueryInstrumentationParser();
+ mDevice.executeShellCommand(command, parser);
+ parser.flush();
+
+ if (parser.wasSuccessful() && parser.getResultCode() == 0 &&
+ parser.getResultMap().containsKey("Supported")) {
+ if ("Yes".equals(parser.getResultMap().get("Supported"))) {
+ return true;
+ } else if ("No".equals(parser.getResultMap().get("Supported"))) {
+ return false;
+ } else {
+ CLog.e("Capability query did not return a result");
+ throw new CapabilityQueryFailureException();
+ }
+ } else if (parser.wasSuccessful()) {
+ CLog.e("Failed to run capability query. Code: %d, Result: %s",
+ parser.getResultCode(), parser.getResultMap().toString());
+ throw new CapabilityQueryFailureException();
+ } else {
+ CLog.e("Failed to run capability query");
+ throw new CapabilityQueryFailureException();
+ }
+ }
+
+ /**
+ * Return feature set supported by the device
+ */
+ private Set<String> getDeviceFeatures(ITestDevice device)
+ throws DeviceNotAvailableException, CapabilityQueryFailureException {
+ if (mDeviceFeatures == null) {
+ mDeviceFeatures = queryDeviceFeatures(device);
+ }
+ return mDeviceFeatures;
+ }
+
+ /**
+ * Query feature set supported by the device
+ */
+ private static Set<String> queryDeviceFeatures(ITestDevice device)
+ throws DeviceNotAvailableException, CapabilityQueryFailureException {
+ // NOTE: Almost identical code in BaseDevicePolicyTest#hasDeviceFeatures
+ // TODO: Move this logic to ITestDevice.
+ String command = "pm list features";
+ String commandOutput = device.executeShellCommand(command);
+
+ // Extract the id of the new user.
+ HashSet<String> availableFeatures = new HashSet<>();
+ for (String feature: commandOutput.split("\\s+")) {
+ // Each line in the output of the command has the format "feature:{FEATURE_VALUE}".
+ String[] tokens = feature.split(":");
+ if (tokens.length < 2 || !"feature".equals(tokens[0])) {
+ CLog.e("Failed parse features. Unexpect format on line \"%s\"", tokens[0]);
+ throw new CapabilityQueryFailureException();
+ }
+ availableFeatures.add(tokens[1]);
+ }
+ return availableFeatures;
+ }
+
+ private boolean isPortraitClassRotation(String rotation) {
+ return BatchRunConfiguration.ROTATION_PORTRAIT.equals(rotation) ||
+ BatchRunConfiguration.ROTATION_REVERSE_PORTRAIT.equals(rotation);
+ }
+
+ private boolean isLandscapeClassRotation(String rotation) {
+ return BatchRunConfiguration.ROTATION_LANDSCAPE.equals(rotation) ||
+ BatchRunConfiguration.ROTATION_REVERSE_LANDSCAPE.equals(rotation);
+ }
+
+ /**
+ * Install dEQP OnDevice Package
+ */
+ private void installTestApk() throws DeviceNotAvailableException {
+ try {
+ File apkFile = new File(mBuildHelper.getTestsDir(), DEQP_ONDEVICE_APK);
+ String[] options = {AbiUtils.createAbiFlag(mAbi.getName())};
+ String errorCode = getDevice().installPackage(apkFile, true, options);
+ if (errorCode != null) {
+ CLog.e("Failed to install %s. Reason: %s", DEQP_ONDEVICE_APK, errorCode);
+ }
+ } catch (FileNotFoundException e) {
+ CLog.e("Could not find test apk %s", DEQP_ONDEVICE_APK);
+ }
+ }
+
+ /**
+ * Uninstall dEQP OnDevice Package
+ */
+ private void uninstallTestApk() throws DeviceNotAvailableException {
+ getDevice().uninstallPackage(DEQP_ONDEVICE_PKG);
+ }
+
+ /**
+ * Parse gl nature from package name
+ */
+ private boolean isOpenGlEsPackage() {
+ if ("dEQP-GLES2".equals(mDeqpPackage) || "dEQP-GLES3".equals(mDeqpPackage) ||
+ "dEQP-GLES31".equals(mDeqpPackage)) {
+ return true;
+ } else if ("dEQP-EGL".equals(mDeqpPackage)) {
+ return false;
+ } else {
+ throw new IllegalStateException("dEQP runner was created with illegal name");
+ }
+ }
+
+ /**
+ * Check GL support (based on package name)
+ */
+ private boolean isSupportedGles() throws DeviceNotAvailableException {
+ return isSupportedGles(mDevice, getGlesMajorVersion(), getGlesMinorVersion());
+ }
+
+ /**
+ * Get GL major version (based on package name)
+ */
+ private int getGlesMajorVersion() throws DeviceNotAvailableException {
+ if ("dEQP-GLES2".equals(mDeqpPackage)) {
+ return 2;
+ } else if ("dEQP-GLES3".equals(mDeqpPackage)) {
+ return 3;
+ } else if ("dEQP-GLES31".equals(mDeqpPackage)) {
+ return 3;
+ } else {
+ throw new IllegalStateException("getGlesMajorVersion called for non gles pkg");
+ }
+ }
+
+ /**
+ * Get GL minor version (based on package name)
+ */
+ private int getGlesMinorVersion() throws DeviceNotAvailableException {
+ if ("dEQP-GLES2".equals(mDeqpPackage)) {
+ return 0;
+ } else if ("dEQP-GLES3".equals(mDeqpPackage)) {
+ return 0;
+ } else if ("dEQP-GLES31".equals(mDeqpPackage)) {
+ return 1;
+ } else {
+ throw new IllegalStateException("getGlesMinorVersion called for non gles pkg");
+ }
+ }
+
+ private static List<Pattern> buildPatternList(List<String> filters) {
+ List<Pattern> patterns = new ArrayList(filters.size());
+ for (String filter : filters) {
+ patterns.add(Pattern.compile(filter.replace(".","\\.").replace("*",".*")));
+ }
+ return patterns;
+ }
+
+ private static boolean matchesAny(TestIdentifier test, List<Pattern> patterns) {
+ for (Pattern pattern : patterns) {
+ if (pattern.matcher(test.toString()).matches()) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /**
+ * Filter tests.
+ *
+ * '*' is 0 or more characters. '.' is interpreted verbatim.
+ *
+ * Optimized for small number of filters against many tests.
+ *
+ */
+ private static void filterTests(Map<TestIdentifier, Set<BatchRunConfiguration>> tests,
+ List<String> includeFilters,
+ List<String> excludeFilters) {
+ // We could filter faster by building the test case tree.
+ // Let's see if this is fast enough.
+ List<Pattern> includes = buildPatternList(includeFilters);
+ List<Pattern> excludes = buildPatternList(excludeFilters);
+
+ List<TestIdentifier> testList = new ArrayList(tests.keySet());
+ for (TestIdentifier test : testList) {
+ // Remove test if it does not match includes or matches
+ // excludes.
+ // Empty include filter includes everything.
+ if (includes.isEmpty() || matchesAny(test, includes)) {
+ if (!matchesAny(test, excludes)) {
+ continue;
+ }
+ }
+ tests.remove(test);
+ }
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void run(ITestInvocationListener listener) throws DeviceNotAvailableException {
+ final Map<String, String> emptyMap = Collections.emptyMap();
+ final boolean isSupportedApi = !isOpenGlEsPackage() || isSupportedGles();
+
+ try {
+ Reader reader = mCaselistReader;
+ if (reader == null) {
+ File testlist = new File(mBuildHelper.getTestsDir(), mCaselistFile);
+ if (!testlist.isFile()) {
+ throw new FileNotFoundException();
+ }
+ reader = new FileReader(testlist);
+ }
+ mTestInstances = generateTestInstances(reader, mConfigName, mScreenRotation, mSurfaceType);
+ mCaselistReader = null;
+ reader.close();
+ }
+ catch (FileNotFoundException e) {
+ throw new RuntimeException("Cannot read deqp test list file: " + mCaselistFile);
+ }
+ catch (IOException e) {
+ CLog.w("Failed to close test list reader.");
+ }
+ if (!mIncludeFilters.isEmpty() || !mExcludeFilters.isEmpty()) {
+ filterTests(mTestInstances, mIncludeFilters, mExcludeFilters);
+ }
+ mRemainingTests = new LinkedList<>(mTestInstances.keySet());
+
+ listener.testRunStarted(getId(), mRemainingTests.size());
+
+ try {
+ if (isSupportedApi) {
+ // Make sure there is no pre-existing package form earlier interrupted test run.
+ uninstallTestApk();
+ installTestApk();
+
+ mInstanceListerner.setSink(listener);
+ mDeviceRecovery.setDevice(mDevice);
+ runTests();
+
+ uninstallTestApk();
+ } else {
+ // Pass all tests if OpenGL ES version is not supported
+ fakePassTests(listener);
+ }
+ } catch (CapabilityQueryFailureException ex) {
+ // Platform is not behaving correctly, for example crashing when trying to create
+ // a window. Instead of silenty failing, signal failure by leaving the rest of the
+ // test cases in "NotExecuted" state
+ uninstallTestApk();
+ }
+
+ listener.testRunEnded(0, emptyMap);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void addIncludeFilter(String filter) {
+ mIncludeFilters.add(filter);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void addAllIncludeFilters(List<String> filters) {
+ mIncludeFilters.addAll(filters);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void addExcludeFilter(String filter) {
+ mExcludeFilters.add(filter);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void addAllExcludeFilters(List<String> filters) {
+ mExcludeFilters.addAll(filters);
+ }
+}
diff --git a/android/cts/runner/tests/Android.mk b/android/cts/runner/tests/Android.mk
new file mode 100644
index 0000000..b4a3df2
--- /dev/null
+++ b/android/cts/runner/tests/Android.mk
@@ -0,0 +1,27 @@
+# Copyright (C) 2010 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+LOCAL_PATH := $(call my-dir)
+
+include $(CLEAR_VARS)
+
+# Only compile source java files in this lib.
+LOCAL_SRC_FILES := $(call all-java-files-under, src)
+
+LOCAL_MODULE := CtsDeqpRunnerTests
+LOCAL_MODULE_TAGS := optional
+LOCAL_JAVA_LIBRARIES := cts-tradefed_v2 compatibility-host-util tradefed-prebuilt CtsDeqp
+LOCAL_STATIC_JAVA_LIBRARIES := easymock
+
+include $(BUILD_HOST_JAVA_LIBRARY)
diff --git a/android/cts/runner/tests/run_tests.sh b/android/cts/runner/tests/run_tests.sh
new file mode 100755
index 0000000..7293894
--- /dev/null
+++ b/android/cts/runner/tests/run_tests.sh
@@ -0,0 +1,62 @@
+#!/bin/bash
+
+# Copyright (C) 2015 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Helper script for running unit tests for compatibility libraries
+
+checkFile() {
+ if [ ! -f "$1" ]; then
+ echo "Unable to locate $1"
+ exit
+ fi;
+}
+
+# check if in Android build env
+if [ ! -z ${ANDROID_BUILD_TOP} ]; then
+ HOST=`uname`
+ if [ "$HOST" == "Linux" ]; then
+ OS="linux-x86"
+ elif [ "$HOST" == "Darwin" ]; then
+ OS="darwin-x86"
+ else
+ echo "Unrecognized OS"
+ exit
+ fi;
+fi;
+
+JAR_DIR=${ANDROID_HOST_OUT}/framework
+TF_CONSOLE=com.android.tradefed.command.Console
+
+############### Run the cts tests ###############
+JARS="
+ compatibility-host-util\
+ cts-tradefed_v2\
+ ddmlib-prebuilt\
+ hosttestlib\
+ CtsDeqp\
+ CtsDeqpRunnerTests\
+ tradefed-prebuilt"
+JAR_PATH=
+for JAR in $JARS; do
+ checkFile ${JAR_DIR}/${JAR}.jar
+ JAR_PATH=${JAR_PATH}:${JAR_DIR}/${JAR}.jar
+done
+
+TEST_CLASSES="
+ com.drawelements.deqp.runner.DeqpTestRunnerTest"
+
+for CLASS in ${TEST_CLASSES}; do
+ java $RDBG_FLAG -cp ${JAR_PATH} ${TF_CONSOLE} run singleCommand host -n --class ${CLASS} "$@"
+done
diff --git a/android/cts/runner/tests/src/com/drawelements/deqp/runner/DeqpTestRunnerTest.java b/android/cts/runner/tests/src/com/drawelements/deqp/runner/DeqpTestRunnerTest.java
new file mode 100644
index 0000000..a84eef5
--- /dev/null
+++ b/android/cts/runner/tests/src/com/drawelements/deqp/runner/DeqpTestRunnerTest.java
@@ -0,0 +1,1920 @@
+/*
+ * Copyright (C) 2014 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.drawelements.deqp.runner;
+
+import com.android.compatibility.common.tradefed.build.CompatibilityBuildHelper;
+import com.android.compatibility.common.tradefed.testtype.Abi;
+import com.android.compatibility.common.util.AbiUtils;
+import com.android.ddmlib.IDevice;
+import com.android.ddmlib.IShellOutputReceiver;
+import com.android.ddmlib.ShellCommandUnresponsiveException;
+import com.android.ddmlib.testrunner.TestIdentifier;
+import com.android.tradefed.build.IFolderBuildInfo;
+import com.android.tradefed.config.ConfigurationException;
+import com.android.tradefed.config.OptionSetter;
+import com.android.tradefed.device.DeviceNotAvailableException;
+import com.android.tradefed.device.ITestDevice;
+import com.android.tradefed.result.ITestInvocationListener;
+import com.android.tradefed.testtype.IAbi;
+import com.android.tradefed.util.IRunUtil;
+import com.android.tradefed.util.RunInterruptedException;
+
+import junit.framework.TestCase;
+
+import org.easymock.EasyMock;
+import org.easymock.IAnswer;
+import org.easymock.IMocksControl;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.StringReader;
+import java.io.StringWriter;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Unit tests for {@link DeqpTestRunner}.
+ */
+public class DeqpTestRunnerTest extends TestCase {
+ private static final String NAME = "dEQP-GLES3";
+ private static final IAbi ABI = new Abi("armeabi-v7a", "32");
+ private static final String CASE_LIST_FILE_NAME = "/sdcard/dEQP-TestCaseList.txt";
+ private static final String LOG_FILE_NAME = "/sdcard/TestLog.qpa";
+ private static final String INSTRUMENTATION_NAME =
+ "com.drawelements.deqp/com.drawelements.deqp.testercore.DeqpInstrumentation";
+ private static final String QUERY_INSTRUMENTATION_NAME =
+ "com.drawelements.deqp/com.drawelements.deqp.platformutil.DeqpPlatformCapabilityQueryInstrumentation";
+ private static final String DEQP_ONDEVICE_APK = "com.drawelements.deqp.apk";
+ private static final String DEQP_ONDEVICE_PKG = "com.drawelements.deqp";
+ private static final String ONLY_LANDSCAPE_FEATURES =
+ "feature:"+DeqpTestRunner.FEATURE_LANDSCAPE;
+ private static final String ALL_FEATURES =
+ ONLY_LANDSCAPE_FEATURES + "\nfeature:"+DeqpTestRunner.FEATURE_PORTRAIT;
+ private static List<Map<String,String>> DEFAULT_INSTANCE_ARGS;
+
+ static {
+ DEFAULT_INSTANCE_ARGS = new ArrayList<>(1);
+ DEFAULT_INSTANCE_ARGS.add(new HashMap<String,String>());
+ DEFAULT_INSTANCE_ARGS.iterator().next().put("glconfig", "rgba8888d24s8");
+ DEFAULT_INSTANCE_ARGS.iterator().next().put("rotation", "unspecified");
+ DEFAULT_INSTANCE_ARGS.iterator().next().put("surfacetype", "window");
+ }
+
+ private static class StubRecovery implements DeqpTestRunner.IRecovery {
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void setSleepProvider(DeqpTestRunner.ISleepProvider sleepProvider) {
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void setDevice(ITestDevice device) {
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void onExecutionProgressed() {
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void recoverConnectionRefused() throws DeviceNotAvailableException {
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void recoverComLinkKilled() throws DeviceNotAvailableException {
+ }
+ };
+
+ public static class BuildHelperMock extends CompatibilityBuildHelper {
+ public BuildHelperMock(IFolderBuildInfo buildInfo) {
+ super(buildInfo);
+ }
+ @Override
+ public File getTestsDir() throws FileNotFoundException {
+ return new File("logs");
+ }
+ }
+
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected void setUp() throws Exception {
+ super.setUp();
+ }
+
+ private static DeqpTestRunner buildGlesTestRunner(int majorVersion,
+ int minorVersion,
+ Collection<TestIdentifier> tests) throws ConfigurationException, FileNotFoundException {
+ StringWriter testlist = new StringWriter();
+ for (TestIdentifier test : tests) {
+ testlist.write(test.getClassName() + "." + test.getTestName() + "\n");
+ }
+ return buildGlesTestRunner(majorVersion, minorVersion, testlist.toString());
+ }
+
+ private static DeqpTestRunner buildGlesTestRunner(int majorVersion,
+ int minorVersion,
+ String testlist) throws ConfigurationException, FileNotFoundException {
+ DeqpTestRunner runner = new DeqpTestRunner();
+ OptionSetter setter = new OptionSetter(runner);
+
+ String deqpPackage = "dEQP-GLES" + Integer.toString(majorVersion)
+ + (minorVersion > 0 ? Integer.toString(minorVersion) : "");
+
+ setter.setOptionValue("deqp-package", deqpPackage);
+ setter.setOptionValue("deqp-gl-config-name", "rgba8888d24s8");
+ setter.setOptionValue("deqp-caselist-file", "dummyfile.txt");
+ setter.setOptionValue("deqp-screen-rotation", "unspecified");
+ setter.setOptionValue("deqp-surface-type", "window");
+
+ runner.setCaselistReader(new StringReader(testlist));
+
+ runner.setAbi(ABI);
+
+ IFolderBuildInfo mockIFolderBuildInfo = EasyMock.createMock(IFolderBuildInfo.class);
+ EasyMock.replay(mockIFolderBuildInfo);
+ CompatibilityBuildHelper mockHelper = new BuildHelperMock(mockIFolderBuildInfo);
+ runner.setBuildHelper(mockHelper);
+
+ return runner;
+ }
+
+ private static String getTestId(DeqpTestRunner runner) {
+ return AbiUtils.createId(ABI.getName(), runner.getPackageName());
+ }
+
+ /**
+ * Test version of OpenGL ES.
+ */
+ private void testGlesVersion(int requiredMajorVersion, int requiredMinorVersion, int majorVersion, int minorVersion) throws Exception {
+ final TestIdentifier testId = new TestIdentifier("dEQP-GLES"
+ + Integer.toString(requiredMajorVersion) + Integer.toString(requiredMinorVersion)
+ + ".info", "version");
+
+ final String testPath = "dEQP-GLES"
+ + Integer.toString(requiredMajorVersion) + Integer.toString(requiredMinorVersion)
+ +".info.version";
+
+ final String testTrie = "{dEQP-GLES"
+ + Integer.toString(requiredMajorVersion) + Integer.toString(requiredMinorVersion)
+ + "{info{version}}}";
+
+ final String resultCode = "Pass";
+
+ /* MultiLineReceiver expects "\r\n" line ending. */
+ final String output = "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=" + testPath + "\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=" + resultCode + "\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Detail" + resultCode + "\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_CODE: 0\r\n";
+
+ ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class);
+ ITestInvocationListener mockListener
+ = EasyMock.createStrictMock(ITestInvocationListener.class);
+ IDevice mockIDevice = EasyMock.createMock(IDevice.class);
+ Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>();
+ tests.add(testId);
+
+ DeqpTestRunner deqpTest = buildGlesTestRunner(requiredMajorVersion, requiredMinorVersion, tests);
+
+ int version = (majorVersion << 16) | minorVersion;
+ EasyMock.expect(mockDevice.getProperty("ro.opengles.version"))
+ .andReturn(Integer.toString(version)).atLeastOnce();
+
+ if (majorVersion > requiredMajorVersion
+ || (majorVersion == requiredMajorVersion && minorVersion >= requiredMinorVersion)) {
+
+ EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG)))
+ .andReturn("").once();
+ EasyMock.expect(mockDevice.installPackage(EasyMock.<File>anyObject(),
+ EasyMock.eq(true),
+ EasyMock.eq(AbiUtils.createAbiFlag(ABI.getName()))))
+ .andReturn(null).once();
+
+ expectRenderConfigQuery(mockDevice, requiredMajorVersion,
+ requiredMinorVersion);
+
+ String commandLine = String.format(
+ "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 "
+ + "--deqp-screen-rotation=unspecified "
+ + "--deqp-surface-type=window "
+ + "--deqp-log-images=disable "
+ + "--deqp-watchdog=enable",
+ CASE_LIST_FILE_NAME);
+
+ runInstrumentationLineAndAnswer(mockDevice, mockIDevice, testTrie, commandLine,
+ output);
+
+ EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG)))
+ .andReturn("").once();
+ }
+
+ mockListener.testRunStarted(getTestId(deqpTest), 1);
+ EasyMock.expectLastCall().once();
+
+ mockListener.testStarted(EasyMock.eq(testId));
+ EasyMock.expectLastCall().once();
+
+ mockListener.testEnded(EasyMock.eq(testId), EasyMock.<Map<String, String>>notNull());
+ EasyMock.expectLastCall().once();
+
+ mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<Map<String, String>>notNull());
+ EasyMock.expectLastCall().once();
+
+ EasyMock.replay(mockDevice, mockIDevice);
+ EasyMock.replay(mockListener);
+
+ deqpTest.setDevice(mockDevice);
+ deqpTest.run(mockListener);
+
+ EasyMock.verify(mockListener);
+ EasyMock.verify(mockDevice, mockIDevice);
+ }
+
+ private void expectRenderConfigQuery(ITestDevice mockDevice, int majorVersion,
+ int minorVersion) throws Exception {
+ expectRenderConfigQuery(mockDevice,
+ String.format("--deqp-gl-config-name=rgba8888d24s8 "
+ + "--deqp-screen-rotation=unspecified "
+ + "--deqp-surface-type=window "
+ + "--deqp-gl-major-version=%d "
+ + "--deqp-gl-minor-version=%d", majorVersion, minorVersion));
+ }
+
+ private void expectRenderConfigQuery(ITestDevice mockDevice, String commandLine)
+ throws Exception {
+ expectRenderConfigQueryAndReturn(mockDevice, commandLine, "Yes");
+ }
+
+ private void expectRenderConfigQueryAndReturn(ITestDevice mockDevice, String commandLine,
+ String output) throws Exception {
+ final String queryOutput = "INSTRUMENTATION_RESULT: Supported=" + output + "\r\n"
+ + "INSTRUMENTATION_CODE: 0\r\n";
+ final String command = String.format(
+ "am instrument %s -w -e deqpQueryType renderConfigSupported -e deqpCmdLine "
+ + "\"%s\" %s",
+ AbiUtils.createAbiFlag(ABI.getName()), commandLine,
+ QUERY_INSTRUMENTATION_NAME);
+
+ mockDevice.executeShellCommand(EasyMock.eq(command),
+ EasyMock.<IShellOutputReceiver>notNull());
+
+ EasyMock.expectLastCall().andAnswer(new IAnswer<Object>() {
+ @Override
+ public Object answer() {
+ IShellOutputReceiver receiver
+ = (IShellOutputReceiver)EasyMock.getCurrentArguments()[1];
+
+ receiver.addOutput(queryOutput.getBytes(), 0, queryOutput.length());
+ receiver.flush();
+
+ return null;
+ }
+ });
+ }
+
+ /**
+ * Test that result code produces correctly pass or fail.
+ */
+ private void testResultCode(final String resultCode, boolean pass) throws Exception {
+ final TestIdentifier testId = new TestIdentifier("dEQP-GLES3.info", "version");
+ final String testPath = "dEQP-GLES3.info.version";
+ final String testTrie = "{dEQP-GLES3{info{version}}}";
+
+ /* MultiLineReceiver expects "\r\n" line ending. */
+ final String output = "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=" + testPath + "\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=" + resultCode + "\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Detail" + resultCode + "\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_CODE: 0\r\n";
+
+ ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class);
+ ITestInvocationListener mockListener
+ = EasyMock.createStrictMock(ITestInvocationListener.class);
+ IDevice mockIDevice = EasyMock.createMock(IDevice.class);
+
+ Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>();
+ tests.add(testId);
+
+ DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests);
+
+ int version = 3 << 16;
+ EasyMock.expect(mockDevice.getProperty("ro.opengles.version"))
+ .andReturn(Integer.toString(version)).atLeastOnce();
+
+ EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).andReturn("")
+ .once();
+
+ EasyMock.expect(mockDevice.installPackage(EasyMock.<File>anyObject(),
+ EasyMock.eq(true), EasyMock.eq(AbiUtils.createAbiFlag(ABI.getName()))))
+ .andReturn(null).once();
+
+ expectRenderConfigQuery(mockDevice, 3, 0);
+
+ String commandLine = String.format(
+ "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 "
+ + "--deqp-screen-rotation=unspecified "
+ + "--deqp-surface-type=window "
+ + "--deqp-log-images=disable "
+ + "--deqp-watchdog=enable",
+ CASE_LIST_FILE_NAME);
+
+ runInstrumentationLineAndAnswer(mockDevice, mockIDevice, testTrie, commandLine, output);
+
+ mockListener.testRunStarted(getTestId(deqpTest), 1);
+ EasyMock.expectLastCall().once();
+
+ mockListener.testStarted(EasyMock.eq(testId));
+ EasyMock.expectLastCall().once();
+
+ if (!pass) {
+ mockListener.testFailed(testId,
+ "=== with config {glformat=rgba8888d24s8,rotation=unspecified,surfacetype=window} ===\n"
+ + resultCode + ": Detail" + resultCode);
+
+ EasyMock.expectLastCall().once();
+ }
+
+ mockListener.testEnded(EasyMock.eq(testId), EasyMock.<Map<String, String>>notNull());
+ EasyMock.expectLastCall().once();
+
+ mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<Map<String, String>>notNull());
+ EasyMock.expectLastCall().once();
+
+ EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).andReturn("")
+ .once();
+
+ EasyMock.replay(mockDevice, mockIDevice);
+ EasyMock.replay(mockListener);
+
+ deqpTest.setDevice(mockDevice);
+ deqpTest.run(mockListener);
+
+ EasyMock.verify(mockListener);
+ EasyMock.verify(mockDevice, mockIDevice);
+ }
+
+ /**
+ * Test running multiple test cases.
+ */
+ public void testRun_multipleTests() throws Exception {
+ /* MultiLineReceiver expects "\r\n" line ending. */
+ final String output = "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.vendor\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.renderer\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.version\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.shading_language_version\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.extensions\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.render_target\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_CODE: 0\r\n";
+
+ final TestIdentifier[] testIds = {
+ new TestIdentifier("dEQP-GLES3.info", "vendor"),
+ new TestIdentifier("dEQP-GLES3.info", "renderer"),
+ new TestIdentifier("dEQP-GLES3.info", "version"),
+ new TestIdentifier("dEQP-GLES3.info", "shading_language_version"),
+ new TestIdentifier("dEQP-GLES3.info", "extensions"),
+ new TestIdentifier("dEQP-GLES3.info", "render_target")
+ };
+
+ final String[] testPaths = {
+ "dEQP-GLES3.info.vendor",
+ "dEQP-GLES3.info.renderer",
+ "dEQP-GLES3.info.version",
+ "dEQP-GLES3.info.shading_language_version",
+ "dEQP-GLES3.info.extensions",
+ "dEQP-GLES3.info.render_target"
+ };
+
+ final String testTrie
+ = "{dEQP-GLES3{info{vendor,renderer,version,shading_language_version,extensions,render_target}}}";
+
+ ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class);
+ ITestInvocationListener mockListener
+ = EasyMock.createStrictMock(ITestInvocationListener.class);
+ IDevice mockIDevice = EasyMock.createMock(IDevice.class);
+
+ Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>();
+
+ for (TestIdentifier id : testIds) {
+ tests.add(id);
+ }
+
+ DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests);
+
+ int version = 3 << 16;
+ EasyMock.expect(mockDevice.getProperty("ro.opengles.version"))
+ .andReturn(Integer.toString(version)).atLeastOnce();
+
+ EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).andReturn("")
+ .once();
+ EasyMock.expect(mockDevice.installPackage(EasyMock.<File>anyObject(),
+ EasyMock.eq(true), EasyMock.eq(AbiUtils.createAbiFlag(ABI.getName()))))
+ .andReturn(null).once();
+
+ expectRenderConfigQuery(mockDevice, 3, 0);
+
+ String commandLine = String.format(
+ "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 "
+ + "--deqp-screen-rotation=unspecified "
+ + "--deqp-surface-type=window "
+ + "--deqp-log-images=disable "
+ + "--deqp-watchdog=enable",
+ CASE_LIST_FILE_NAME);
+
+ runInstrumentationLineAndAnswer(mockDevice, mockIDevice, testTrie, commandLine, output);
+
+ mockListener.testRunStarted(getTestId(deqpTest), testPaths.length);
+ EasyMock.expectLastCall().once();
+
+ for (int i = 0; i < testPaths.length; i++) {
+ mockListener.testStarted(EasyMock.eq(testIds[i]));
+ EasyMock.expectLastCall().once();
+
+ mockListener.testEnded(EasyMock.eq(testIds[i]),
+ EasyMock.<Map<String, String>>notNull());
+
+ EasyMock.expectLastCall().once();
+ }
+
+ mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<Map<String, String>>notNull());
+ EasyMock.expectLastCall().once();
+
+ EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).andReturn("")
+ .once();
+
+ EasyMock.replay(mockDevice, mockIDevice);
+ EasyMock.replay(mockListener);
+
+ deqpTest.setDevice(mockDevice);
+ deqpTest.run(mockListener);
+
+ EasyMock.verify(mockListener);
+ EasyMock.verify(mockDevice, mockIDevice);
+ }
+
+ private void testFiltering(List<String> includes,
+ List<String> excludes,
+ List<TestIdentifier> fullTestList,
+ String expectedTrie,
+ List<TestIdentifier> expectedTests) throws Exception{
+
+ /* MultiLineReceiver expects "\r\n" line ending. */
+ final String outputHeader = "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n";
+
+ final String outputEnd = "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_CODE: 0\r\n";
+
+ StringWriter output = new StringWriter();
+ output.write(outputHeader);
+ for (TestIdentifier test : expectedTests) {
+ output.write("INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n");
+ output.write("INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=");
+ output.write(test.getClassName());
+ output.write(".");
+ output.write(test.getTestName());
+ output.write("\r\n");
+ output.write("INSTRUMENTATION_STATUS_CODE: 0\r\n");
+ output.write("INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n");
+ output.write("INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n");
+ output.write("INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n");
+ output.write("INSTRUMENTATION_STATUS_CODE: 0\r\n");
+ output.write("INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n");
+ output.write("INSTRUMENTATION_STATUS_CODE: 0\r\n");
+ }
+ output.write(outputEnd);
+
+ ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class);
+ ITestInvocationListener mockListener
+ = EasyMock.createStrictMock(ITestInvocationListener.class);
+ IDevice mockIDevice = EasyMock.createMock(IDevice.class);
+
+ DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, fullTestList);
+ if (includes != null) {
+ deqpTest.addAllIncludeFilters(includes);
+ }
+ if (excludes != null) {
+ deqpTest.addAllExcludeFilters(excludes);
+ }
+
+ int version = 3 << 16;
+ EasyMock.expect(mockDevice.getProperty("ro.opengles.version"))
+ .andReturn(Integer.toString(version)).atLeastOnce();
+
+ EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).andReturn("")
+ .once();
+ EasyMock.expect(mockDevice.installPackage(EasyMock.<File>anyObject(),
+ EasyMock.eq(true), EasyMock.eq(AbiUtils.createAbiFlag(ABI.getName()))))
+ .andReturn(null).once();
+
+ mockListener.testRunStarted(getTestId(deqpTest), expectedTests.size());
+ EasyMock.expectLastCall().once();
+
+ if (expectedTests.size() > 0)
+ {
+ expectRenderConfigQuery(mockDevice, 3, 0);
+
+ String commandLine = String.format(
+ "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 "
+ + "--deqp-screen-rotation=unspecified "
+ + "--deqp-surface-type=window "
+ + "--deqp-log-images=disable "
+ + "--deqp-watchdog=enable",
+ CASE_LIST_FILE_NAME);
+
+ runInstrumentationLineAndAnswer(mockDevice, mockIDevice, expectedTrie, commandLine, output.toString());
+
+ for (int i = 0; i < expectedTests.size(); i++) {
+ mockListener.testStarted(EasyMock.eq(expectedTests.get(i)));
+ EasyMock.expectLastCall().once();
+
+ mockListener.testEnded(EasyMock.eq(expectedTests.get(i)),
+ EasyMock.<Map<String, String>>notNull());
+
+ EasyMock.expectLastCall().once();
+ }
+ }
+
+ mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<Map<String, String>>notNull());
+ EasyMock.expectLastCall().once();
+
+ EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).andReturn("")
+ .once();
+
+ EasyMock.replay(mockDevice, mockIDevice);
+ EasyMock.replay(mockListener);
+
+ deqpTest.setDevice(mockDevice);
+ deqpTest.run(mockListener);
+
+ EasyMock.verify(mockListener);
+ EasyMock.verify(mockDevice, mockIDevice);
+ output.close();
+ }
+
+ public void testRun_trivialIncludeFilter() throws Exception {
+ final TestIdentifier[] testIds = {
+ new TestIdentifier("dEQP-GLES3.missing", "no"),
+ new TestIdentifier("dEQP-GLES3.missing", "nope"),
+ new TestIdentifier("dEQP-GLES3.missing", "donotwant"),
+ new TestIdentifier("dEQP-GLES3.pick_me", "yes"),
+ new TestIdentifier("dEQP-GLES3.pick_me", "ok"),
+ new TestIdentifier("dEQP-GLES3.pick_me", "accepted"),
+ };
+
+ List<TestIdentifier> allTests = new ArrayList<TestIdentifier>();
+ for (TestIdentifier id : testIds) {
+ allTests.add(id);
+ }
+
+ List<TestIdentifier> activeTests = new ArrayList<TestIdentifier>();
+ activeTests.add(testIds[3]);
+ activeTests.add(testIds[4]);
+ activeTests.add(testIds[5]);
+
+ String expectedTrie = "{dEQP-GLES3{pick_me{yes,ok,accepted}}}";
+
+ ArrayList<String> includes = new ArrayList();
+ includes.add("dEQP-GLES3.pick_me#*");
+ testFiltering(includes, null, allTests, expectedTrie, activeTests);
+ }
+
+ public void testRun_trivialExcludeFilter() throws Exception {
+ final TestIdentifier[] testIds = {
+ new TestIdentifier("dEQP-GLES3.missing", "no"),
+ new TestIdentifier("dEQP-GLES3.missing", "nope"),
+ new TestIdentifier("dEQP-GLES3.missing", "donotwant"),
+ new TestIdentifier("dEQP-GLES3.pick_me", "yes"),
+ new TestIdentifier("dEQP-GLES3.pick_me", "ok"),
+ new TestIdentifier("dEQP-GLES3.pick_me", "accepted"),
+ };
+
+ List<TestIdentifier> allTests = new ArrayList<TestIdentifier>();
+ for (TestIdentifier id : testIds) {
+ allTests.add(id);
+ }
+
+ List<TestIdentifier> activeTests = new ArrayList<TestIdentifier>();
+ activeTests.add(testIds[3]);
+ activeTests.add(testIds[4]);
+ activeTests.add(testIds[5]);
+
+ String expectedTrie = "{dEQP-GLES3{pick_me{yes,ok,accepted}}}";
+
+ ArrayList<String> excludes = new ArrayList();
+ excludes.add("dEQP-GLES3.missing#*");
+ testFiltering(null, excludes, allTests, expectedTrie, activeTests);
+ }
+
+ public void testRun_includeAndExcludeFilter() throws Exception {
+ final TestIdentifier[] testIds = {
+ new TestIdentifier("dEQP-GLES3.group1", "foo"),
+ new TestIdentifier("dEQP-GLES3.group1", "nope"),
+ new TestIdentifier("dEQP-GLES3.group1", "donotwant"),
+ new TestIdentifier("dEQP-GLES3.group2", "foo"),
+ new TestIdentifier("dEQP-GLES3.group2", "yes"),
+ new TestIdentifier("dEQP-GLES3.group2", "thoushallnotpass"),
+ };
+
+ List<TestIdentifier> allTests = new ArrayList<TestIdentifier>();
+ for (TestIdentifier id : testIds) {
+ allTests.add(id);
+ }
+
+ List<TestIdentifier> activeTests = new ArrayList<TestIdentifier>();
+ activeTests.add(testIds[4]);
+
+ String expectedTrie = "{dEQP-GLES3{group2{yes}}}";
+
+ ArrayList<String> includes = new ArrayList();
+ includes.add("dEQP-GLES3.group2#*");
+
+ ArrayList<String> excludes = new ArrayList();
+ excludes.add("*foo");
+ excludes.add("*thoushallnotpass");
+ testFiltering(includes, excludes, allTests, expectedTrie, activeTests);
+ }
+
+ public void testRun_includeAll() throws Exception {
+ final TestIdentifier[] testIds = {
+ new TestIdentifier("dEQP-GLES3.group1", "mememe"),
+ new TestIdentifier("dEQP-GLES3.group1", "yeah"),
+ new TestIdentifier("dEQP-GLES3.group1", "takeitall"),
+ new TestIdentifier("dEQP-GLES3.group2", "jeba"),
+ new TestIdentifier("dEQP-GLES3.group2", "yes"),
+ new TestIdentifier("dEQP-GLES3.group2", "granted"),
+ };
+
+ List<TestIdentifier> allTests = new ArrayList<TestIdentifier>();
+ for (TestIdentifier id : testIds) {
+ allTests.add(id);
+ }
+
+ String expectedTrie = "{dEQP-GLES3{group1{mememe,yeah,takeitall},group2{jeba,yes,granted}}}";
+
+ ArrayList<String> includes = new ArrayList();
+ includes.add("*");
+
+ testFiltering(includes, null, allTests, expectedTrie, allTests);
+ }
+
+ public void testRun_excludeAll() throws Exception {
+ final TestIdentifier[] testIds = {
+ new TestIdentifier("dEQP-GLES3.group1", "no"),
+ new TestIdentifier("dEQP-GLES3.group1", "nope"),
+ new TestIdentifier("dEQP-GLES3.group1", "nottoday"),
+ new TestIdentifier("dEQP-GLES3.group2", "banned"),
+ new TestIdentifier("dEQP-GLES3.group2", "notrecognized"),
+ new TestIdentifier("dEQP-GLES3.group2", "-2"),
+ };
+
+ List<TestIdentifier> allTests = new ArrayList<TestIdentifier>();
+ for (TestIdentifier id : testIds) {
+ allTests.add(id);
+ }
+
+ String expectedTrie = "";
+
+ ArrayList<String> excludes = new ArrayList();
+ excludes.add("*");
+
+ testFiltering(null, excludes, allTests, expectedTrie, new ArrayList<TestIdentifier>());
+ }
+
+ /**
+ * Test running a unexecutable test.
+ */
+ public void testRun_unexecutableTests() throws Exception {
+ final String instrumentationAnswerNoExecs =
+ "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_CODE: 0\r\n";
+
+ final TestIdentifier[] testIds = {
+ new TestIdentifier("dEQP-GLES3.missing", "no"),
+ new TestIdentifier("dEQP-GLES3.missing", "nope"),
+ new TestIdentifier("dEQP-GLES3.missing", "donotwant"),
+ };
+
+ final String[] testPaths = {
+ "dEQP-GLES3.missing.no",
+ "dEQP-GLES3.missing.nope",
+ "dEQP-GLES3.missing.donotwant",
+ };
+
+ ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class);
+ ITestInvocationListener mockListener
+ = EasyMock.createStrictMock(ITestInvocationListener.class);
+ IDevice mockIDevice = EasyMock.createMock(IDevice.class);
+
+ Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>();
+
+ for (TestIdentifier id : testIds) {
+ tests.add(id);
+ }
+
+ DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests);
+
+ int version = 3 << 16;
+ EasyMock.expect(mockDevice.getProperty("ro.opengles.version"))
+ .andReturn(Integer.toString(version)).atLeastOnce();
+
+ EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).andReturn("")
+ .once();
+ EasyMock.expect(mockDevice.installPackage(EasyMock.<File>anyObject(),
+ EasyMock.eq(true), EasyMock.eq(AbiUtils.createAbiFlag(ABI.getName()))))
+ .andReturn(null).once();
+
+ expectRenderConfigQuery(mockDevice, 3, 0);
+
+ String commandLine = String.format(
+ "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 "
+ + "--deqp-screen-rotation=unspecified "
+ + "--deqp-surface-type=window "
+ + "--deqp-log-images=disable "
+ + "--deqp-watchdog=enable",
+ CASE_LIST_FILE_NAME);
+
+ // first try
+ runInstrumentationLineAndAnswer(mockDevice, mockIDevice,
+ "{dEQP-GLES3{missing{no,nope,donotwant}}}", commandLine, instrumentationAnswerNoExecs);
+
+ // splitting begins
+ runInstrumentationLineAndAnswer(mockDevice, mockIDevice,
+ "{dEQP-GLES3{missing{no}}}", commandLine, instrumentationAnswerNoExecs);
+ runInstrumentationLineAndAnswer(mockDevice, mockIDevice,
+ "{dEQP-GLES3{missing{nope,donotwant}}}", commandLine, instrumentationAnswerNoExecs);
+ runInstrumentationLineAndAnswer(mockDevice, mockIDevice,
+ "{dEQP-GLES3{missing{nope}}}", commandLine, instrumentationAnswerNoExecs);
+ runInstrumentationLineAndAnswer(mockDevice, mockIDevice,
+ "{dEQP-GLES3{missing{donotwant}}}", commandLine, instrumentationAnswerNoExecs);
+
+ mockListener.testRunStarted(getTestId(deqpTest), testPaths.length);
+ EasyMock.expectLastCall().once();
+
+ for (int i = 0; i < testPaths.length; i++) {
+ mockListener.testStarted(EasyMock.eq(testIds[i]));
+ EasyMock.expectLastCall().once();
+
+ mockListener.testFailed(EasyMock.eq(testIds[i]),
+ EasyMock.eq("=== with config {glformat=rgba8888d24s8,rotation=unspecified,surfacetype=window} ===\n"
+ + "Abort: Test cannot be executed"));
+ EasyMock.expectLastCall().once();
+
+ mockListener.testEnded(EasyMock.eq(testIds[i]),
+ EasyMock.<Map<String, String>>notNull());
+ EasyMock.expectLastCall().once();
+ }
+
+ mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<Map<String, String>>notNull());
+ EasyMock.expectLastCall().once();
+
+ EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).andReturn("")
+ .once();
+
+ EasyMock.replay(mockDevice, mockIDevice);
+ EasyMock.replay(mockListener);
+
+ deqpTest.setDevice(mockDevice);
+ deqpTest.run(mockListener);
+
+ EasyMock.verify(mockListener);
+ EasyMock.verify(mockDevice, mockIDevice);
+ }
+
+ /**
+ * Test that test are left unexecuted if pm list query fails
+ */
+ public void testRun_queryPmListFailure()
+ throws Exception {
+ final TestIdentifier testId = new TestIdentifier("dEQP-GLES3.orientation", "test");
+
+ ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class);
+ ITestInvocationListener mockListener
+ = EasyMock.createStrictMock(ITestInvocationListener.class);
+ Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>();
+ tests.add(testId);
+
+ DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests);
+ OptionSetter setter = new OptionSetter(deqpTest);
+ // Note: If the rotation is the default unspecified, features are not queried at all
+ setter.setOptionValue("deqp-screen-rotation", "90");
+
+ deqpTest.setDevice(mockDevice);
+
+ int version = 3 << 16;
+ EasyMock.expect(mockDevice.getProperty("ro.opengles.version"))
+ .andReturn(Integer.toString(version)).atLeastOnce();
+
+ EasyMock.expect(mockDevice.executeShellCommand("pm list features"))
+ .andReturn("not a valid format");
+
+ EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).
+ andReturn("").once();
+
+ EasyMock.expect(mockDevice.installPackage(EasyMock.<File>anyObject(),
+ EasyMock.eq(true),
+ EasyMock.eq(AbiUtils.createAbiFlag(ABI.getName())))).andReturn(null)
+ .once();
+
+ EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG)))
+ .andReturn("").once();
+
+ mockListener.testRunStarted(getTestId(deqpTest), 1);
+ EasyMock.expectLastCall().once();
+
+ mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<Map<String, String>>notNull());
+ EasyMock.expectLastCall().once();
+
+ EasyMock.replay(mockDevice);
+ EasyMock.replay(mockListener);
+ deqpTest.run(mockListener);
+ EasyMock.verify(mockListener);
+ EasyMock.verify(mockDevice);
+ }
+
+ /**
+ * Test that test are left unexecuted if renderablity query fails
+ */
+ public void testRun_queryRenderabilityFailure()
+ throws Exception {
+ final TestIdentifier testId = new TestIdentifier("dEQP-GLES3.orientation", "test");
+
+ ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class);
+ ITestInvocationListener mockListener
+ = EasyMock.createStrictMock(ITestInvocationListener.class);
+
+ Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>();
+ tests.add(testId);
+
+ DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests);
+
+ deqpTest.setDevice(mockDevice);
+
+ int version = 3 << 16;
+ EasyMock.expect(mockDevice.getProperty("ro.opengles.version"))
+ .andReturn(Integer.toString(version)).atLeastOnce();
+
+ EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).
+ andReturn("").once();
+
+ EasyMock.expect(mockDevice.installPackage(EasyMock.<File>anyObject(),
+ EasyMock.eq(true),
+ EasyMock.eq(AbiUtils.createAbiFlag(ABI.getName())))).andReturn(null)
+ .once();
+
+ expectRenderConfigQueryAndReturn(mockDevice,
+ "--deqp-gl-config-name=rgba8888d24s8 "
+ + "--deqp-screen-rotation=unspecified "
+ + "--deqp-surface-type=window "
+ + "--deqp-gl-major-version=3 "
+ + "--deqp-gl-minor-version=0", "Maybe?");
+
+ EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG)))
+ .andReturn("").once();
+
+ mockListener.testRunStarted(getTestId(deqpTest), 1);
+ EasyMock.expectLastCall().once();
+
+ mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<Map<String, String>>notNull());
+ EasyMock.expectLastCall().once();
+
+ EasyMock.replay(mockDevice);
+ EasyMock.replay(mockListener);
+ deqpTest.run(mockListener);
+ EasyMock.verify(mockListener);
+ EasyMock.verify(mockDevice);
+ }
+
+ /**
+ * Test that orientation is supplied to runner correctly
+ */
+ private void testOrientation(final String rotation, final String featureString)
+ throws Exception {
+ final TestIdentifier testId = new TestIdentifier("dEQP-GLES3.orientation", "test");
+ final String testPath = "dEQP-GLES3.orientation.test";
+ final String testTrie = "{dEQP-GLES3{orientation{test}}}";
+ final String output = "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=" + testPath + "\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_CODE: 0\r\n";
+
+ ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class);
+ ITestInvocationListener mockListener
+ = EasyMock.createStrictMock(ITestInvocationListener.class);
+ IDevice mockIDevice = EasyMock.createMock(IDevice.class);
+
+ Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>();
+ tests.add(testId);
+
+ DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests);
+ OptionSetter setter = new OptionSetter(deqpTest);
+ setter.setOptionValue("deqp-screen-rotation", rotation);
+
+ deqpTest.setDevice(mockDevice);
+
+ int version = 3 << 16;
+ EasyMock.expect(mockDevice.getProperty("ro.opengles.version"))
+ .andReturn(Integer.toString(version)).atLeastOnce();
+
+ if (!rotation.equals(DeqpTestRunner.BatchRunConfiguration.ROTATION_UNSPECIFIED)) {
+ EasyMock.expect(mockDevice.executeShellCommand("pm list features"))
+ .andReturn(featureString);
+ }
+
+ final boolean isPortraitOrientation =
+ rotation.equals(DeqpTestRunner.BatchRunConfiguration.ROTATION_PORTRAIT) ||
+ rotation.equals(DeqpTestRunner.BatchRunConfiguration.ROTATION_REVERSE_PORTRAIT);
+ final boolean isLandscapeOrientation =
+ rotation.equals(DeqpTestRunner.BatchRunConfiguration.ROTATION_LANDSCAPE) ||
+ rotation.equals(DeqpTestRunner.BatchRunConfiguration.ROTATION_REVERSE_LANDSCAPE);
+ final boolean executable =
+ rotation.equals(DeqpTestRunner.BatchRunConfiguration.ROTATION_UNSPECIFIED) ||
+ (isPortraitOrientation &&
+ featureString.contains(DeqpTestRunner.FEATURE_PORTRAIT)) ||
+ (isLandscapeOrientation &&
+ featureString.contains(DeqpTestRunner.FEATURE_LANDSCAPE));
+
+ EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).
+ andReturn("").once();
+
+ EasyMock.expect(mockDevice.installPackage(EasyMock.<File>anyObject(),
+ EasyMock.eq(true),
+ EasyMock.eq(AbiUtils.createAbiFlag(ABI.getName())))).andReturn(null)
+ .once();
+
+ if (executable) {
+ expectRenderConfigQuery(mockDevice, String.format(
+ "--deqp-gl-config-name=rgba8888d24s8 --deqp-screen-rotation=%s "
+ + "--deqp-surface-type=window --deqp-gl-major-version=3 "
+ + "--deqp-gl-minor-version=0", rotation));
+
+ String commandLine = String.format(
+ "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 "
+ + "--deqp-screen-rotation=%s "
+ + "--deqp-surface-type=window "
+ + "--deqp-log-images=disable "
+ + "--deqp-watchdog=enable",
+ CASE_LIST_FILE_NAME, rotation);
+
+ runInstrumentationLineAndAnswer(mockDevice, mockIDevice, testTrie, commandLine,
+ output);
+ }
+
+ EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG)))
+ .andReturn("").once();
+
+ mockListener.testRunStarted(getTestId(deqpTest), 1);
+ EasyMock.expectLastCall().once();
+
+ mockListener.testStarted(EasyMock.eq(testId));
+ EasyMock.expectLastCall().once();
+
+ mockListener.testEnded(EasyMock.eq(testId), EasyMock.<Map<String, String>>notNull());
+ EasyMock.expectLastCall().once();
+
+ mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<Map<String, String>>notNull());
+ EasyMock.expectLastCall().once();
+
+ EasyMock.replay(mockDevice, mockIDevice);
+ EasyMock.replay(mockListener);
+ deqpTest.run(mockListener);
+ EasyMock.verify(mockListener);
+ EasyMock.verify(mockDevice, mockIDevice);
+ }
+
+ /**
+ * Test OpeGL ES3 tests on device with OpenGL ES2.
+ */
+ public void testRun_require30DeviceVersion20() throws Exception {
+ testGlesVersion(3, 0, 2, 0);
+ }
+
+ /**
+ * Test OpeGL ES3.1 tests on device with OpenGL ES2.
+ */
+ public void testRun_require31DeviceVersion20() throws Exception {
+ testGlesVersion(3, 1, 2, 0);
+ }
+
+ /**
+ * Test OpeGL ES3 tests on device with OpenGL ES3.
+ */
+ public void testRun_require30DeviceVersion30() throws Exception {
+ testGlesVersion(3, 0, 3, 0);
+ }
+
+ /**
+ * Test OpeGL ES3.1 tests on device with OpenGL ES3.
+ */
+ public void testRun_require31DeviceVersion30() throws Exception {
+ testGlesVersion(3, 1, 3, 0);
+ }
+
+ /**
+ * Test OpeGL ES3 tests on device with OpenGL ES3.1.
+ */
+ public void testRun_require30DeviceVersion31() throws Exception {
+ testGlesVersion(3, 0, 3, 1);
+ }
+
+ /**
+ * Test OpeGL ES3.1 tests on device with OpenGL ES3.1.
+ */
+ public void testRun_require31DeviceVersion31() throws Exception {
+ testGlesVersion(3, 1, 3, 1);
+ }
+
+ /**
+ * Test dEQP Pass result code.
+ */
+ public void testRun_resultPass() throws Exception {
+ testResultCode("Pass", true);
+ }
+
+ /**
+ * Test dEQP Fail result code.
+ */
+ public void testRun_resultFail() throws Exception {
+ testResultCode("Fail", false);
+ }
+
+ /**
+ * Test dEQP NotSupported result code.
+ */
+ public void testRun_resultNotSupported() throws Exception {
+ testResultCode("NotSupported", true);
+ }
+
+ /**
+ * Test dEQP QualityWarning result code.
+ */
+ public void testRun_resultQualityWarning() throws Exception {
+ testResultCode("QualityWarning", true);
+ }
+
+ /**
+ * Test dEQP CompatibilityWarning result code.
+ */
+ public void testRun_resultCompatibilityWarning() throws Exception {
+ testResultCode("CompatibilityWarning", true);
+ }
+
+ /**
+ * Test dEQP ResourceError result code.
+ */
+ public void testRun_resultResourceError() throws Exception {
+ testResultCode("ResourceError", false);
+ }
+
+ /**
+ * Test dEQP InternalError result code.
+ */
+ public void testRun_resultInternalError() throws Exception {
+ testResultCode("InternalError", false);
+ }
+
+ /**
+ * Test dEQP Crash result code.
+ */
+ public void testRun_resultCrash() throws Exception {
+ testResultCode("Crash", false);
+ }
+
+ /**
+ * Test dEQP Timeout result code.
+ */
+ public void testRun_resultTimeout() throws Exception {
+ testResultCode("Timeout", false);
+ }
+ /**
+ * Test dEQP Orientation
+ */
+ public void testRun_orientationLandscape() throws Exception {
+ testOrientation("90", ALL_FEATURES);
+ }
+
+ /**
+ * Test dEQP Orientation
+ */
+ public void testRun_orientationPortrait() throws Exception {
+ testOrientation("0", ALL_FEATURES);
+ }
+
+ /**
+ * Test dEQP Orientation
+ */
+ public void testRun_orientationReverseLandscape() throws Exception {
+ testOrientation("270", ALL_FEATURES);
+ }
+
+ /**
+ * Test dEQP Orientation
+ */
+ public void testRun_orientationReversePortrait() throws Exception {
+ testOrientation("180", ALL_FEATURES);
+ }
+
+ /**
+ * Test dEQP Orientation
+ */
+ public void testRun_orientationUnspecified() throws Exception {
+ testOrientation("unspecified", ALL_FEATURES);
+ }
+
+ /**
+ * Test dEQP Orientation with limited features
+ */
+ public void testRun_orientationUnspecifiedLimitedFeatures() throws Exception {
+ testOrientation("unspecified", ONLY_LANDSCAPE_FEATURES);
+ }
+
+ /**
+ * Test dEQP Orientation with limited features
+ */
+ public void testRun_orientationLandscapeLimitedFeatures() throws Exception {
+ testOrientation("90", ONLY_LANDSCAPE_FEATURES);
+ }
+
+ /**
+ * Test dEQP Orientation with limited features
+ */
+ public void testRun_orientationPortraitLimitedFeatures() throws Exception {
+ testOrientation("0", ONLY_LANDSCAPE_FEATURES);
+ }
+
+ /**
+ * Test dEQP unsupported pixel format
+ */
+ public void testRun_unsupportedPixelFormat() throws Exception {
+ final String pixelFormat = "rgba5658d16m4";
+ final TestIdentifier testId = new TestIdentifier("dEQP-GLES3.pixelformat", "test");
+ final String testPath = "dEQP-GLES3.pixelformat.test";
+ final String testTrie = "{dEQP-GLES3{pixelformat{test}}}";
+ final String output = "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=" + testPath + "\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_CODE: 0\r\n";
+
+ ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class);
+ ITestInvocationListener mockListener
+ = EasyMock.createStrictMock(ITestInvocationListener.class);
+
+ Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>();
+ tests.add(testId);
+
+ DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests);
+ OptionSetter setter = new OptionSetter(deqpTest);
+ setter.setOptionValue("deqp-gl-config-name", pixelFormat);
+
+ deqpTest.setDevice(mockDevice);
+
+ int version = 3 << 16;
+ EasyMock.expect(mockDevice.getProperty("ro.opengles.version"))
+ .andReturn(Integer.toString(version)).atLeastOnce();
+
+ EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).
+ andReturn("").once();
+
+ EasyMock.expect(mockDevice.installPackage(EasyMock.<File>anyObject(),
+ EasyMock.eq(true),
+ EasyMock.eq(AbiUtils.createAbiFlag(ABI.getName())))).andReturn(null)
+ .once();
+
+ expectRenderConfigQueryAndReturn(mockDevice, String.format(
+ "--deqp-gl-config-name=%s --deqp-screen-rotation=unspecified "
+ + "--deqp-surface-type=window "
+ + "--deqp-gl-major-version=3 "
+ + "--deqp-gl-minor-version=0", pixelFormat), "No");
+
+ EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG)))
+ .andReturn("").once();
+
+ mockListener.testRunStarted(getTestId(deqpTest), 1);
+ EasyMock.expectLastCall().once();
+
+ mockListener.testStarted(EasyMock.eq(testId));
+ EasyMock.expectLastCall().once();
+
+ mockListener.testEnded(EasyMock.eq(testId), EasyMock.<Map<String, String>>notNull());
+ EasyMock.expectLastCall().once();
+
+ mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<Map<String, String>>notNull());
+ EasyMock.expectLastCall().once();
+
+ EasyMock.replay(mockDevice);
+ EasyMock.replay(mockListener);
+ deqpTest.run(mockListener);
+ EasyMock.verify(mockListener);
+ EasyMock.verify(mockDevice);
+ }
+
+ public static interface RecoverableTestDevice extends ITestDevice {
+ public void recoverDevice() throws DeviceNotAvailableException;
+ }
+
+ private static enum RecoveryEvent {
+ PROGRESS,
+ FAIL_CONNECTION_REFUSED,
+ FAIL_LINK_KILLED,
+ };
+
+ private void runRecoveryWithPattern(DeqpTestRunner.Recovery recovery, RecoveryEvent[] events)
+ throws DeviceNotAvailableException {
+ for (RecoveryEvent event : events) {
+ switch (event) {
+ case PROGRESS:
+ recovery.onExecutionProgressed();
+ break;
+ case FAIL_CONNECTION_REFUSED:
+ recovery.recoverConnectionRefused();
+ break;
+ case FAIL_LINK_KILLED:
+ recovery.recoverComLinkKilled();
+ break;
+ }
+ }
+ }
+
+ private void setRecoveryExpectationWait(DeqpTestRunner.ISleepProvider mockSleepProvider) {
+ mockSleepProvider.sleep(EasyMock.gt(0));
+ EasyMock.expectLastCall().once();
+ }
+
+ private void setRecoveryExpectationKillProcess(RecoverableTestDevice mockDevice,
+ DeqpTestRunner.ISleepProvider mockSleepProvider) throws DeviceNotAvailableException {
+ EasyMock.expect(mockDevice.executeShellCommand(EasyMock.contains("ps"))).
+ andReturn("root 1234 com.drawelement.deqp").once();
+
+ EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("kill -9 1234"))).
+ andReturn("").once();
+
+ // Recovery checks if kill failed
+ mockSleepProvider.sleep(EasyMock.gt(0));
+ EasyMock.expectLastCall().once();
+ EasyMock.expect(mockDevice.executeShellCommand(EasyMock.contains("ps"))).
+ andReturn("").once();
+ }
+
+ private void setRecoveryExpectationRecovery(RecoverableTestDevice mockDevice)
+ throws DeviceNotAvailableException {
+ mockDevice.recoverDevice();
+ EasyMock.expectLastCall().once();
+ }
+
+ private void setRecoveryExpectationReboot(RecoverableTestDevice mockDevice)
+ throws DeviceNotAvailableException {
+ mockDevice.reboot();
+ EasyMock.expectLastCall().once();
+ }
+
+ private int setRecoveryExpectationOfAConnFailure(RecoverableTestDevice mockDevice,
+ DeqpTestRunner.ISleepProvider mockSleepProvider, int numConsecutiveErrors)
+ throws DeviceNotAvailableException {
+ switch (numConsecutiveErrors) {
+ case 0:
+ case 1:
+ setRecoveryExpectationRecovery(mockDevice);
+ return 2;
+ case 2:
+ setRecoveryExpectationReboot(mockDevice);
+ return 3;
+ default:
+ return 4;
+ }
+ }
+
+ private int setRecoveryExpectationOfAComKilled(RecoverableTestDevice mockDevice,
+ DeqpTestRunner.ISleepProvider mockSleepProvider, int numConsecutiveErrors)
+ throws DeviceNotAvailableException {
+ switch (numConsecutiveErrors) {
+ case 0:
+ setRecoveryExpectationWait(mockSleepProvider);
+ setRecoveryExpectationKillProcess(mockDevice, mockSleepProvider);
+ return 1;
+ case 1:
+ setRecoveryExpectationRecovery(mockDevice);
+ setRecoveryExpectationKillProcess(mockDevice, mockSleepProvider);
+ return 2;
+ case 2:
+ setRecoveryExpectationReboot(mockDevice);
+ return 3;
+ default:
+ return 4;
+ }
+ }
+
+ private void setRecoveryExpectationsOfAPattern(RecoverableTestDevice mockDevice,
+ DeqpTestRunner.ISleepProvider mockSleepProvider, RecoveryEvent[] events)
+ throws DeviceNotAvailableException {
+ int numConsecutiveErrors = 0;
+ for (RecoveryEvent event : events) {
+ switch (event) {
+ case PROGRESS:
+ numConsecutiveErrors = 0;
+ break;
+ case FAIL_CONNECTION_REFUSED:
+ numConsecutiveErrors = setRecoveryExpectationOfAConnFailure(mockDevice,
+ mockSleepProvider, numConsecutiveErrors);
+ break;
+ case FAIL_LINK_KILLED:
+ numConsecutiveErrors = setRecoveryExpectationOfAComKilled(mockDevice,
+ mockSleepProvider, numConsecutiveErrors);
+ break;
+ }
+ }
+ }
+
+ /**
+ * Test dEQP runner recovery state machine.
+ */
+ private void testRecoveryWithPattern(boolean expectSuccess, RecoveryEvent...pattern)
+ throws Exception {
+ DeqpTestRunner.Recovery recovery = new DeqpTestRunner.Recovery();
+ IMocksControl orderedControl = EasyMock.createStrictControl();
+ RecoverableTestDevice mockDevice = orderedControl.createMock(RecoverableTestDevice.class);
+ DeqpTestRunner.ISleepProvider mockSleepProvider =
+ orderedControl.createMock(DeqpTestRunner.ISleepProvider.class);
+
+ setRecoveryExpectationsOfAPattern(mockDevice, mockSleepProvider, pattern);
+
+ orderedControl.replay();
+
+ recovery.setDevice(mockDevice);
+ recovery.setSleepProvider(mockSleepProvider);
+ try {
+ runRecoveryWithPattern(recovery, pattern);
+ if (!expectSuccess) {
+ fail("Expected DeviceNotAvailableException");
+ }
+ } catch (DeviceNotAvailableException ex) {
+ if (expectSuccess) {
+ fail("Did not expect DeviceNotAvailableException");
+ }
+ }
+
+ orderedControl.verify();
+ }
+
+ // basic patterns
+
+ public void testRecovery_NoEvents() throws Exception {
+ testRecoveryWithPattern(true);
+ }
+
+ public void testRecovery_AllOk() throws Exception {
+ testRecoveryWithPattern(true, RecoveryEvent.PROGRESS, RecoveryEvent.PROGRESS);
+ }
+
+ // conn fail patterns
+
+ public void testRecovery_OneConnectionFailureBegin() throws Exception {
+ testRecoveryWithPattern(true, RecoveryEvent.FAIL_CONNECTION_REFUSED,
+ RecoveryEvent.PROGRESS);
+ }
+
+ public void testRecovery_TwoConnectionFailuresBegin() throws Exception {
+ testRecoveryWithPattern(true, RecoveryEvent.FAIL_CONNECTION_REFUSED,
+ RecoveryEvent.FAIL_CONNECTION_REFUSED, RecoveryEvent.PROGRESS);
+ }
+
+ public void testRecovery_ThreeConnectionFailuresBegin() throws Exception {
+ testRecoveryWithPattern(false, RecoveryEvent.FAIL_CONNECTION_REFUSED,
+ RecoveryEvent.FAIL_CONNECTION_REFUSED, RecoveryEvent.FAIL_CONNECTION_REFUSED);
+ }
+
+ public void testRecovery_OneConnectionFailureMid() throws Exception {
+ testRecoveryWithPattern(true, RecoveryEvent.PROGRESS,
+ RecoveryEvent.FAIL_CONNECTION_REFUSED, RecoveryEvent.PROGRESS);
+ }
+
+ public void testRecovery_TwoConnectionFailuresMid() throws Exception {
+ testRecoveryWithPattern(true, RecoveryEvent.PROGRESS,
+ RecoveryEvent.FAIL_CONNECTION_REFUSED, RecoveryEvent.FAIL_CONNECTION_REFUSED,
+ RecoveryEvent.PROGRESS);
+ }
+
+ public void testRecovery_ThreeConnectionFailuresMid() throws Exception {
+ testRecoveryWithPattern(false, RecoveryEvent.PROGRESS,
+ RecoveryEvent.FAIL_CONNECTION_REFUSED, RecoveryEvent.FAIL_CONNECTION_REFUSED,
+ RecoveryEvent.FAIL_CONNECTION_REFUSED);
+ }
+
+ // link fail patterns
+
+ public void testRecovery_OneLinkFailureBegin() throws Exception {
+ testRecoveryWithPattern(true, RecoveryEvent.FAIL_LINK_KILLED,
+ RecoveryEvent.PROGRESS);
+ }
+
+ public void testRecovery_TwoLinkFailuresBegin() throws Exception {
+ testRecoveryWithPattern(true, RecoveryEvent.FAIL_LINK_KILLED,
+ RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.PROGRESS);
+ }
+
+ public void testRecovery_ThreeLinkFailuresBegin() throws Exception {
+ testRecoveryWithPattern(true, RecoveryEvent.FAIL_LINK_KILLED,
+ RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.FAIL_LINK_KILLED,
+ RecoveryEvent.PROGRESS);
+ }
+
+ public void testRecovery_FourLinkFailuresBegin() throws Exception {
+ testRecoveryWithPattern(false, RecoveryEvent.FAIL_LINK_KILLED,
+ RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.FAIL_LINK_KILLED,
+ RecoveryEvent.FAIL_LINK_KILLED);
+ }
+
+ public void testRecovery_OneLinkFailureMid() throws Exception {
+ testRecoveryWithPattern(true, RecoveryEvent.PROGRESS,
+ RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.PROGRESS);
+ }
+
+ public void testRecovery_TwoLinkFailuresMid() throws Exception {
+ testRecoveryWithPattern(true, RecoveryEvent.PROGRESS,
+ RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.FAIL_LINK_KILLED,
+ RecoveryEvent.PROGRESS);
+ }
+
+ public void testRecovery_ThreeLinkFailuresMid() throws Exception {
+ testRecoveryWithPattern(true, RecoveryEvent.PROGRESS,
+ RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.FAIL_LINK_KILLED,
+ RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.PROGRESS);
+ }
+
+ public void testRecovery_FourLinkFailuresMid() throws Exception {
+ testRecoveryWithPattern(false, RecoveryEvent.PROGRESS, RecoveryEvent.FAIL_LINK_KILLED,
+ RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.FAIL_LINK_KILLED,
+ RecoveryEvent.FAIL_LINK_KILLED);
+ }
+
+ // mixed patterns
+
+ public void testRecovery_MixedFailuresProgressBetween() throws Exception {
+ testRecoveryWithPattern(true,
+ RecoveryEvent.PROGRESS, RecoveryEvent.FAIL_LINK_KILLED,
+ RecoveryEvent.PROGRESS, RecoveryEvent.FAIL_CONNECTION_REFUSED,
+ RecoveryEvent.PROGRESS, RecoveryEvent.FAIL_LINK_KILLED,
+ RecoveryEvent.PROGRESS, RecoveryEvent.FAIL_CONNECTION_REFUSED,
+ RecoveryEvent.PROGRESS);
+ }
+
+ public void testRecovery_MixedFailuresNoProgressBetween() throws Exception {
+ testRecoveryWithPattern(true,
+ RecoveryEvent.PROGRESS, RecoveryEvent.FAIL_LINK_KILLED,
+ RecoveryEvent.FAIL_CONNECTION_REFUSED, RecoveryEvent.FAIL_LINK_KILLED,
+ RecoveryEvent.PROGRESS);
+ }
+
+ /**
+ * Test recovery if process cannot be killed
+ */
+ public void testRecovery_unkillableProcess () throws Exception {
+ DeqpTestRunner.Recovery recovery = new DeqpTestRunner.Recovery();
+ IMocksControl orderedControl = EasyMock.createStrictControl();
+ RecoverableTestDevice mockDevice = orderedControl.createMock(RecoverableTestDevice.class);
+ DeqpTestRunner.ISleepProvider mockSleepProvider =
+ orderedControl.createMock(DeqpTestRunner.ISleepProvider.class);
+
+ // recovery attempts to kill the process after a timeout
+ mockSleepProvider.sleep(EasyMock.gt(0));
+ EasyMock.expect(mockDevice.executeShellCommand(EasyMock.contains("ps"))).
+ andReturn("root 1234 com.drawelement.deqp").once();
+ EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("kill -9 1234"))).
+ andReturn("").once();
+
+ // Recovery checks if kill failed
+ mockSleepProvider.sleep(EasyMock.gt(0));
+ EasyMock.expectLastCall().once();
+ EasyMock.expect(mockDevice.executeShellCommand(EasyMock.contains("ps"))).
+ andReturn("root 1234 com.drawelement.deqp").once();
+
+ // Recovery resets the connection
+ mockDevice.recoverDevice();
+ EasyMock.expectLastCall().once();
+
+ // and attempts to kill the process again
+ EasyMock.expect(mockDevice.executeShellCommand(EasyMock.contains("ps"))).
+ andReturn("root 1234 com.drawelement.deqp").once();
+ EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("kill -9 1234"))).
+ andReturn("").once();
+
+ // Recovery checks if kill failed
+ mockSleepProvider.sleep(EasyMock.gt(0));
+ EasyMock.expectLastCall().once();
+ EasyMock.expect(mockDevice.executeShellCommand(EasyMock.contains("ps"))).
+ andReturn("root 1234 com.drawelement.deqp").once();
+
+ // recovery reboots the device
+ mockDevice.reboot();
+ EasyMock.expectLastCall().once();
+
+ orderedControl.replay();
+ recovery.setDevice(mockDevice);
+ recovery.setSleepProvider(mockSleepProvider);
+ recovery.recoverComLinkKilled();
+ orderedControl.verify();
+ }
+
+ /**
+ * Test external interruption before batch run.
+ */
+ public void testInterrupt_killBeforeBatch() throws Exception {
+ final TestIdentifier testId = new TestIdentifier("dEQP-GLES3.interrupt", "test");
+
+ Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>();
+ tests.add(testId);
+
+ ITestInvocationListener mockListener
+ = EasyMock.createStrictMock(ITestInvocationListener.class);
+ ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class);
+ IDevice mockIDevice = EasyMock.createMock(IDevice.class);
+ IRunUtil mockRunUtil = EasyMock.createMock(IRunUtil.class);
+
+ DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests);
+
+ deqpTest.setDevice(mockDevice);
+ deqpTest.setRunUtil(mockRunUtil);
+
+ int version = 3 << 16;
+ EasyMock.expect(mockDevice.getProperty("ro.opengles.version"))
+ .andReturn(Integer.toString(version)).atLeastOnce();
+
+ EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).
+ andReturn("").once();
+
+ EasyMock.expect(mockDevice.installPackage(EasyMock.<File>anyObject(),
+ EasyMock.eq(true),
+ EasyMock.eq(AbiUtils.createAbiFlag(ABI.getName())))).andReturn(null)
+ .once();
+
+ expectRenderConfigQuery(mockDevice,
+ "--deqp-gl-config-name=rgba8888d24s8 --deqp-screen-rotation=unspecified "
+ + "--deqp-surface-type=window --deqp-gl-major-version=3 "
+ + "--deqp-gl-minor-version=0");
+
+ mockRunUtil.sleep(0);
+ EasyMock.expectLastCall().andThrow(new RunInterruptedException());
+
+ mockListener.testRunStarted(getTestId(deqpTest), 1);
+ EasyMock.expectLastCall().once();
+
+ EasyMock.replay(mockDevice, mockIDevice);
+ EasyMock.replay(mockListener);
+ EasyMock.replay(mockRunUtil);
+ try {
+ deqpTest.run(mockListener);
+ fail("expected RunInterruptedException");
+ } catch (RunInterruptedException ex) {
+ // expected
+ }
+ EasyMock.verify(mockRunUtil);
+ EasyMock.verify(mockListener);
+ EasyMock.verify(mockDevice, mockIDevice);
+ }
+
+ /**
+ * Test external interruption in testFailed().
+ */
+ public void testInterrupt_killReportTestFailed() throws Exception {
+ final TestIdentifier testId = new TestIdentifier("dEQP-GLES3.interrupt", "test");
+ final String testPath = "dEQP-GLES3.interrupt.test";
+ final String testTrie = "{dEQP-GLES3{interrupt{test}}}";
+ final String output = "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=" + testPath + "\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Fail\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Fail\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n"
+ + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
+ + "INSTRUMENTATION_CODE: 0\r\n";
+
+ Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>();
+ tests.add(testId);
+
+ ITestInvocationListener mockListener
+ = EasyMock.createStrictMock(ITestInvocationListener.class);
+ ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class);
+ IDevice mockIDevice = EasyMock.createMock(IDevice.class);
+ IRunUtil mockRunUtil = EasyMock.createMock(IRunUtil.class);
+
+ DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests);
+
+ deqpTest.setDevice(mockDevice);
+ deqpTest.setRunUtil(mockRunUtil);
+
+ int version = 3 << 16;
+ EasyMock.expect(mockDevice.getProperty("ro.opengles.version"))
+ .andReturn(Integer.toString(version)).atLeastOnce();
+
+ EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).
+ andReturn("").once();
+
+ EasyMock.expect(mockDevice.installPackage(EasyMock.<File>anyObject(),
+ EasyMock.eq(true),
+ EasyMock.eq(AbiUtils.createAbiFlag(ABI.getName())))).andReturn(null)
+ .once();
+
+ expectRenderConfigQuery(mockDevice,
+ "--deqp-gl-config-name=rgba8888d24s8 --deqp-screen-rotation=unspecified "
+ + "--deqp-surface-type=window --deqp-gl-major-version=3 "
+ + "--deqp-gl-minor-version=0");
+
+ mockRunUtil.sleep(0);
+ EasyMock.expectLastCall().once();
+
+ String commandLine = String.format(
+ "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 "
+ + "--deqp-screen-rotation=unspecified "
+ + "--deqp-surface-type=window "
+ + "--deqp-log-images=disable "
+ + "--deqp-watchdog=enable",
+ CASE_LIST_FILE_NAME);
+
+ runInstrumentationLineAndAnswer(mockDevice, mockIDevice, testTrie, commandLine,
+ output);
+
+ mockListener.testRunStarted(getTestId(deqpTest), 1);
+ EasyMock.expectLastCall().once();
+
+ mockListener.testStarted(EasyMock.eq(testId));
+ EasyMock.expectLastCall().once();
+
+ mockListener.testFailed(EasyMock.eq(testId), EasyMock.<String>notNull());
+ EasyMock.expectLastCall().andThrow(new RunInterruptedException());
+
+ EasyMock.replay(mockDevice, mockIDevice);
+ EasyMock.replay(mockListener);
+ EasyMock.replay(mockRunUtil);
+ try {
+ deqpTest.run(mockListener);
+ fail("expected RunInterruptedException");
+ } catch (RunInterruptedException ex) {
+ // expected
+ }
+ EasyMock.verify(mockRunUtil);
+ EasyMock.verify(mockListener);
+ EasyMock.verify(mockDevice, mockIDevice);
+ }
+
+ private void runInstrumentationLineAndAnswer(ITestDevice mockDevice, IDevice mockIDevice,
+ final String testTrie, final String cmd, final String output) throws Exception {
+ EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("rm " + CASE_LIST_FILE_NAME)))
+ .andReturn("").once();
+
+ EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("rm " + LOG_FILE_NAME)))
+ .andReturn("").once();
+
+ EasyMock.expect(mockDevice.pushString(testTrie + "\n", CASE_LIST_FILE_NAME))
+ .andReturn(true).once();
+
+ String command = String.format(
+ "am instrument %s -w -e deqpLogFileName \"%s\" -e deqpCmdLine \"%s\" "
+ + "-e deqpLogData \"%s\" %s",
+ AbiUtils.createAbiFlag(ABI.getName()), LOG_FILE_NAME, cmd, false,
+ INSTRUMENTATION_NAME);
+
+ EasyMock.expect(mockDevice.getIDevice()).andReturn(mockIDevice);
+ mockIDevice.executeShellCommand(EasyMock.eq(command),
+ EasyMock.<IShellOutputReceiver>notNull(), EasyMock.anyLong(),
+ EasyMock.isA(TimeUnit.class));
+
+ EasyMock.expectLastCall().andAnswer(new IAnswer<Object>() {
+ @Override
+ public Object answer() {
+ IShellOutputReceiver receiver
+ = (IShellOutputReceiver)EasyMock.getCurrentArguments()[1];
+
+ receiver.addOutput(output.getBytes(), 0, output.length());
+ receiver.flush();
+
+ return null;
+ }
+ });
+ }
+}
diff --git a/android/package/Android.mk b/android/package/Android.mk
index d4015a3..555a4ef 100644
--- a/android/package/Android.mk
+++ b/android/package/Android.mk
@@ -1,8 +1,30 @@
+# Copyright (C) 2014-2015 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
LOCAL_PATH := $(call my-dir)
+
include $(CLEAR_VARS)
+# don't include this package in any target ??????
+LOCAL_MODULE_TAGS := optional
+# and when built explicitly put it in the data partition
+LOCAL_MODULE_PATH := $(TARGET_OUT_DATA_APPS)
+
LOCAL_MODULE_TAGS := tests
+LOCAL_COMPATIBILITY_SUITE := cts_v2
+
LOCAL_SRC_FILES := $(call all-java-files-under,src)
LOCAL_JNI_SHARED_LIBRARIES := libdeqp
@@ -10,4 +32,7 @@
LOCAL_PACKAGE_NAME := com.drawelements.deqp
LOCAL_MULTILIB := both
+# We could go down all the way to API-13 for 32bit. 22 is required for 64bit ARM.
+LOCAL_SDK_VERSION := 22
+
include $(BUILD_PACKAGE)
diff --git a/scripts/build_android_mustpass.py b/scripts/build_android_mustpass.py
index 285f9d1..c991ced 100644
--- a/scripts/build_android_mustpass.py
+++ b/scripts/build_android_mustpass.py
@@ -31,6 +31,27 @@
import xml.dom.minidom as minidom
CTS_DATA_DIR = os.path.join(DEQP_DIR, "android", "cts")
+APK_NAME = "com.drawelements.deqp.apk"
+
+COPYRIGHT_DECLARATION = """
+ Copyright (C) 2015 The Android Open Source Project
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ """
+
+GENERATED_FILE_WARNING = """
+ This file has been automatically generated. Edit with caution.
+ """
class Configuration:
def __init__ (self, name, glconfig, rotation, surfacetype, filters):
@@ -248,6 +269,8 @@
return Filter(Filter.TYPE_EXCLUDE, filename)
def prettifyXML (doc):
+ doc.insert(0, ElementTree.Comment(COPYRIGHT_DECLARATION))
+ doc.insert(1, ElementTree.Comment(GENERATED_FILE_WARNING))
uglyString = ElementTree.tostring(doc, 'utf-8')
reparsed = minidom.parseString(uglyString)
return reparsed.toprettyxml(indent='\t', encoding='utf-8')
@@ -315,6 +338,32 @@
return mustpassElem
+def addOptionElement (parent, optionName, optionValue):
+ ElementTree.SubElement(parent, "option", name=optionName, value=optionValue)
+
+def genAndroidTestXml (mustpass):
+ INSTALLER_CLASS = "com.android.compatibility.common.tradefed.targetprep.ApkInstaller"
+ RUNNER_CLASS = "com.drawelements.deqp.runner.DeqpTestRunner"
+ configElement = ElementTree.Element("configuration")
+ preparerElement = ElementTree.SubElement(configElement, "target_preparer")
+ preparerElement.set("class", INSTALLER_CLASS)
+ addOptionElement(preparerElement, "cleanup-apks", "true")
+ addOptionElement(preparerElement, "test-file-name", APK_NAME)
+
+ for package in mustpass.packages:
+ for config in package.configurations:
+ testElement = ElementTree.SubElement(configElement, "test")
+ testElement.set("class", RUNNER_CLASS)
+ addOptionElement(testElement, "deqp-package", package.module.name)
+ addOptionElement(testElement, "deqp-caselist-file", getCaseListFileName(package,config))
+ # \todo [2015-10-16 kalle]: Replace with just command line? - requires simplifications in the runner/tests as well.
+ addOptionElement(testElement, "deqp-gl-config-name", config.glconfig)
+ addOptionElement(testElement, "deqp-surface-type", config.surfacetype)
+ addOptionElement(testElement, "deqp-screen-rotation", config.rotation)
+
+ return configElement
+
+
def genMustpass (mustpass, moduleCaseLists):
print "Generating mustpass '%s'" % mustpass.version
@@ -343,6 +392,7 @@
for case in matchingByConfig[config]:
testCaseMap[case].configurations.append(config)
+ # NOTE: CTS v2 does not need package XML files. Remove when transition is complete.
packageXml = genCTSPackageXML(package, root)
xmlFilename = os.path.join(CTS_DATA_DIR, mustpass.version, getCTSPackageName(package) + ".xml")
@@ -355,6 +405,14 @@
print " Writing spec: " + specFilename
writeFile(specFilename, prettifyXML(specXML))
+ # TODO: Which is the best selector mechanism?
+ if (mustpass.version == "mnc"):
+ androidTestXML = genAndroidTestXml(mustpass)
+ androidTestFilename = os.path.join(CTS_DATA_DIR, "AndroidTest.xml")
+
+ print " Writing AndroidTest.xml: " + androidTestFilename
+ writeFile(androidTestFilename, prettifyXML(androidTestXML))
+
print "Done!"
def genMustpassLists (mustpassLists, generator, buildCfg):