Tomeu Vizoso | d62dd8b | 2019-12-13 10:20:23 +0100 | [diff] [blame] | 1 | #!/bin/sh |
Eric Anholt | 46daaca | 2019-06-28 16:35:32 -0700 | [diff] [blame] | 2 | |
| 3 | set -ex |
| 4 | |
Rob Clark | aee1c08 | 2020-08-31 09:04:24 -0700 | [diff] [blame] | 5 | DEQP_WIDTH=${DEQP_WIDTH:-256} |
| 6 | DEQP_HEIGHT=${DEQP_HEIGHT:-256} |
| 7 | DEQP_CONFIG=${DEQP_CONFIG:-rgba8888d24s8ms0} |
| 8 | DEQP_VARIANT=${DEQP_VARIANT:-master} |
| 9 | |
| 10 | DEQP_OPTIONS="$DEQP_OPTIONS --deqp-surface-width=$DEQP_WIDTH --deqp-surface-height=$DEQP_HEIGHT" |
Tomeu Vizoso | d62dd8b | 2019-12-13 10:20:23 +0100 | [diff] [blame] | 11 | DEQP_OPTIONS="$DEQP_OPTIONS --deqp-surface-type=pbuffer" |
Rob Clark | aee1c08 | 2020-08-31 09:04:24 -0700 | [diff] [blame] | 12 | DEQP_OPTIONS="$DEQP_OPTIONS --deqp-gl-config-name=$DEQP_CONFIG" |
Tomeu Vizoso | d62dd8b | 2019-12-13 10:20:23 +0100 | [diff] [blame] | 13 | DEQP_OPTIONS="$DEQP_OPTIONS --deqp-visibility=hidden" |
Eric Anholt | 46daaca | 2019-06-28 16:35:32 -0700 | [diff] [blame] | 14 | |
Eric Anholt | eca02ec | 2020-05-28 12:37:29 -0700 | [diff] [blame] | 15 | # deqp's shader cache (for vulkan) is not multiprocess safe for a common |
| 16 | # filename, see: |
| 17 | # https://gitlab.freedesktop.org/mesa/parallel-deqp-runner/-/merge_requests/13 |
| 18 | DEQP_OPTIONS="$DEQP_OPTIONS --deqp-shadercache=disable" |
| 19 | |
Eric Anholt | 46daaca | 2019-06-28 16:35:32 -0700 | [diff] [blame] | 20 | if [ -z "$DEQP_VER" ]; then |
Samuel Pitoiset | 16b999b | 2019-11-19 08:39:00 +0100 | [diff] [blame] | 21 | echo 'DEQP_VER must be set to something like "gles2", "gles31" or "vk" for the test run' |
Eric Anholt | 46daaca | 2019-06-28 16:35:32 -0700 | [diff] [blame] | 22 | exit 1 |
| 23 | fi |
| 24 | |
Tomeu Vizoso | d62dd8b | 2019-12-13 10:20:23 +0100 | [diff] [blame] | 25 | if [ "$DEQP_VER" = "vk" ]; then |
Samuel Pitoiset | f32bf4f | 2019-11-14 12:09:44 +0100 | [diff] [blame] | 26 | if [ -z "$VK_DRIVER" ]; then |
| 27 | echo 'VK_DRIVER must be to something like "radeon" or "intel" for the test run' |
| 28 | exit 1 |
| 29 | fi |
| 30 | fi |
| 31 | |
Eric Anholt | 46daaca | 2019-06-28 16:35:32 -0700 | [diff] [blame] | 32 | if [ -z "$DEQP_SKIPS" ]; then |
| 33 | echo 'DEQP_SKIPS must be set to something like "deqp-default-skips.txt"' |
| 34 | exit 1 |
| 35 | fi |
| 36 | |
Tomeu Vizoso | 92f3c51 | 2020-03-24 12:58:30 +0100 | [diff] [blame] | 37 | INSTALL=`pwd`/install |
Eric Anholt | 46daaca | 2019-06-28 16:35:32 -0700 | [diff] [blame] | 38 | |
| 39 | # Set up the driver environment. |
| 40 | export LD_LIBRARY_PATH=`pwd`/install/lib/ |
| 41 | export EGL_PLATFORM=surfaceless |
Eric Anholt | 6766d51 | 2020-05-26 11:18:33 -0700 | [diff] [blame] | 42 | export VK_ICD_FILENAMES=`pwd`/install/share/vulkan/icd.d/"$VK_DRIVER"_icd.`uname -m`.json |
Eric Anholt | 46daaca | 2019-06-28 16:35:32 -0700 | [diff] [blame] | 43 | |
| 44 | # the runner was failing to look for libkms in /usr/local/lib for some reason |
| 45 | # I never figured out. |
| 46 | export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib |
| 47 | |
| 48 | RESULTS=`pwd`/results |
| 49 | mkdir -p $RESULTS |
| 50 | |
Samuel Pitoiset | 16b999b | 2019-11-19 08:39:00 +0100 | [diff] [blame] | 51 | # Generate test case list file. |
Tomeu Vizoso | d62dd8b | 2019-12-13 10:20:23 +0100 | [diff] [blame] | 52 | if [ "$DEQP_VER" = "vk" ]; then |
Rob Clark | aee1c08 | 2020-08-31 09:04:24 -0700 | [diff] [blame] | 53 | cp /deqp/mustpass/vk-$DEQP_VARIANT.txt /tmp/case-list.txt |
Samuel Pitoiset | 16b999b | 2019-11-19 08:39:00 +0100 | [diff] [blame] | 54 | DEQP=/deqp/external/vulkancts/modules/vulkan/deqp-vk |
Tomeu Vizoso | 287bf5f | 2020-05-13 09:46:06 +0200 | [diff] [blame] | 55 | elif [ "$DEQP_VER" = "gles2" -o "$DEQP_VER" = "gles3" -o "$DEQP_VER" = "gles31" ]; then |
Rob Clark | aee1c08 | 2020-08-31 09:04:24 -0700 | [diff] [blame] | 56 | cp /deqp/mustpass/$DEQP_VER-$DEQP_VARIANT.txt /tmp/case-list.txt |
Samuel Pitoiset | 16b999b | 2019-11-19 08:39:00 +0100 | [diff] [blame] | 57 | DEQP=/deqp/modules/$DEQP_VER/deqp-$DEQP_VER |
Tomeu Vizoso | 287bf5f | 2020-05-13 09:46:06 +0200 | [diff] [blame] | 58 | SUITE=dEQP |
| 59 | else |
Rob Clark | aee1c08 | 2020-08-31 09:04:24 -0700 | [diff] [blame] | 60 | cp /deqp/mustpass/$DEQP_VER-$DEQP_VARIANT.txt /tmp/case-list.txt |
Tomeu Vizoso | 287bf5f | 2020-05-13 09:46:06 +0200 | [diff] [blame] | 61 | DEQP=/deqp/external/openglcts/modules/glcts |
| 62 | SUITE=KHR |
Samuel Pitoiset | 16b999b | 2019-11-19 08:39:00 +0100 | [diff] [blame] | 63 | fi |
Eric Anholt | 46daaca | 2019-06-28 16:35:32 -0700 | [diff] [blame] | 64 | |
Eric Anholt | 46daaca | 2019-06-28 16:35:32 -0700 | [diff] [blame] | 65 | # If the job is parallel, take the corresponding fraction of the caselist. |
| 66 | # Note: N~M is a gnu sed extension to match every nth line (first line is #1). |
| 67 | if [ -n "$CI_NODE_INDEX" ]; then |
| 68 | sed -ni $CI_NODE_INDEX~$CI_NODE_TOTAL"p" /tmp/case-list.txt |
| 69 | fi |
| 70 | |
Eric Anholt | 5082ac0 | 2020-04-17 12:02:37 -0700 | [diff] [blame] | 71 | if [ -n "$DEQP_CASELIST_FILTER" ]; then |
Eric Anholt | 90cf494 | 2020-05-15 17:18:12 -0700 | [diff] [blame] | 72 | sed -ni "/$DEQP_CASELIST_FILTER/p" /tmp/case-list.txt |
Eric Anholt | 5082ac0 | 2020-04-17 12:02:37 -0700 | [diff] [blame] | 73 | fi |
| 74 | |
Eric Anholt | 46daaca | 2019-06-28 16:35:32 -0700 | [diff] [blame] | 75 | if [ ! -s /tmp/case-list.txt ]; then |
| 76 | echo "Caselist generation failed" |
| 77 | exit 1 |
| 78 | fi |
| 79 | |
Eric Anholt | f08c810 | 2019-11-04 10:54:41 -0800 | [diff] [blame] | 80 | if [ -n "$DEQP_EXPECTED_FAILS" ]; then |
Tomeu Vizoso | 92f3c51 | 2020-03-24 12:58:30 +0100 | [diff] [blame] | 81 | XFAIL="--xfail-list $INSTALL/$DEQP_EXPECTED_FAILS" |
Eric Anholt | f08c810 | 2019-11-04 10:54:41 -0800 | [diff] [blame] | 82 | fi |
Eric Anholt | 46daaca | 2019-06-28 16:35:32 -0700 | [diff] [blame] | 83 | |
Eric Anholt | f08c810 | 2019-11-04 10:54:41 -0800 | [diff] [blame] | 84 | set +e |
| 85 | |
Samuel Pitoiset | 4668a08 | 2020-03-05 15:20:34 +0100 | [diff] [blame] | 86 | if [ -n "$DEQP_PARALLEL" ]; then |
| 87 | JOB="--job $DEQP_PARALLEL" |
Eric Anholt | fd24a95 | 2020-06-26 10:59:41 -0700 | [diff] [blame] | 88 | elif [ -n "$FDO_CI_CONCURRENT" ]; then |
| 89 | JOB="--job $FDO_CI_CONCURRENT" |
| 90 | else |
| 91 | JOB="--job 4" |
Samuel Pitoiset | 4668a08 | 2020-03-05 15:20:34 +0100 | [diff] [blame] | 92 | fi |
| 93 | |
Rob Clark | fdaf777 | 2019-11-17 11:33:01 -0800 | [diff] [blame] | 94 | run_cts() { |
Samuel Pitoiset | 16b999b | 2019-11-19 08:39:00 +0100 | [diff] [blame] | 95 | deqp=$1 |
| 96 | caselist=$2 |
| 97 | output=$3 |
Rob Clark | fdaf777 | 2019-11-17 11:33:01 -0800 | [diff] [blame] | 98 | deqp-runner \ |
Samuel Pitoiset | 16b999b | 2019-11-19 08:39:00 +0100 | [diff] [blame] | 99 | --deqp $deqp \ |
Rob Clark | fdaf777 | 2019-11-17 11:33:01 -0800 | [diff] [blame] | 100 | --output $output \ |
| 101 | --caselist $caselist \ |
Tomeu Vizoso | 92f3c51 | 2020-03-24 12:58:30 +0100 | [diff] [blame] | 102 | --exclude-list $INSTALL/$DEQP_SKIPS \ |
Eric Anholt | a9e6a3e | 2020-04-17 12:28:58 -0700 | [diff] [blame] | 103 | --compact-display false \ |
Rob Clark | fdaf777 | 2019-11-17 11:33:01 -0800 | [diff] [blame] | 104 | $XFAIL \ |
Samuel Pitoiset | 4668a08 | 2020-03-05 15:20:34 +0100 | [diff] [blame] | 105 | $JOB \ |
Rob Clark | fdaf777 | 2019-11-17 11:33:01 -0800 | [diff] [blame] | 106 | --allow-flakes true \ |
Tomeu Vizoso | d62dd8b | 2019-12-13 10:20:23 +0100 | [diff] [blame] | 107 | $DEQP_RUNNER_OPTIONS \ |
Rob Clark | fdaf777 | 2019-11-17 11:33:01 -0800 | [diff] [blame] | 108 | -- \ |
Tomeu Vizoso | d62dd8b | 2019-12-13 10:20:23 +0100 | [diff] [blame] | 109 | $DEQP_OPTIONS |
Rob Clark | fdaf777 | 2019-11-17 11:33:01 -0800 | [diff] [blame] | 110 | } |
| 111 | |
| 112 | report_flakes() { |
| 113 | if [ -z "$FLAKES_CHANNEL" ]; then |
| 114 | return 0 |
| 115 | fi |
| 116 | flakes=$1 |
Eric Anholt | 2637961 | 2020-05-05 12:17:49 -0700 | [diff] [blame] | 117 | # The nick needs to be something unique so that multiple runners |
| 118 | # connecting at the same time don't race for one nick and get blocked. |
| 119 | # freenode has a 16-char limit on nicks (9 is the IETF standard, but |
| 120 | # various servers extend that). So, trim off the common prefixes of the |
| 121 | # runner name, and append the job ID so that software runners with more |
| 122 | # than one concurrent job (think swrast) don't collide. For freedreno, |
| 123 | # that gives us a nick as long as db410c-N-JJJJJJJJ, and it'll be a while |
| 124 | # before we make it to 9-digit jobs (we're at 7 so far). |
| 125 | runner=`echo $CI_RUNNER_DESCRIPTION | sed 's|mesa-||' | sed 's|google-freedreno-||g'` |
| 126 | bot="$runner-$CI_JOB_ID" |
Rob Clark | fdaf777 | 2019-11-17 11:33:01 -0800 | [diff] [blame] | 127 | channel="$FLAKES_CHANNEL" |
| 128 | ( |
| 129 | echo NICK $bot |
| 130 | echo USER $bot unused unused :Gitlab CI Notifier |
| 131 | sleep 10 |
| 132 | echo "JOIN $channel" |
| 133 | sleep 1 |
| 134 | desc="Flakes detected in job: $CI_JOB_URL on $CI_RUNNER_DESCRIPTION" |
Eric Anholt | 2c50176 | 2020-05-05 10:44:46 -0700 | [diff] [blame] | 135 | if [ -n "$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME" ]; then |
Rob Clark | fdaf777 | 2019-11-17 11:33:01 -0800 | [diff] [blame] | 136 | desc="$desc on branch $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME ($CI_MERGE_REQUEST_TITLE)" |
Eric Anholt | 2c50176 | 2020-05-05 10:44:46 -0700 | [diff] [blame] | 137 | elif [ -n "$CI_COMMIT_BRANCH" ]; then |
| 138 | desc="$desc on branch $CI_COMMIT_BRANCH ($CI_COMMIT_TITLE)" |
Rob Clark | fdaf777 | 2019-11-17 11:33:01 -0800 | [diff] [blame] | 139 | fi |
| 140 | echo "PRIVMSG $channel :$desc" |
| 141 | for flake in `cat $flakes`; do |
| 142 | echo "PRIVMSG $channel :$flake" |
| 143 | done |
| 144 | echo "PRIVMSG $channel :See $CI_JOB_URL/artifacts/browse/results/" |
| 145 | echo "QUIT" |
| 146 | ) | nc irc.freenode.net 6667 > /dev/null |
| 147 | |
| 148 | } |
| 149 | |
Rob Clark | 415d565 | 2019-11-17 11:57:26 -0800 | [diff] [blame] | 150 | extract_xml_result() { |
| 151 | testcase=$1 |
| 152 | shift 1 |
| 153 | qpas=$* |
| 154 | start="#beginTestCaseResult $testcase" |
Eric Anholt | f0c102c | 2020-05-26 13:28:30 -0700 | [diff] [blame] | 155 | |
| 156 | # Pick the first QPA mentioning our testcase |
| 157 | qpa=`grep -l "$start" $qpas | head -n 1` |
| 158 | |
| 159 | # If we found one, go extract just that testcase's contents from the QPA |
| 160 | # to a new QPA, then do testlog-to-xml on that. |
| 161 | if [ -n "$qpa" ]; then |
Rob Clark | 415d565 | 2019-11-17 11:57:26 -0800 | [diff] [blame] | 162 | while IFS= read -r line; do |
| 163 | if [ "$line" = "$start" ]; then |
| 164 | dst="$testcase.qpa" |
| 165 | echo "#beginSession" > $dst |
Eric Anholt | 44d28dc | 2020-06-11 13:19:11 -0700 | [diff] [blame] | 166 | echo "$line" >> $dst |
Rob Clark | 415d565 | 2019-11-17 11:57:26 -0800 | [diff] [blame] | 167 | while IFS= read -r line; do |
| 168 | if [ "$line" = "#endTestCaseResult" ]; then |
Eric Anholt | 44d28dc | 2020-06-11 13:19:11 -0700 | [diff] [blame] | 169 | echo "$line" >> $dst |
Rob Clark | 415d565 | 2019-11-17 11:57:26 -0800 | [diff] [blame] | 170 | echo "#endSession" >> $dst |
Eric Anholt | 951e101 | 2020-04-17 12:58:59 -0700 | [diff] [blame] | 171 | /deqp/executor/testlog-to-xml $dst "$RESULTS/$testcase$DEQP_RUN_SUFFIX.xml" |
Rob Clark | 415d565 | 2019-11-17 11:57:26 -0800 | [diff] [blame] | 172 | # copy the stylesheets here so they only end up in artifacts |
| 173 | # if we have one or more result xml in artifacts |
Rob Clark | b706a15 | 2020-01-15 08:50:55 -0800 | [diff] [blame] | 174 | cp /deqp/testlog.css "$RESULTS/" |
| 175 | cp /deqp/testlog.xsl "$RESULTS/" |
Rob Clark | 415d565 | 2019-11-17 11:57:26 -0800 | [diff] [blame] | 176 | return 0 |
| 177 | fi |
Eric Anholt | 44d28dc | 2020-06-11 13:19:11 -0700 | [diff] [blame] | 178 | echo "$line" >> $dst |
Rob Clark | 415d565 | 2019-11-17 11:57:26 -0800 | [diff] [blame] | 179 | done |
| 180 | return 1 |
| 181 | fi |
| 182 | done < $qpa |
Eric Anholt | f0c102c | 2020-05-26 13:28:30 -0700 | [diff] [blame] | 183 | fi |
Rob Clark | 415d565 | 2019-11-17 11:57:26 -0800 | [diff] [blame] | 184 | } |
| 185 | |
| 186 | extract_xml_results() { |
| 187 | qpas=$* |
| 188 | while IFS= read -r testcase; do |
| 189 | testcase=${testcase%,*} |
| 190 | extract_xml_result $testcase $qpas |
| 191 | done |
| 192 | } |
| 193 | |
Rob Clark | 9f422cb | 2019-11-17 12:04:50 -0800 | [diff] [blame] | 194 | # Generate junit results |
| 195 | generate_junit() { |
| 196 | results=$1 |
| 197 | echo "<?xml version=\"1.0\" encoding=\"utf-8\"?>" |
| 198 | echo "<testsuites>" |
| 199 | echo "<testsuite name=\"$DEQP_VER-$CI_NODE_INDEX\">" |
| 200 | while read line; do |
| 201 | testcase=${line%,*} |
| 202 | result=${line#*,} |
| 203 | # avoid counting Skip's in the # of tests: |
| 204 | if [ "$result" = "Skip" ]; then |
| 205 | continue; |
| 206 | fi |
| 207 | echo "<testcase name=\"$testcase\">" |
| 208 | if [ "$result" != "Pass" ]; then |
| 209 | echo "<failure type=\"$result\">" |
| 210 | echo "$result: See $CI_JOB_URL/artifacts/results/$testcase.xml" |
| 211 | echo "</failure>" |
| 212 | fi |
| 213 | echo "</testcase>" |
| 214 | done < $results |
| 215 | echo "</testsuite>" |
| 216 | echo "</testsuites>" |
| 217 | } |
| 218 | |
Eric Anholt | ff11836 | 2020-03-05 14:35:55 -0800 | [diff] [blame] | 219 | parse_renderer() { |
| 220 | RENDERER=`grep -A1 TestCaseResult.\*info.renderer $RESULTS/deqp-info.qpa | grep '<Text' | sed 's|.*<Text>||g' | sed 's|</Text>||g'` |
| 221 | VERSION=`grep -A1 TestCaseResult.\*info.version $RESULTS/deqp-info.qpa | grep '<Text' | sed 's|.*<Text>||g' | sed 's|</Text>||g'` |
| 222 | echo "Renderer: $RENDERER" |
| 223 | echo "Version: $VERSION " |
Eric Anholt | 8b221e0 | 2020-04-17 12:39:32 -0700 | [diff] [blame] | 224 | |
| 225 | if ! echo $RENDERER | grep -q $DEQP_EXPECTED_RENDERER; then |
| 226 | echo "Expected GL_RENDERER $DEQP_EXPECTED_RENDERER" |
| 227 | exit 1 |
| 228 | fi |
Eric Anholt | ff11836 | 2020-03-05 14:35:55 -0800 | [diff] [blame] | 229 | } |
| 230 | |
| 231 | check_renderer() { |
Eric Anholt | 33e0821 | 2020-05-26 10:53:05 -0700 | [diff] [blame] | 232 | echo "Capturing renderer info for GLES driver sanity checks" |
Eric Anholt | ff11836 | 2020-03-05 14:35:55 -0800 | [diff] [blame] | 233 | # If you're having trouble loading your driver, uncommenting this may help |
| 234 | # debug. |
| 235 | # export EGL_LOG_LEVEL=debug |
Tomeu Vizoso | ad3ef6d | 2020-04-01 09:17:25 +0200 | [diff] [blame] | 236 | VERSION=`echo $DEQP_VER | tr '[a-z]' '[A-Z]'` |
Tomeu Vizoso | 287bf5f | 2020-05-13 09:46:06 +0200 | [diff] [blame] | 237 | $DEQP $DEQP_OPTIONS --deqp-case=$SUITE-$VERSION.info.\* --deqp-log-filename=$RESULTS/deqp-info.qpa |
Eric Anholt | ff11836 | 2020-03-05 14:35:55 -0800 | [diff] [blame] | 238 | parse_renderer |
| 239 | } |
| 240 | |
Eric Anholt | 33e0821 | 2020-05-26 10:53:05 -0700 | [diff] [blame] | 241 | check_vk_device_name() { |
| 242 | echo "Capturing device info for VK driver sanity checks" |
| 243 | $DEQP $DEQP_OPTIONS --deqp-case=dEQP-VK.info.device --deqp-log-filename=$RESULTS/deqp-info.qpa |
| 244 | DEVICENAME=`grep deviceName $RESULTS/deqp-info.qpa | sed 's|deviceName: ||g'` |
| 245 | echo "deviceName: $DEVICENAME" |
Eric Anholt | 2d1c607 | 2020-08-13 15:10:18 -0700 | [diff] [blame] | 246 | if [ -n "$DEQP_EXPECTED_RENDERER" -a "x$DEVICENAME" != "x$DEQP_EXPECTED_RENDERER" ]; then |
Eric Anholt | 33e0821 | 2020-05-26 10:53:05 -0700 | [diff] [blame] | 247 | echo "Expected deviceName $DEQP_EXPECTED_RENDERER" |
| 248 | exit 1 |
| 249 | fi |
| 250 | } |
| 251 | |
Rob Clark | fdaf777 | 2019-11-17 11:33:01 -0800 | [diff] [blame] | 252 | # wrapper to supress +x to avoid spamming the log |
| 253 | quiet() { |
| 254 | set +x |
| 255 | "$@" |
| 256 | set -x |
| 257 | } |
| 258 | |
Tomeu Vizoso | ad3ef6d | 2020-04-01 09:17:25 +0200 | [diff] [blame] | 259 | if [ "$GALLIUM_DRIVER" = "virpipe" ]; then |
| 260 | # deqp is to use virpipe, and virgl_test_server llvmpipe |
| 261 | export GALLIUM_DRIVER="$GALLIUM_DRIVER" |
| 262 | |
Tomeu Vizoso | 2102d5e | 2020-05-12 10:18:48 +0200 | [diff] [blame] | 263 | VTEST_ARGS="--use-egl-surfaceless" |
| 264 | if [ "$VIRGL_HOST_API" = "GLES" ]; then |
| 265 | VTEST_ARGS="$VTEST_ARGS --use-gles" |
| 266 | fi |
| 267 | |
Tomeu Vizoso | ad3ef6d | 2020-04-01 09:17:25 +0200 | [diff] [blame] | 268 | GALLIUM_DRIVER=llvmpipe \ |
| 269 | GALLIVM_PERF="nopt,no_filter_hacks" \ |
Tomeu Vizoso | 2102d5e | 2020-05-12 10:18:48 +0200 | [diff] [blame] | 270 | virgl_test_server $VTEST_ARGS >$RESULTS/vtest-log.txt 2>&1 & |
Tomeu Vizoso | ad3ef6d | 2020-04-01 09:17:25 +0200 | [diff] [blame] | 271 | |
| 272 | sleep 1 |
| 273 | fi |
| 274 | |
Eric Anholt | 33e0821 | 2020-05-26 10:53:05 -0700 | [diff] [blame] | 275 | if [ $DEQP_VER = vk ]; then |
| 276 | quiet check_vk_device_name |
| 277 | else |
Eric Anholt | ff11836 | 2020-03-05 14:35:55 -0800 | [diff] [blame] | 278 | quiet check_renderer |
| 279 | fi |
| 280 | |
Eric Anholt | 951e101 | 2020-04-17 12:58:59 -0700 | [diff] [blame] | 281 | RESULTSFILE=$RESULTS/cts-runner-results$DEQP_RUN_SUFFIX.txt |
| 282 | UNEXPECTED_RESULTSFILE=$RESULTS/cts-runner-unexpected-results$DEQP_RUN_SUFFIX.txt |
| 283 | FLAKESFILE=$RESULTS/cts-runner-flakes$DEQP_RUN_SUFFIX.txt |
| 284 | |
| 285 | run_cts $DEQP /tmp/case-list.txt $RESULTSFILE |
Eric Anholt | 46daaca | 2019-06-28 16:35:32 -0700 | [diff] [blame] | 286 | DEQP_EXITCODE=$? |
| 287 | |
Tomeu Vizoso | 7455403 | 2020-07-28 08:22:34 +0200 | [diff] [blame] | 288 | echo "System load: $(cut -d' ' -f1-3 < /proc/loadavg)" |
| 289 | echo "# of CPU cores: $(cat /proc/cpuinfo | grep processor | wc -l)" |
| 290 | |
Eric Anholt | 7f3f9b2 | 2020-02-19 10:29:32 -0800 | [diff] [blame] | 291 | # junit is disabled, because it overloads gitlab.freedesktop.org to parse it. |
Eric Anholt | 951e101 | 2020-04-17 12:58:59 -0700 | [diff] [blame] | 292 | #quiet generate_junit $RESULTSFILE > $RESULTS/results.xml |
Rob Clark | 9f422cb | 2019-11-17 12:04:50 -0800 | [diff] [blame] | 293 | |
Eric Anholt | 46daaca | 2019-06-28 16:35:32 -0700 | [diff] [blame] | 294 | if [ $DEQP_EXITCODE -ne 0 ]; then |
Rob Clark | cc6484f | 2019-11-17 11:28:16 -0800 | [diff] [blame] | 295 | # preserve caselist files in case of failures: |
Eric Anholt | 62998f6 | 2019-12-17 11:19:07 -0800 | [diff] [blame] | 296 | cp /tmp/deqp_runner.*.txt $RESULTS/ |
Eric Anholt | c343d00 | 2020-05-26 10:15:19 -0700 | [diff] [blame] | 297 | egrep -v ",Pass|,Skip|,ExpectedFail" $RESULTSFILE > $UNEXPECTED_RESULTSFILE |
Rob Clark | fdaf777 | 2019-11-17 11:33:01 -0800 | [diff] [blame] | 298 | |
Eric Anholt | 30da82c | 2020-08-19 10:30:12 -0700 | [diff] [blame] | 299 | # deqp-runner's flake detection won't perfectly detect all flakes, so |
| 300 | # allow the driver to list some known flakes that won't intermittently |
| 301 | # fail people's pipelines (while still allowing them to run and be |
| 302 | # reported to IRC in the usual flake detection path). If we had some |
| 303 | # fails listed (so this wasn't a total runner failure), then filter out |
| 304 | # the known flakes and see if there are any issues left. |
| 305 | if [ -n "$DEQP_FLAKES" -a -s $UNEXPECTED_RESULTSFILE ]; then |
| 306 | set +x |
| 307 | while read line; do |
| 308 | line=`echo $line | sed 's|#.*||g'` |
| 309 | if [ -n "$line" ]; then |
| 310 | sed -i "/$line/d" $UNEXPECTED_RESULTSFILE |
| 311 | fi |
| 312 | done < $INSTALL/$DEQP_FLAKES |
| 313 | set -x |
| 314 | |
| 315 | if [ ! -s $UNEXPECTED_RESULTSFILE ]; then |
| 316 | exit 0 |
| 317 | fi |
| 318 | fi |
| 319 | |
Tomeu Vizoso | d62dd8b | 2019-12-13 10:20:23 +0100 | [diff] [blame] | 320 | if [ -z "$DEQP_NO_SAVE_RESULTS" ]; then |
Eric Anholt | 994e258 | 2020-02-13 16:44:04 -0800 | [diff] [blame] | 321 | echo "Some unexpected results found (see cts-runner-results.txt in artifacts for full results):" |
Eric Anholt | c343d00 | 2020-05-26 10:15:19 -0700 | [diff] [blame] | 322 | head -n 50 $UNEXPECTED_RESULTSFILE |
Eric Anholt | 994e258 | 2020-02-13 16:44:04 -0800 | [diff] [blame] | 323 | |
Tomeu Vizoso | d62dd8b | 2019-12-13 10:20:23 +0100 | [diff] [blame] | 324 | # Save the logs for up to the first 50 unexpected results: |
Eric Anholt | c343d00 | 2020-05-26 10:15:19 -0700 | [diff] [blame] | 325 | head -n 50 $UNEXPECTED_RESULTSFILE | quiet extract_xml_results /tmp/*.qpa |
Eric Anholt | 994e258 | 2020-02-13 16:44:04 -0800 | [diff] [blame] | 326 | else |
| 327 | echo "Unexpected results found:" |
Eric Anholt | c343d00 | 2020-05-26 10:15:19 -0700 | [diff] [blame] | 328 | cat $UNEXPECTED_RESULTSFILE |
Tomeu Vizoso | d62dd8b | 2019-12-13 10:20:23 +0100 | [diff] [blame] | 329 | fi |
Rob Clark | fdaf777 | 2019-11-17 11:33:01 -0800 | [diff] [blame] | 330 | else |
Eric Anholt | b8c66ae | 2020-04-17 13:01:50 -0700 | [diff] [blame] | 331 | grep ",Flake" $RESULTSFILE > $FLAKESFILE |
Rob Clark | fdaf777 | 2019-11-17 11:33:01 -0800 | [diff] [blame] | 332 | |
Samuel Pitoiset | cc2c3b4 | 2020-04-30 11:40:42 +0200 | [diff] [blame] | 333 | count=`cat $FLAKESFILE | wc -l` |
Rob Clark | fdaf777 | 2019-11-17 11:33:01 -0800 | [diff] [blame] | 334 | if [ $count -gt 0 ]; then |
| 335 | echo "Some flakes found (see cts-runner-flakes.txt in artifacts for full results):" |
Eric Anholt | 951e101 | 2020-04-17 12:58:59 -0700 | [diff] [blame] | 336 | head -n 50 $FLAKESFILE |
Rob Clark | fdaf777 | 2019-11-17 11:33:01 -0800 | [diff] [blame] | 337 | |
Tomeu Vizoso | d62dd8b | 2019-12-13 10:20:23 +0100 | [diff] [blame] | 338 | if [ -z "$DEQP_NO_SAVE_RESULTS" ]; then |
| 339 | # Save the logs for up to the first 50 flakes: |
Eric Anholt | 951e101 | 2020-04-17 12:58:59 -0700 | [diff] [blame] | 340 | head -n 50 $FLAKESFILE | quiet extract_xml_results /tmp/*.qpa |
Tomeu Vizoso | d62dd8b | 2019-12-13 10:20:23 +0100 | [diff] [blame] | 341 | fi |
Rob Clark | 8af7551 | 2019-11-22 13:30:18 -0800 | [diff] [blame] | 342 | |
Rob Clark | fdaf777 | 2019-11-17 11:33:01 -0800 | [diff] [blame] | 343 | # Report the flakes to IRC channel for monitoring (if configured): |
Eric Anholt | 951e101 | 2020-04-17 12:58:59 -0700 | [diff] [blame] | 344 | quiet report_flakes $FLAKESFILE |
Rob Clark | fdaf777 | 2019-11-17 11:33:01 -0800 | [diff] [blame] | 345 | else |
| 346 | # no flakes, so clean-up: |
Eric Anholt | 951e101 | 2020-04-17 12:58:59 -0700 | [diff] [blame] | 347 | rm $FLAKESFILE |
Rob Clark | fdaf777 | 2019-11-17 11:33:01 -0800 | [diff] [blame] | 348 | fi |
Eric Anholt | 7859eb1 | 2019-09-12 12:34:50 -0700 | [diff] [blame] | 349 | fi |
Rob Clark | 59ed90f | 2019-11-17 11:16:09 -0800 | [diff] [blame] | 350 | |
| 351 | exit $DEQP_EXITCODE |