Eric Anholt | 46daaca | 2019-06-28 16:35:32 -0700 | [diff] [blame] | 1 | #!/bin/bash |
| 2 | |
| 3 | set -ex |
| 4 | |
| 5 | DEQP_OPTIONS=(--deqp-surface-width=256 --deqp-surface-height=256) |
| 6 | DEQP_OPTIONS+=(--deqp-surface-type=pbuffer) |
| 7 | DEQP_OPTIONS+=(--deqp-gl-config-name=rgba8888d24s8ms0) |
| 8 | DEQP_OPTIONS+=(--deqp-visibility=hidden) |
Eric Anholt | 46daaca | 2019-06-28 16:35:32 -0700 | [diff] [blame] | 9 | |
Eric Anholt | 8d4742f | 2019-09-03 15:52:33 -0700 | [diff] [blame] | 10 | # It would be nice to be able to enable the watchdog, so that hangs in a test |
| 11 | # don't need to wait the full hour for the run to time out. However, some |
| 12 | # shaders end up taking long enough to compile |
| 13 | # (dEQP-GLES31.functional.ubo.random.all_per_block_buffers.20 for example) |
| 14 | # that they'll sporadically trigger the watchdog. |
| 15 | #DEQP_OPTIONS+=(--deqp-watchdog=enable) |
| 16 | |
Eric Anholt | 46daaca | 2019-06-28 16:35:32 -0700 | [diff] [blame] | 17 | if [ -z "$DEQP_VER" ]; then |
| 18 | echo 'DEQP_VER must be set to something like "gles2" or "gles31" for the test run' |
| 19 | exit 1 |
| 20 | fi |
| 21 | |
| 22 | if [ -z "$DEQP_SKIPS" ]; then |
| 23 | echo 'DEQP_SKIPS must be set to something like "deqp-default-skips.txt"' |
| 24 | exit 1 |
| 25 | fi |
| 26 | |
Eric Anholt | f08c810 | 2019-11-04 10:54:41 -0800 | [diff] [blame] | 27 | ARTIFACTS=`pwd`/artifacts |
Eric Anholt | 46daaca | 2019-06-28 16:35:32 -0700 | [diff] [blame] | 28 | |
| 29 | # Set up the driver environment. |
| 30 | export LD_LIBRARY_PATH=`pwd`/install/lib/ |
| 31 | export EGL_PLATFORM=surfaceless |
| 32 | |
| 33 | # the runner was failing to look for libkms in /usr/local/lib for some reason |
| 34 | # I never figured out. |
| 35 | export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib |
| 36 | |
| 37 | RESULTS=`pwd`/results |
| 38 | mkdir -p $RESULTS |
| 39 | |
Eric Anholt | 46daaca | 2019-06-28 16:35:32 -0700 | [diff] [blame] | 40 | # Generate test case list file |
| 41 | cp /deqp/mustpass/$DEQP_VER-master.txt /tmp/case-list.txt |
| 42 | |
Eric Anholt | 46daaca | 2019-06-28 16:35:32 -0700 | [diff] [blame] | 43 | # If the job is parallel, take the corresponding fraction of the caselist. |
| 44 | # Note: N~M is a gnu sed extension to match every nth line (first line is #1). |
| 45 | if [ -n "$CI_NODE_INDEX" ]; then |
| 46 | sed -ni $CI_NODE_INDEX~$CI_NODE_TOTAL"p" /tmp/case-list.txt |
| 47 | fi |
| 48 | |
| 49 | if [ ! -s /tmp/case-list.txt ]; then |
| 50 | echo "Caselist generation failed" |
| 51 | exit 1 |
| 52 | fi |
| 53 | |
Eric Anholt | f08c810 | 2019-11-04 10:54:41 -0800 | [diff] [blame] | 54 | if [ -n "$DEQP_EXPECTED_FAILS" ]; then |
| 55 | XFAIL="--xfail-list $ARTIFACTS/$DEQP_EXPECTED_FAILS" |
| 56 | fi |
Eric Anholt | 46daaca | 2019-06-28 16:35:32 -0700 | [diff] [blame] | 57 | |
Eric Anholt | f08c810 | 2019-11-04 10:54:41 -0800 | [diff] [blame] | 58 | set +e |
| 59 | |
Rob Clark | fdaf777 | 2019-11-17 11:33:01 -0800 | [diff] [blame] | 60 | run_cts() { |
| 61 | caselist=$1 |
| 62 | output=$2 |
| 63 | deqp-runner \ |
| 64 | --deqp /deqp/modules/$DEQP_VER/deqp-$DEQP_VER \ |
| 65 | --output $output \ |
| 66 | --caselist $caselist \ |
| 67 | --exclude-list $ARTIFACTS/$DEQP_SKIPS \ |
| 68 | $XFAIL \ |
| 69 | --job ${DEQP_PARALLEL:-1} \ |
| 70 | --allow-flakes true \ |
| 71 | -- \ |
| 72 | "${DEQP_OPTIONS[@]}" |
| 73 | } |
| 74 | |
| 75 | report_flakes() { |
| 76 | if [ -z "$FLAKES_CHANNEL" ]; then |
| 77 | return 0 |
| 78 | fi |
| 79 | flakes=$1 |
| 80 | bot="$CI_RUNNER_DESCRIPTION-$CI_PIPELINE_ID" |
| 81 | channel="$FLAKES_CHANNEL" |
| 82 | ( |
| 83 | echo NICK $bot |
| 84 | echo USER $bot unused unused :Gitlab CI Notifier |
| 85 | sleep 10 |
| 86 | echo "JOIN $channel" |
| 87 | sleep 1 |
| 88 | desc="Flakes detected in job: $CI_JOB_URL on $CI_RUNNER_DESCRIPTION" |
| 89 | if [ -n "CI_MERGE_REQUEST_SOURCE_BRANCH_NAME" ]; then |
| 90 | desc="$desc on branch $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME ($CI_MERGE_REQUEST_TITLE)" |
| 91 | fi |
| 92 | echo "PRIVMSG $channel :$desc" |
| 93 | for flake in `cat $flakes`; do |
| 94 | echo "PRIVMSG $channel :$flake" |
| 95 | done |
| 96 | echo "PRIVMSG $channel :See $CI_JOB_URL/artifacts/browse/results/" |
| 97 | echo "QUIT" |
| 98 | ) | nc irc.freenode.net 6667 > /dev/null |
| 99 | |
| 100 | } |
| 101 | |
| 102 | # wrapper to supress +x to avoid spamming the log |
| 103 | quiet() { |
| 104 | set +x |
| 105 | "$@" |
| 106 | set -x |
| 107 | } |
| 108 | |
| 109 | run_cts /tmp/case-list.txt $RESULTS/cts-runner-results.txt |
Eric Anholt | 46daaca | 2019-06-28 16:35:32 -0700 | [diff] [blame] | 110 | DEQP_EXITCODE=$? |
| 111 | |
Eric Anholt | 46daaca | 2019-06-28 16:35:32 -0700 | [diff] [blame] | 112 | if [ $DEQP_EXITCODE -ne 0 ]; then |
Rob Clark | cc6484f | 2019-11-17 11:28:16 -0800 | [diff] [blame] | 113 | # preserve caselist files in case of failures: |
| 114 | cp /tmp/cts_runner.*.txt $RESULTS/ |
Eric Anholt | f08c810 | 2019-11-04 10:54:41 -0800 | [diff] [blame] | 115 | echo "Some unexpected results found (see cts-runner-results.txt in artifacts for full results):" |
| 116 | cat $RESULTS/cts-runner-results.txt | \ |
| 117 | grep -v ",Pass" | \ |
| 118 | grep -v ",Skip" | \ |
Rob Clark | 59ed90f | 2019-11-17 11:16:09 -0800 | [diff] [blame] | 119 | grep -v ",ExpectedFail" > \ |
| 120 | $RESULTS/cts-runner-unexpected-results.txt |
| 121 | head -n 50 $RESULTS/cts-runner-unexpected-results.txt |
Rob Clark | fdaf777 | 2019-11-17 11:33:01 -0800 | [diff] [blame] | 122 | |
| 123 | count=`cat $RESULTS/cts-runner-unexpected-results.txt | wc -l` |
| 124 | |
| 125 | # Re-run fails to detect flakes. But use a small threshold, if |
| 126 | # something was fundamentally broken, we don't want to re-run |
| 127 | # the entire caselist |
| 128 | else |
| 129 | cat $RESULTS/cts-runner-results.txt | \ |
| 130 | grep ",Flake" > \ |
| 131 | $RESULTS/cts-runner-flakes.txt |
| 132 | |
| 133 | count=`cat $RESULTS/cts-runner-flakes.txt | wc -l` |
| 134 | if [ $count -gt 0 ]; then |
| 135 | echo "Some flakes found (see cts-runner-flakes.txt in artifacts for full results):" |
| 136 | head -n 50 $RESULTS/cts-runner-flakes.txt |
| 137 | |
Rob Clark | 8af7551 | 2019-11-22 13:30:18 -0800 | [diff] [blame^] | 138 | # Save the logs for up to the first 50 flakes: |
| 139 | head -n 50 $RESULTS/cts-runner-flakes.txt | quiet extract_xml_results /tmp/*.qpa |
| 140 | |
Rob Clark | fdaf777 | 2019-11-17 11:33:01 -0800 | [diff] [blame] | 141 | # Report the flakes to IRC channel for monitoring (if configured): |
| 142 | quiet report_flakes $RESULTS/cts-runner-flakes.txt |
| 143 | else |
| 144 | # no flakes, so clean-up: |
| 145 | rm $RESULTS/cts-runner-flakes.txt |
| 146 | fi |
Eric Anholt | 7859eb1 | 2019-09-12 12:34:50 -0700 | [diff] [blame] | 147 | fi |
Rob Clark | 59ed90f | 2019-11-17 11:16:09 -0800 | [diff] [blame] | 148 | |
| 149 | exit $DEQP_EXITCODE |