epoger@google.com | a2b793c | 2012-05-15 14:58:53 +0000 | [diff] [blame] | 1 | #!/bin/bash |
| 2 | |
| 3 | # Tests for our tools. |
epoger@google.com | 3aa3358 | 2013-01-02 15:53:25 +0000 | [diff] [blame] | 4 | # |
epoger@google.com | 3aa3358 | 2013-01-02 15:53:25 +0000 | [diff] [blame] | 5 | # TODO: currently, this only passes on Linux (which is the platform that |
| 6 | # the housekeeper bot runs on, e.g. |
| 7 | # http://70.32.156.51:10117/builders/Skia_PerCommit_House_Keeping/builds/1415/steps/RunToolSelfTests/logs/stdio ) |
| 8 | # See https://code.google.com/p/skia/issues/detail?id=677 |
| 9 | # ('make tools/tests/run.sh work cross-platform') |
| 10 | # Ideally, these tests should pass on all development platforms... |
| 11 | # otherwise, how can developers be expected to test them before committing a |
| 12 | # change? |
epoger@google.com | a413a53 | 2012-11-12 18:04:51 +0000 | [diff] [blame] | 13 | |
| 14 | # cd into .../trunk so all the paths will work |
| 15 | cd $(dirname $0)/../.. |
epoger@google.com | a2b793c | 2012-05-15 14:58:53 +0000 | [diff] [blame] | 16 | |
| 17 | # TODO: make it look in Release and/or Debug |
| 18 | SKDIFF_BINARY=out/Debug/skdiff |
| 19 | |
bensong@google.com | 4a28f59 | 2013-10-24 18:08:45 +0000 | [diff] [blame] | 20 | # Suffixes of the raw bench data files we want to process. |
epoger@google.com | bd9669a | 2013-02-17 08:33:27 +0000 | [diff] [blame] | 21 | BENCHDATA_FILE_SUFFIXES_YES_INDIVIDUAL_TILES=\ |
bensong@google.com | 4a28f59 | 2013-10-24 18:08:45 +0000 | [diff] [blame] | 22 | "data_skp_scale_1.3061_config_8888_mode_tile_256_256_timeIndividualTiles_bbh_rtree "\ |
| 23 | "data_skp_scale_1.3061_config_8888_mode_tile_256_256_timeIndividualTiles" |
epoger@google.com | bd9669a | 2013-02-17 08:33:27 +0000 | [diff] [blame] | 24 | BENCHDATA_FILE_SUFFIXES_NO_INDIVIDUAL_TILES=\ |
bensong@google.com | 4a28f59 | 2013-10-24 18:08:45 +0000 | [diff] [blame] | 25 | "data_skp_multi_4_scale_1.3061_config_8888_mode_tile_256_256 "\ |
| 26 | "data_skp_scale_1.3061_config_8888_mode_record" |
epoger@google.com | 673816c | 2013-02-15 15:50:23 +0000 | [diff] [blame] | 27 | |
epoger@google.com | a444352 | 2012-05-17 17:12:38 +0000 | [diff] [blame] | 28 | # Compare contents of all files within directories $1 and $2, |
| 29 | # EXCEPT for any dotfiles. |
| 30 | # If there are any differences, a description is written to stdout and |
| 31 | # we exit with a nonzero return value. |
| 32 | # Otherwise, we write nothing to stdout and return. |
epoger@google.com | a2b793c | 2012-05-15 14:58:53 +0000 | [diff] [blame] | 33 | function compare_directories { |
epoger@google.com | a413a53 | 2012-11-12 18:04:51 +0000 | [diff] [blame] | 34 | if [ $# != 2 ]; then |
| 35 | echo "compare_directories requires exactly 2 parameters, got $#" |
| 36 | exit 1 |
| 37 | fi |
epoger@google.com | 34b73a2 | 2013-07-03 17:43:11 +0000 | [diff] [blame] | 38 | diff --recursive --exclude=.* $1 $2 |
epoger@google.com | a2b793c | 2012-05-15 14:58:53 +0000 | [diff] [blame] | 39 | if [ $? != 0 ]; then |
| 40 | echo "failed in: compare_directories $1 $2" |
| 41 | exit 1 |
| 42 | fi |
| 43 | } |
| 44 | |
epoger@google.com | a444352 | 2012-05-17 17:12:38 +0000 | [diff] [blame] | 45 | # Run skdiff with arguments in $1 (plus implicit final argument causing skdiff |
| 46 | # to write its output, if any, to directory $2/output-actual). |
| 47 | # Then compare its results against those in $2/output-expected. |
| 48 | function skdiff_test { |
epoger@google.com | a413a53 | 2012-11-12 18:04:51 +0000 | [diff] [blame] | 49 | if [ $# != 2 ]; then |
| 50 | echo "skdiff_test requires exactly 2 parameters, got $#" |
| 51 | exit 1 |
| 52 | fi |
epoger@google.com | a444352 | 2012-05-17 17:12:38 +0000 | [diff] [blame] | 53 | SKDIFF_ARGS="$1" |
| 54 | ACTUAL_OUTPUT_DIR="$2/output-actual" |
| 55 | EXPECTED_OUTPUT_DIR="$2/output-expected" |
epoger@google.com | a2b793c | 2012-05-15 14:58:53 +0000 | [diff] [blame] | 56 | |
epoger@google.com | a444352 | 2012-05-17 17:12:38 +0000 | [diff] [blame] | 57 | rm -rf $ACTUAL_OUTPUT_DIR |
| 58 | mkdir -p $ACTUAL_OUTPUT_DIR |
epoger@google.com | 90cb9b3 | 2012-05-18 18:25:26 +0000 | [diff] [blame] | 59 | COMMAND="$SKDIFF_BINARY $SKDIFF_ARGS $ACTUAL_OUTPUT_DIR" |
| 60 | echo "$COMMAND" >$ACTUAL_OUTPUT_DIR/command_line |
| 61 | $COMMAND &>$ACTUAL_OUTPUT_DIR/stdout |
| 62 | echo $? >$ACTUAL_OUTPUT_DIR/return_value |
| 63 | |
epoger@google.com | a444352 | 2012-05-17 17:12:38 +0000 | [diff] [blame] | 64 | compare_directories $EXPECTED_OUTPUT_DIR $ACTUAL_OUTPUT_DIR |
| 65 | } |
| 66 | |
epoger@google.com | 673816c | 2013-02-15 15:50:23 +0000 | [diff] [blame] | 67 | # Download a subset of the raw bench data for platform $1 at revision $2. |
epoger@google.com | bd9669a | 2013-02-17 08:33:27 +0000 | [diff] [blame] | 68 | # (For the subset, download all files matching any of the suffixes in |
| 69 | # whitespace-separated list $3.) |
epoger@google.com | 673816c | 2013-02-15 15:50:23 +0000 | [diff] [blame] | 70 | # If any of those files already exist locally, we assume that they are |
| 71 | # correct and up to date, and we don't download them again. |
bensong@google.com | 4a28f59 | 2013-10-24 18:08:45 +0000 | [diff] [blame] | 72 | function download_bench_rawdata { |
epoger@google.com | bd9669a | 2013-02-17 08:33:27 +0000 | [diff] [blame] | 73 | if [ $# != 3 ]; then |
bensong@google.com | 4a28f59 | 2013-10-24 18:08:45 +0000 | [diff] [blame] | 74 | echo "download_bench_rawdata requires exactly 3 parameters, got $#" |
epoger@google.com | 673816c | 2013-02-15 15:50:23 +0000 | [diff] [blame] | 75 | exit 1 |
| 76 | fi |
| 77 | PLATFORM="$1" |
| 78 | REV="$2" |
epoger@google.com | bd9669a | 2013-02-17 08:33:27 +0000 | [diff] [blame] | 79 | FILE_SUFFIXES="$3" |
epoger@google.com | 673816c | 2013-02-15 15:50:23 +0000 | [diff] [blame] | 80 | |
bensong@google.com | 4a28f59 | 2013-10-24 18:08:45 +0000 | [diff] [blame] | 81 | PLATFORM_DIR="tools/tests/benchalerts/$PLATFORM" |
epoger@google.com | 673816c | 2013-02-15 15:50:23 +0000 | [diff] [blame] | 82 | RAW_BENCH_DATA_DIR="$PLATFORM_DIR/raw-bench-data" |
| 83 | mkdir -p $RAW_BENCH_DATA_DIR |
| 84 | |
epoger@google.com | bd9669a | 2013-02-17 08:33:27 +0000 | [diff] [blame] | 85 | for FILE_SUFFIX in $FILE_SUFFIXES; do |
bensong@google.com | 4a28f59 | 2013-10-24 18:08:45 +0000 | [diff] [blame] | 86 | FILE=bench_${REV}_${FILE_SUFFIX} |
epoger@google.com | 673816c | 2013-02-15 15:50:23 +0000 | [diff] [blame] | 87 | DESTFILE=$RAW_BENCH_DATA_DIR/$FILE |
| 88 | if [ ! -f $DESTFILE ]; |
| 89 | then |
bensong@google.com | 4a28f59 | 2013-10-24 18:08:45 +0000 | [diff] [blame] | 90 | URL=http://chromium-skia-gm.commondatastorage.googleapis.com/perfdata/${PLATFORM}/${FILE} |
epoger@google.com | aa6470a | 2013-02-15 16:42:21 +0000 | [diff] [blame] | 91 | echo Downloading $URL ... |
epoger@google.com | 673816c | 2013-02-15 15:50:23 +0000 | [diff] [blame] | 92 | curl $URL --output $DESTFILE |
| 93 | fi |
| 94 | done |
| 95 | } |
| 96 | |
bensong@google.com | 4a28f59 | 2013-10-24 18:08:45 +0000 | [diff] [blame] | 97 | # Run check_bench_regressions.py across the data from platform $1, |
epoger@google.com | f4eeeab | 2013-02-14 15:10:24 +0000 | [diff] [blame] | 98 | # writing its output to output-actual and comparing those results against |
| 99 | # output-expected. |
bensong@google.com | 4a28f59 | 2013-10-24 18:08:45 +0000 | [diff] [blame] | 100 | function benchalert_test { |
| 101 | if [ $# != 2 ]; then |
| 102 | echo "benchalert_test requires exactly 2 parameter, got $#" |
epoger@google.com | f4eeeab | 2013-02-14 15:10:24 +0000 | [diff] [blame] | 103 | exit 1 |
| 104 | fi |
| 105 | PLATFORM="$1" |
bensong@google.com | 4a28f59 | 2013-10-24 18:08:45 +0000 | [diff] [blame] | 106 | REVISION="$2" |
epoger@google.com | f4eeeab | 2013-02-14 15:10:24 +0000 | [diff] [blame] | 107 | |
bensong@google.com | 4a28f59 | 2013-10-24 18:08:45 +0000 | [diff] [blame] | 108 | PLATFORM_DIR="tools/tests/benchalerts/$PLATFORM" |
epoger@google.com | f4eeeab | 2013-02-14 15:10:24 +0000 | [diff] [blame] | 109 | RAW_BENCH_DATA_DIR="$PLATFORM_DIR/raw-bench-data" |
| 110 | ACTUAL_OUTPUT_DIR="$PLATFORM_DIR/output-actual" |
| 111 | EXPECTED_OUTPUT_DIR="$PLATFORM_DIR/output-expected" |
| 112 | |
bensong@google.com | 4a28f59 | 2013-10-24 18:08:45 +0000 | [diff] [blame] | 113 | # Run check_bench_regressions.py . |
epoger@google.com | f4eeeab | 2013-02-14 15:10:24 +0000 | [diff] [blame] | 114 | rm -rf $ACTUAL_OUTPUT_DIR |
| 115 | mkdir -p $ACTUAL_OUTPUT_DIR |
bensong@google.com | 4a28f59 | 2013-10-24 18:08:45 +0000 | [diff] [blame] | 116 | COMMAND="python bench/check_bench_regressions.py -a 25th -b $PLATFORM -d $RAW_BENCH_DATA_DIR -e $PLATFORM_DIR/expectations.txt -r $REVISION" |
epoger@google.com | f4eeeab | 2013-02-14 15:10:24 +0000 | [diff] [blame] | 117 | echo "$COMMAND" >$ACTUAL_OUTPUT_DIR/command_line |
| 118 | START_TIMESTAMP=$(date +%s) |
bensong@google.com | 4a28f59 | 2013-10-24 18:08:45 +0000 | [diff] [blame] | 119 | $COMMAND 2>$ACTUAL_OUTPUT_DIR/stderr |
epoger@google.com | f4eeeab | 2013-02-14 15:10:24 +0000 | [diff] [blame] | 120 | echo $? >$ACTUAL_OUTPUT_DIR/return_value |
| 121 | END_TIMESTAMP=$(date +%s) |
| 122 | |
| 123 | SECONDS_RUN=$(expr $END_TIMESTAMP - $START_TIMESTAMP) |
bensong@google.com | 4a28f59 | 2013-10-24 18:08:45 +0000 | [diff] [blame] | 124 | echo "check_bench_regressions.py took $SECONDS_RUN seconds to complete" |
epoger@google.com | f4eeeab | 2013-02-14 15:10:24 +0000 | [diff] [blame] | 125 | |
| 126 | compare_directories $EXPECTED_OUTPUT_DIR $ACTUAL_OUTPUT_DIR |
| 127 | } |
| 128 | |
epoger@google.com | 53953b4 | 2013-07-02 20:22:27 +0000 | [diff] [blame] | 129 | # Run jsondiff.py with arguments in $1, recording its output. |
| 130 | # Then compare that output to the content of $2/output-expected. |
| 131 | function jsondiff_test { |
| 132 | if [ $# != 2 ]; then |
| 133 | echo "jsondiff_test requires exactly 2 parameters, got $#" |
| 134 | exit 1 |
| 135 | fi |
| 136 | ARGS="$1" |
| 137 | ACTUAL_OUTPUT_DIR="$2/output-actual" |
| 138 | EXPECTED_OUTPUT_DIR="$2/output-expected" |
| 139 | |
| 140 | rm -rf $ACTUAL_OUTPUT_DIR |
| 141 | mkdir -p $ACTUAL_OUTPUT_DIR |
| 142 | COMMAND="python tools/jsondiff.py $ARGS" |
| 143 | echo "$COMMAND" >$ACTUAL_OUTPUT_DIR/command_line |
| 144 | $COMMAND &>$ACTUAL_OUTPUT_DIR/stdout |
| 145 | echo $? >$ACTUAL_OUTPUT_DIR/return_value |
| 146 | |
| 147 | compare_directories $EXPECTED_OUTPUT_DIR $ACTUAL_OUTPUT_DIR |
| 148 | } |
| 149 | |
epoger@google.com | db29a31 | 2013-06-04 14:58:47 +0000 | [diff] [blame] | 150 | |
| 151 | |
| 152 | # |
| 153 | # Run skdiff tests... |
| 154 | # |
| 155 | |
| 156 | SKDIFF_TESTDIR=tools/tests/skdiff |
| 157 | |
| 158 | # Run skdiff over a variety of file pair types: identical bits, identical pixels, missing from |
| 159 | # baseDir, etc. |
| 160 | skdiff_test "$SKDIFF_TESTDIR/baseDir $SKDIFF_TESTDIR/comparisonDir" "$SKDIFF_TESTDIR/test1" |
| 161 | |
| 162 | # Run skdiff over the same set of files, but with arguments as used by our buildbots: |
| 163 | # - return the number of mismatching file pairs (but ignore any files missing from either |
| 164 | # baseDir or comparisonDir) |
| 165 | # - list filenames with each result type to stdout |
| 166 | # - don't generate HTML output files |
| 167 | skdiff_test "--failonresult DifferentPixels --failonresult DifferentSizes --failonresult Unknown --failonstatus CouldNotDecode,CouldNotRead any --failonstatus any CouldNotDecode,CouldNotRead --listfilenames --nodiffs $SKDIFF_TESTDIR/baseDir $SKDIFF_TESTDIR/comparisonDir" "$SKDIFF_TESTDIR/test2" |
| 168 | |
| 169 | # Run skdiff over just the files that have identical bits. |
| 170 | skdiff_test "--nodiffs --match identical-bits $SKDIFF_TESTDIR/baseDir $SKDIFF_TESTDIR/comparisonDir" "$SKDIFF_TESTDIR/identical-bits" |
| 171 | |
| 172 | # Run skdiff over just the files that have identical bits or identical pixels. |
| 173 | skdiff_test "--nodiffs --match identical-bits --match identical-pixels $SKDIFF_TESTDIR/baseDir $SKDIFF_TESTDIR/comparisonDir" "$SKDIFF_TESTDIR/identical-bits-or-pixels" |
| 174 | |
| 175 | # |
bensong@google.com | 4a28f59 | 2013-10-24 18:08:45 +0000 | [diff] [blame] | 176 | # Run bench alerts tests... |
epoger@google.com | db29a31 | 2013-06-04 14:58:47 +0000 | [diff] [blame] | 177 | # |
| 178 | |
bensong@google.com | 4a28f59 | 2013-10-24 18:08:45 +0000 | [diff] [blame] | 179 | # Parse a collection of bench data |
| 180 | PLATFORM=Perf-Android-Nexus7-Tegra3-Arm7-Release |
| 181 | REVISION=69c9e1a7261a3c8361e2b2c109d6340862149e34 |
| 182 | download_bench_rawdata $PLATFORM $REVISION "$BENCHDATA_FILE_SUFFIXES_NO_INDIVIDUAL_TILES" |
| 183 | download_bench_rawdata $PLATFORM $REVISION "$BENCHDATA_FILE_SUFFIXES_YES_INDIVIDUAL_TILES" |
| 184 | benchalert_test $PLATFORM $REVISION |
epoger@google.com | f4eeeab | 2013-02-14 15:10:24 +0000 | [diff] [blame] | 185 | |
epoger@google.com | db29a31 | 2013-06-04 14:58:47 +0000 | [diff] [blame] | 186 | # |
epoger@google.com | 53953b4 | 2013-07-02 20:22:27 +0000 | [diff] [blame] | 187 | # Test jsondiff.py ... |
| 188 | # |
| 189 | |
| 190 | JSONDIFF_INPUT=tools/tests/jsondiff/input |
| 191 | JSONDIFF_OUTPUT=tools/tests/jsondiff/output |
| 192 | jsondiff_test "$JSONDIFF_INPUT/old.json $JSONDIFF_INPUT/new.json" "$JSONDIFF_OUTPUT/old-vs-new" |
| 193 | |
| 194 | |
commit-bot@chromium.org | 9e8f88b | 2013-12-19 18:22:32 +0000 | [diff] [blame] | 195 | # |
| 196 | # Launch all the self-tests which have been written in Python. |
| 197 | # |
| 198 | # TODO: Over time, we should move all of our tests into Python, and delete |
| 199 | # the bash tests above. |
| 200 | # See https://code.google.com/p/skia/issues/detail?id=677 |
| 201 | # ('make tools/tests/run.sh work cross-platform') |
| 202 | # |
| 203 | |
| 204 | COMMAND="python tools/tests/run_all.py" |
| 205 | echo "$COMMAND" |
| 206 | $COMMAND |
| 207 | ret=$? |
| 208 | if [ $ret -ne 0 ]; then |
| 209 | echo "failure in Python self-tests; see stack trace above" |
| 210 | exit 1 |
| 211 | fi |
| 212 | |
| 213 | |
epoger@google.com | a2b793c | 2012-05-15 14:58:53 +0000 | [diff] [blame] | 214 | echo "All tests passed." |