travis: handle reduced max run time by splitting autobahn tests across two instances
diff --git a/.travis.yml b/.travis.yml
index 6181435..cf439b4 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -5,6 +5,7 @@
- secure: "KhAdQ9ja+LBObWNQTYO7Df5J4DyOih6S+eerDMu8UPSO+CoWV2pWoQzbOfocjyOscGOwC+2PrrHDNZyGfqkCLDXg1BxynXPCFerHC1yc2IajvKpGXmAAygNIvp4KACDfGv/dkXrViqIzr/CdcNaU4vIMHSVb5xkeLi0W1dPnQOI="
matrix:
- LWS_METHOD=lwsws CMAKE_ARGS="-DLWS_WITH_LWSWS=ON -DLWS_WITHOUT_EXTENSIONS=0 -DLWS_WITH_HTTP2=1 -DLWS_WITH_ACME=1 -DLWS_WITH_MINIMAL_EXAMPLES=1 -DCMAKE_BUILD_TYPE=DEBUG"
+ - LWS_METHOD=lwsws2 CMAKE_ARGS="-DLWS_WITH_LWSWS=ON -DLWS_WITHOUT_EXTENSIONS=0 -DLWS_WITH_HTTP2=1 -DLWS_WITH_ACME=1 -DLWS_WITH_MINIMAL_EXAMPLES=1 -DCMAKE_BUILD_TYPE=DEBUG"
- LWS_METHOD=default CMAKE_ARGS="-DLWS_WITH_MINIMAL_EXAMPLES=1"
- LWS_METHOD=mbedtls CMAKE_ARGS="-DLWS_WITH_MBEDTLS=1 -DLWS_WITH_HTTP2=1 -DLWS_WITH_LWSWS=1 -DLWS_WITH_MINIMAL_EXAMPLES=1 -DCMAKE_BUILD_TYPE=DEBUG"
- LWS_METHOD=noserver CMAKE_ARGS="-DLWS_WITHOUT_SERVER=ON -DLWS_WITH_MINIMAL_EXAMPLES=1"
diff --git a/scripts/autobahn-test.sh b/scripts/autobahn-test-client.sh
similarity index 85%
rename from scripts/autobahn-test.sh
rename to scripts/autobahn-test-client.sh
index f813684..e6db7dc 100755
--- a/scripts/autobahn-test.sh
+++ b/scripts/autobahn-test-client.sh
@@ -142,49 +142,6 @@
echo
fi
-# 2) lws-as-server tests
-
-echo
-echo "----------------------------------------------"
-echo "------- tests: autobahn as server"
-echo
-
-$SERV -p 9001 -d7 &
-wstest -m fuzzingclient
-R=$?
-echo "Autobahn client exit $R"
-
-killall lws-minimal-ws-server-echo
-sleep 1s
-
-# repeat the client results
-
-R=`cat /tmp/ji | grep -v '"behavior": "OK"' | grep -v '"behavior": "NON-STRICT"' | grep -v '"behavior": "INFORMATIONAL"' | wc -l`
-echo -n "AUTOBAHN SERVER / LWS CLIENT: Total tests: " `cat /tmp/ji | wc -l` " : "
-if [ "$R" == "0" ] ;then
- echo "All pass"
-else
- RESULT=1
- echo -n "$R FAIL : "
- cat /tmp/ji | grep -v '"behavior": "OK"' | grep -v '"behavior": "NON-STRICT"' | grep -v '"behavior": "INFORMATIONAL"' | cut -d\" -f2 | tr '\n' ','
- echo
-fi
-
-# and then the server results
-
-cat reports/servers/index.json | tr '\n' '!' | sed "s|\},\!|\n|g" | tr '!' ' ' | tr -s ' ' > /tmp/jis
-R=`cat /tmp/jis | grep -v '"behavior": "OK"' | grep -v '"behavior": "NON-STRICT"' | grep -v '"behavior": "INFORMATIONAL"' | wc -l`
-
-echo -n "AUTOBAHN CLIENT / LWS SERVER: Total tests: " `cat /tmp/jis | wc -l` " : "
-if [ "$R" == "0" ] ;then
- echo "All pass"
-else
- RESULT=$(( $RESULT + 2 ))
- echo -n "$R FAIL : "
- cat /tmp/jis | grep -v '"behavior": "OK"' | grep -v '"behavior": "NON-STRICT"' | grep -v '"behavior": "INFORMATIONAL"' | cut -d\" -f2 | tr '\n' ','
- echo
-fi
-
echo $RESULT
exit $RESULT
diff --git a/scripts/autobahn-test-server.sh b/scripts/autobahn-test-server.sh
new file mode 100755
index 0000000..d2afd68
--- /dev/null
+++ b/scripts/autobahn-test-server.sh
@@ -0,0 +1,107 @@
+#!/bin/bash
+#
+# Requires pip install autobahntestsuite
+#
+# you should run this from ./build, after building with
+# cmake .. -DLWS_WITH_MINIMAL_EXAMPLES=1
+#
+# It will use the minimal echo client and server to run
+# autobahn ws tests as both client and server.
+
+set -u
+
+PARALLEL=8
+N=1
+OS=`uname`
+
+CLIE=bin/lws-minimal-ws-client-echo
+SERV=bin/lws-minimal-ws-server-echo
+
+RESULT=0
+
+which wstest 2>/dev/null
+if [ $? -ne 0 ]; then
+ echo "wstest is not installed"
+ exit 8
+fi
+
+killall wstest 2>/dev/null
+
+#
+# 2.10 / 2.11: There is no requirement to handle multiple PING / PONG
+# in flight in RFC6455. lws doesn't waste memory on it
+# since it is useless.
+#
+# 12.3.1 / 12.3.2
+# 12.4.* / 12.5.*: Autobahn has been broken for these tests since Aug 2017
+# https://github.com/crossbario/autobahn-testsuite/issues/71
+
+
+cat << EOF >fuzzingserver.json
+{
+ "url": "ws://127.0.0.1:9001",
+ "outdir": "./reports/clients",
+ "cases": ["*"],
+ "exclude-cases": [ "2.10", "2.11", "12.3.1", "12.3.2", "12.4.*", "12.5.*"],
+ "exclude-agent-cases": {}
+}
+EOF
+
+cat << EOF >fuzzingclient.json
+{
+ "outdir": "./reports/servers",
+ "servers": [
+ {
+ "url": "ws://127.0.0.1:9001"
+ }
+ ],
+ "cases": ["*"],
+ "exclude-cases": ["2.10", "2.11", "12.3.1", "12.3.2", "12.4.*", "12.5.*" ],
+ "exclude-agent-cases": {}
+}
+EOF
+
+echo
+echo "----------------------------------------------"
+echo "------- tests: autobahn as server"
+echo
+
+$SERV -p 9001 -d7 &
+wstest -m fuzzingclient
+R=$?
+echo "Autobahn client exit $R"
+
+killall lws-minimal-ws-server-echo
+sleep 1s
+
+# repeat the client results
+
+R=`cat /tmp/ji | grep -v '"behavior": "OK"' | grep -v '"behavior": "NON-STRICT"' | grep -v '"behavior": "INFORMATIONAL"' | wc -l`
+echo -n "AUTOBAHN SERVER / LWS CLIENT: Total tests: " `cat /tmp/ji | wc -l` " : "
+if [ "$R" == "0" ] ;then
+ echo "All pass"
+else
+ RESULT=1
+ echo -n "$R FAIL : "
+ cat /tmp/ji | grep -v '"behavior": "OK"' | grep -v '"behavior": "NON-STRICT"' | grep -v '"behavior": "INFORMATIONAL"' | cut -d\" -f2 | tr '\n' ','
+ echo
+fi
+
+# and then the server results
+
+cat reports/servers/index.json | tr '\n' '!' | sed "s|\},\!|\n|g" | tr '!' ' ' | tr -s ' ' > /tmp/jis
+R=`cat /tmp/jis | grep -v '"behavior": "OK"' | grep -v '"behavior": "NON-STRICT"' | grep -v '"behavior": "INFORMATIONAL"' | wc -l`
+
+echo -n "AUTOBAHN CLIENT / LWS SERVER: Total tests: " `cat /tmp/jis | wc -l` " : "
+if [ "$R" == "0" ] ;then
+ echo "All pass"
+else
+ RESULT=$(( $RESULT + 2 ))
+ echo -n "$R FAIL : "
+ cat /tmp/jis | grep -v '"behavior": "OK"' | grep -v '"behavior": "NON-STRICT"' | grep -v '"behavior": "INFORMATIONAL"' | cut -d\" -f2 | tr '\n' ','
+ echo
+fi
+
+echo $RESULT
+exit $RESULT
+
diff --git a/scripts/travis_control.sh b/scripts/travis_control.sh
index 6603953..92e086b 100755
--- a/scripts/travis_control.sh
+++ b/scripts/travis_control.sh
@@ -18,25 +18,33 @@
../scripts/h2spec.sh &&
../scripts/attack.sh &&
../scripts/h2load.sh &&
- ../scripts/autobahn-test.sh
+ ../scripts/autobahn-test-client.sh
else
- if [ "$LWS_METHOD" = "smp" ] ; then
+ if [ "$LWS_METHOD" = "lwsws2" ] ; then
cmake -DLWS_OPENSSL_LIBRARIES="/usr/local/lib/libssl.so;/usr/local/lib/libcrypto.so" \
-DLWS_OPENSSL_INCLUDE_DIRS="/usr/local/include/openssl" $CMAKE_ARGS .. &&
cmake --build . &&
- ../scripts/h2load-smp.sh
+ sudo make install &&
+ ../scripts/autobahn-test-server.sh
else
- if [ "$LWS_METHOD" = "mbedtls" ] ; then
- cmake $CMAKE_ARGS .. &&
+ if [ "$LWS_METHOD" = "smp" ] ; then
+ cmake -DLWS_OPENSSL_LIBRARIES="/usr/local/lib/libssl.so;/usr/local/lib/libcrypto.so" \
+ -DLWS_OPENSSL_INCLUDE_DIRS="/usr/local/include/openssl" $CMAKE_ARGS .. &&
cmake --build . &&
- sudo make install &&
- ../minimal-examples/selftests.sh &&
- ../scripts/h2spec.sh &&
- ../scripts/h2load.sh &&
- ../scripts/attack.sh
+ ../scripts/h2load-smp.sh
else
- cmake $CMAKE_ARGS .. &&
- cmake --build .
+ if [ "$LWS_METHOD" = "mbedtls" ] ; then
+ cmake $CMAKE_ARGS .. &&
+ cmake --build . &&
+ sudo make install &&
+ ../minimal-examples/selftests.sh &&
+ ../scripts/h2spec.sh &&
+ ../scripts/h2load.sh &&
+ ../scripts/attack.sh
+ else
+ cmake $CMAKE_ARGS .. &&
+ cmake --build .
+ fi
fi
fi
fi
diff --git a/scripts/travis_install.sh b/scripts/travis_install.sh
index c52bb44..4cdeccb 100755
--- a/scripts/travis_install.sh
+++ b/scripts/travis_install.sh
@@ -6,7 +6,7 @@
then
sudo apt-get update -qq
- if [ "$LWS_METHOD" == "lwsws" ];
+ if [ "$LWS_METHOD" == "lwsws" -o "$LWS_METHOD" == "lwsws2" ];
then
sudo apt-get install -y -qq realpath libjemalloc1 libev4 libuv-dev
sudo apt-get remove python-six
@@ -46,7 +46,7 @@
sudo apt-get install -y -qq libev-dev;
fi
- if [ "$LWS_METHOD" == "libuv" -o "$LWS_METHOD" == "lwsws" ];
+ if [ "$LWS_METHOD" == "libuv" -o "$LWS_METHOD" == "lwsws" -o "$LWS_METHOD" == "lwsws2" ];
then
sudo apt-get install -y -qq libuv-dev;
#libuv1 libuv1-dev;
@@ -62,7 +62,7 @@
brew install libev;
fi
- if [ "$LWS_METHOD" == "libuv" -o "$LWS_METHOD" == "lwsws" ];
+ if [ "$LWS_METHOD" == "libuv" -o "$LWS_METHOD" == "lwsws" -o "$LWS_METHOD" == "lwsws2" ];
then
brew update;
brew install libuv;