Rearranged and commented files for running under Jenkins.
diff --git a/.travis.yml b/.travis.yml
index 26b9dca..f8d2347 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -10,7 +10,7 @@
# The Objective C build needs Xcode 7.0 or later.
osx_image: xcode7.2
script:
- - ./tools/run_tests/tests.sh $CONFIG
+ - ./tests.sh $CONFIG
env:
- CONFIG=cpp
- CONFIG=cpp_distcheck
diff --git a/jenkins/README.md b/jenkins/README.md
new file mode 100644
index 0000000..29f664f
--- /dev/null
+++ b/jenkins/README.md
@@ -0,0 +1,6 @@
+
+Jenkins Infrastructure
+----------------------
+
+The scripts in this directory serve as plumbing for running the protobuf
+tests under Jenkins.
diff --git a/jenkins/build_and_run_docker.sh b/jenkins/build_and_run_docker.sh
new file mode 100755
index 0000000..abc6f05
--- /dev/null
+++ b/jenkins/build_and_run_docker.sh
@@ -0,0 +1,56 @@
+#!/bin/bash
+#
+# Builds docker image and runs a command under it.
+# This is a generic script that is configured with the following variables:
+#
+# DOCKERFILE_DIR - Directory in which Dockerfile file is located.
+# DOCKER_RUN_SCRIPT - Script to run under docker (relative to protobuf repo root)
+# OUTPUT_DIR - Directory that will be copied from inside docker after finishing.
+# $@ - Extra args to pass to docker run
+
+
+set -ex
+
+cd $(dirname $0)/..
+git_root=$(pwd)
+cd -
+
+# Use image name based on Dockerfile location checksum
+DOCKER_IMAGE_NAME=$(basename $DOCKERFILE_DIR)_$(sha1sum $DOCKERFILE_DIR/Dockerfile | cut -f1 -d\ )
+
+# Make sure docker image has been built. Should be instantaneous if so.
+docker build -t $DOCKER_IMAGE_NAME $DOCKERFILE_DIR
+
+# Ensure existence of ccache directory
+CCACHE_DIR=/tmp/protobuf-ccache
+mkdir -p $CCACHE_DIR
+
+# Choose random name for docker container
+CONTAINER_NAME="build_and_run_docker_$(uuidgen)"
+
+# Run command inside docker
+docker run \
+ "$@" \
+ -e CCACHE_DIR=$CCACHE_DIR \
+ -e EXTERNAL_GIT_ROOT="/var/local/jenkins/protobuf" \
+ -e THIS_IS_REALLY_NEEDED='see https://github.com/docker/docker/issues/14203 for why docker is awful' \
+ -v "$git_root:/var/local/jenkins/protobuf:ro" \
+ -v $CCACHE_DIR:$CCACHE_DIR \
+ -w /var/local/git/protobuf \
+ --name=$CONTAINER_NAME \
+ $DOCKER_IMAGE_NAME \
+ bash -l "/var/local/jenkins/protobuf/$DOCKER_RUN_SCRIPT" || FAILED="true"
+
+# Copy output artifacts
+if [ "$OUTPUT_DIR" != "" ]
+then
+ docker cp "$CONTAINER_NAME:/var/local/git/protobuf/$OUTPUT_DIR" "$git_root" || FAILED="true"
+fi
+
+# remove the container, possibly killing it first
+docker rm -f $CONTAINER_NAME || true
+
+if [ "$FAILED" != "" ]
+then
+ exit 1
+fi
diff --git a/jenkins/buildcmds/README.md b/jenkins/buildcmds/README.md
new file mode 100644
index 0000000..7a48f2d
--- /dev/null
+++ b/jenkins/buildcmds/README.md
@@ -0,0 +1,6 @@
+
+Jenkins Build Commands
+----------------------
+
+The scripts in this directory are designed to be top-level entry points for
+Jenkins projects.
diff --git a/jenkins/buildcmds/pull_request.sh b/jenkins/buildcmds/pull_request.sh
new file mode 100755
index 0000000..01fda79
--- /dev/null
+++ b/jenkins/buildcmds/pull_request.sh
@@ -0,0 +1,15 @@
+#!/bin/bash
+#
+# This is the top-level script we give to Jenkins as the entry point for
+# running the "pull request" project:
+#
+# https://grpc-testing.appspot.com/view/Protocol%20Buffers/job/protobuf_pull_request/
+#
+# This script selects a specific Dockerfile (for building a Docker image) and
+# a script to run inside that image. Then we delegate to the general
+# build_and_run_docker.sh script.
+
+export DOCKERFILE_DIR=jenkins/docker
+export DOCKER_RUN_SCRIPT=jenkins/pull_request_in_docker.sh
+export OUTPUT_DIR=testoutput
+./jenkins/build_and_run_docker.sh
diff --git a/tools/docker/Dockerfile b/jenkins/docker/Dockerfile
similarity index 70%
rename from tools/docker/Dockerfile
rename to jenkins/docker/Dockerfile
index 5136ee6..8467aef 100644
--- a/tools/docker/Dockerfile
+++ b/jenkins/docker/Dockerfile
@@ -1,31 +1,10 @@
-# Copyright 2015, Google Inc.
-# All rights reserved.
+# This Dockerfile specifies the recipe for creating an image for the tests
+# to run in.
#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are
-# met:
-#
-# * Redistributions of source code must retain the above copyright
-# notice, this list of conditions and the following disclaimer.
-# * Redistributions in binary form must reproduce the above
-# copyright notice, this list of conditions and the following disclaimer
-# in the documentation and/or other materials provided with the
-# distribution.
-# * Neither the name of Google Inc. nor the names of its
-# contributors may be used to endorse or promote products derived from
-# this software without specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# We install as many test dependencies here as we can, because these setup
+# steps can be cached. They do *not* run every time we run the build.
+# The Docker image is only rebuilt when the Dockerfile (ie. this file)
+# changes.
# Base Dockerfile for gRPC dev images
FROM debian:latest
diff --git a/tools/jenkins/make_test_output.py b/jenkins/make_test_output.py
similarity index 91%
rename from tools/jenkins/make_test_output.py
rename to jenkins/make_test_output.py
index 986d197..b1f2e2c 100644
--- a/tools/jenkins/make_test_output.py
+++ b/jenkins/make_test_output.py
@@ -1,9 +1,9 @@
-"""Gather output from test runs and create an XML file in JUnit format.
+"""Gathers output from test runs and create an XML file in JUnit format.
The output files from the individual tests have been written in a directory
structure like:
- $DIR/joblog (--joblog from "parallel")
+ $DIR/joblog (output from "parallel --joblog joblog")
$DIR/logs/1/cpp/stdout
$DIR/logs/1/cpp/stderr
$DIR/logs/1/csharp/stdout
@@ -13,7 +13,8 @@
etc.
This script bundles them into a single output XML file so Jenkins can show
-detailed test results.
+detailed test results. It runs as the last step before the Jenkins build
+finishes.
"""
import os;
diff --git a/tools/run_tests/jenkins.sh b/jenkins/pull_request_in_docker.sh
similarity index 69%
rename from tools/run_tests/jenkins.sh
rename to jenkins/pull_request_in_docker.sh
index ea67345..887f97c 100755
--- a/tools/run_tests/jenkins.sh
+++ b/jenkins/pull_request_in_docker.sh
@@ -1,22 +1,25 @@
#!/bin/bash
+#
+# This is the script that runs inside Docker, once the image has been built,
+# to execute all tests for the "pull request" project.
WORKSPACE_BASE=`pwd`
MY_DIR="$(dirname "$0")"
-TEST_SCRIPT=$MY_DIR/tests.sh
+TEST_SCRIPT=$MY_DIR/../tests.sh
BUILD_DIR=/tmp/protobuf
-# Set value used in tests.sh.
-PARALLELISM=-j8
-
set -e # exit immediately on error
set -x # display all commands
+# The protobuf repository is mounted into our Docker image, but read-only.
+# We clone into a directory inside Docker (this is faster than cp).
rm -rf $BUILD_DIR
mkdir -p $BUILD_DIR
cd $BUILD_DIR
git clone /var/local/jenkins/protobuf
cd protobuf
+# Set up the directory where our test output is going to go.
OUTPUT_DIR=`mktemp -d`
LOG_OUTPUT_DIR=$OUTPUT_DIR/logs
mkdir -p $LOG_OUTPUT_DIR/1/cpp
@@ -35,14 +38,14 @@
CPP_STDOUT=$LOG_OUTPUT_DIR/1/cpp/stdout
CPP_STDERR=$LOG_OUTPUT_DIR/1/cpp/stderr
+# Time the C++ build, so we can put this info in the test output.
# It's important that we get /usr/bin/time (which supports -f and -o) and not
# the bash builtin "time" which doesn't.
TIME_CMD="/usr/bin/time -f %e -o $LOG_OUTPUT_DIR/1/cpp/build_time"
$TIME_CMD $TEST_SCRIPT cpp > >(tee $CPP_STDOUT) 2> >(tee $CPP_STDERR >&2)
-# Other tests are run in parallel. The overall run fails if any one of them
-# fails.
+# Other tests are run in parallel.
parallel --results $LOG_OUTPUT_DIR --joblog $OUTPUT_DIR/joblog $TEST_SCRIPT ::: \
csharp \
@@ -55,14 +58,15 @@
ruby21 \
|| true # Process test results even if tests fail.
+cat $OUTPUT_DIR/joblog
+
# The directory that is copied from Docker back into the Jenkins workspace.
COPY_FROM_DOCKER=/var/local/git/protobuf/testoutput
mkdir -p $COPY_FROM_DOCKER
TESTOUTPUT_XML_FILE=$COPY_FROM_DOCKER/testresults.xml
-python $MY_DIR/../jenkins/make_test_output.py $OUTPUT_DIR > $TESTOUTPUT_XML_FILE
+# Process all the output files from "parallel" and package them into a single
+# .xml file with detailed, broken-down test output.
+python $MY_DIR/make_test_output.py $OUTPUT_DIR > $TESTOUTPUT_XML_FILE
ls -l $TESTOUTPUT_XML_FILE
-
-### disabled tests
-# java_jdk6 \
diff --git a/tools/run_tests/tests.sh b/tests.sh
similarity index 96%
rename from tools/run_tests/tests.sh
rename to tests.sh
index c28a5da..a4624c0 100755
--- a/tools/run_tests/tests.sh
+++ b/tests.sh
@@ -1,4 +1,8 @@
#!/bin/bash
+#
+# Build and runs tests for the protobuf project. The tests as written here are
+# used by both Jenkins and Travis, though some specialized logic is required to
+# handle the differences between them.
on_travis() {
if [ "$TRAVIS" == "true" ]; then
@@ -25,19 +29,19 @@
./autogen.sh
./configure
- make $PARALLELISM
+ make -j2
}
build_cpp() {
internal_build_cpp
- make check $PARALLELISM
+ make check -j2
cd conformance && make test_cpp && cd ..
}
build_cpp_distcheck() {
./autogen.sh
./configure
- make distcheck $PARALLELISM
+ make distcheck -j2
}
build_csharp() {
@@ -308,8 +312,6 @@
cd js && npm install && npm test && cd ..
}
-[ -n "${PARALLELISM}" ] && PARALLELISM=-j8
-
# Note: travis currently does not support testing more than one language so the
# .travis.yml cheats and claims to only be cpp. If they add multiple language
# support, this should probably get updated to install steps and/or
@@ -320,9 +322,6 @@
# -------- main --------
-# Set value used in tests.sh.
-PARALLELISM=-j2
-
if [ "$#" -ne 1 ]; then
echo "
Usage: $0 { cpp |
diff --git a/tools/jenkins/build_and_run_docker.sh b/tools/jenkins/build_and_run_docker.sh
deleted file mode 100755
index ad1075f..0000000
--- a/tools/jenkins/build_and_run_docker.sh
+++ /dev/null
@@ -1,84 +0,0 @@
-#!/bin/bash
-# Copyright 2016, Google Inc.
-# All rights reserved.
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are
-# met:
-#
-# * Redistributions of source code must retain the above copyright
-# notice, this list of conditions and the following disclaimer.
-# * Redistributions in binary form must reproduce the above
-# copyright notice, this list of conditions and the following disclaimer
-# in the documentation and/or other materials provided with the
-# distribution.
-# * Neither the name of Google Inc. nor the names of its
-# contributors may be used to endorse or promote products derived from
-# this software without specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-#
-# Builds docker image and runs a command under it.
-# You should never need to call this script on your own.
-
-set -ex
-
-cd $(dirname $0)/../..
-git_root=$(pwd)
-cd -
-
-# Inputs
-# DOCKERFILE_DIR - Directory in which Dockerfile file is located.
-# DOCKER_RUN_SCRIPT - Script to run under docker (relative to protobuf repo root)
-# OUTPUT_DIR - Directory that will be copied from inside docker after finishing.
-# $@ - Extra args to pass to docker run
-
-# Use image name based on Dockerfile location checksum
-DOCKER_IMAGE_NAME=$(basename $DOCKERFILE_DIR)_$(sha1sum $DOCKERFILE_DIR/Dockerfile | cut -f1 -d\ )
-
-# Make sure docker image has been built. Should be instantaneous if so.
-docker build -t $DOCKER_IMAGE_NAME $DOCKERFILE_DIR
-
-# Ensure existence of ccache directory
-CCACHE_DIR=/tmp/protobuf-ccache
-mkdir -p $CCACHE_DIR
-
-# Choose random name for docker container
-CONTAINER_NAME="build_and_run_docker_$(uuidgen)"
-
-# Run command inside docker
-docker run \
- "$@" \
- -e CCACHE_DIR=$CCACHE_DIR \
- -e EXTERNAL_GIT_ROOT="/var/local/jenkins/protobuf" \
- -e THIS_IS_REALLY_NEEDED='see https://github.com/docker/docker/issues/14203 for why docker is awful' \
- -v "$git_root:/var/local/jenkins/protobuf:ro" \
- -v $CCACHE_DIR:$CCACHE_DIR \
- -w /var/local/git/protobuf \
- --name=$CONTAINER_NAME \
- $DOCKER_IMAGE_NAME \
- bash -l "/var/local/jenkins/protobuf/$DOCKER_RUN_SCRIPT" || FAILED="true"
-
-# Copy output artifacts
-if [ "$OUTPUT_DIR" != "" ]
-then
- docker cp "$CONTAINER_NAME:/var/local/git/protobuf/$OUTPUT_DIR" "$git_root" || FAILED="true"
-fi
-
-# remove the container, possibly killing it first
-docker rm -f $CONTAINER_NAME || true
-
-if [ "$FAILED" != "" ]
-then
- exit 1
-fi
diff --git a/tools/jenkins/pull_request.sh b/tools/jenkins/pull_request.sh
deleted file mode 100755
index 00538b9..0000000
--- a/tools/jenkins/pull_request.sh
+++ /dev/null
@@ -1,7 +0,0 @@
-#!/bin/bash
-
-
-export DOCKERFILE_DIR=tools/docker
-export DOCKER_RUN_SCRIPT=tools/run_tests/jenkins.sh
-export OUTPUT_DIR=testoutput
-./tools/jenkins/build_and_run_docker.sh