Merge pull request #9178 from jtattermusch/docker_inception_is_evil
Cleanup and speedup sanity tests
diff --git a/templates/tools/dockerfile/clang_format.include b/templates/tools/dockerfile/clang_format.include
new file mode 100644
index 0000000..9a2b60b
--- /dev/null
+++ b/templates/tools/dockerfile/clang_format.include
@@ -0,0 +1,5 @@
+RUN apt-get update && apt-get -y install wget
+RUN echo deb http://llvm.org/apt/wily/ llvm-toolchain-wily-3.8 main >> /etc/apt/sources.list
+RUN echo deb-src http://llvm.org/apt/wily/ llvm-toolchain-wily-3.8 main >> /etc/apt/sources.list
+RUN wget -O - http://llvm.org/apt/llvm-snapshot.gpg.key| apt-key add -
+RUN apt-get update && apt-get -y install clang-format-3.8
diff --git a/templates/tools/dockerfile/grpc_clang_format/Dockerfile.template b/templates/tools/dockerfile/grpc_clang_format/Dockerfile.template
new file mode 100644
index 0000000..8360fc1
--- /dev/null
+++ b/templates/tools/dockerfile/grpc_clang_format/Dockerfile.template
@@ -0,0 +1,37 @@
+%YAML 1.2
+--- |
+ # Copyright 2015, Google Inc.
+ # All rights reserved.
+ #
+ # Redistribution and use in source and binary forms, with or without
+ # modification, are permitted provided that the following conditions are
+ # met:
+ #
+ # * Redistributions of source code must retain the above copyright
+ # notice, this list of conditions and the following disclaimer.
+ # * Redistributions in binary form must reproduce the above
+ # copyright notice, this list of conditions and the following disclaimer
+ # in the documentation and/or other materials provided with the
+ # distribution.
+ # * Neither the name of Google Inc. nor the names of its
+ # contributors may be used to endorse or promote products derived from
+ # this software without specific prior written permission.
+ #
+ # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+ FROM ubuntu:15.10
+
+ <%include file="../clang_format.include"/>
+ ADD clang_format_all_the_things.sh /
+ CMD ["echo 'Run with tools/distrib/clang_format_code.sh'"]
+
diff --git a/templates/tools/dockerfile/test/sanity/Dockerfile.template b/templates/tools/dockerfile/test/sanity/Dockerfile.template
index 0168353..8617666 100644
--- a/templates/tools/dockerfile/test/sanity/Dockerfile.template
+++ b/templates/tools/dockerfile/test/sanity/Dockerfile.template
@@ -52,18 +52,12 @@
# ./compile.sh without a local protoc dependency
# TODO(mattkwong): install dependencies to support latest Bazel version if newer
# version is needed
- RUN git clone https://github.com/bazelbuild/bazel.git /bazel && \
+ RUN git clone https://github.com/bazelbuild/bazel.git /bazel && ${"\\"}
cd /bazel && git checkout tags/0.4.1 && ./compile.sh
RUN ln -s /bazel/output/bazel /bin/
- #===================
- # Docker "inception"
- # Note this is quite the ugly hack.
- # This makes sure that the docker binary we inject has its dependencies.
- RUN curl https://get.docker.com/ | sh
- RUN apt-get remove --purge -y docker-engine
-
- RUN mkdir /var/local/jenkins
+ <%include file="../../clang_format.include"/>
+ <%include file="../../run_tests_addons.include"/>
# Define the default command.
CMD ["bash"]
diff --git a/tools/distrib/check_nanopb_output.sh b/tools/distrib/check_nanopb_output.sh
index c070705..eb64e23 100755
--- a/tools/distrib/check_nanopb_output.sh
+++ b/tools/distrib/check_nanopb_output.sh
@@ -37,7 +37,7 @@
pushd third_party/protobuf
./autogen.sh
./configure --prefix="$PROTOBUF_INSTALL_PREFIX"
-make
+make -j 8
make install
#ldconfig
popd
@@ -51,7 +51,7 @@
# stack up and change to nanopb's proto generator directory
pushd third_party/nanopb/generator/proto
export PATH="$PROTOC_BIN_PATH:$PATH"
-make
+make -j 8
# back to the root directory
popd
diff --git a/tools/distrib/clang_format_code.sh b/tools/distrib/clang_format_code.sh
index 858e074..13e0187 100755
--- a/tools/distrib/clang_format_code.sh
+++ b/tools/distrib/clang_format_code.sh
@@ -32,9 +32,15 @@
# change to root directory
cd $(dirname $0)/../..
+REPO_ROOT=$(pwd)
-# build clang-format docker image
-docker build -t grpc_clang_format tools/dockerfile/grpc_clang_format
+if [ "$CLANG_FORMAT_SKIP_DOCKER" == "" ]
+then
+ # build clang-format docker image
+ docker build -t grpc_clang_format tools/dockerfile/grpc_clang_format
-# run clang-format against the checked out codebase
-docker run -e TEST=$TEST -e CHANGED_FILES="$CHANGED_FILES" --rm=true -v ${HOST_GIT_ROOT:-`pwd`}:/local-code -t grpc_clang_format /clang_format_all_the_things.sh
+ # run clang-format against the checked out codebase
+ docker run -e TEST=$TEST -e CHANGED_FILES="$CHANGED_FILES" -e CLANG_FORMAT_ROOT="/local-code" --rm=true -v "${REPO_ROOT}":/local-code -t grpc_clang_format /clang_format_all_the_things.sh
+else
+ CLANG_FORMAT_ROOT="${REPO_ROOT}" tools/dockerfile/grpc_clang_format/clang_format_all_the_things.sh
+fi
diff --git a/tools/dockerfile/grpc_clang_format/Dockerfile b/tools/dockerfile/grpc_clang_format/Dockerfile
index ab58017..85f5e4d 100644
--- a/tools/dockerfile/grpc_clang_format/Dockerfile
+++ b/tools/dockerfile/grpc_clang_format/Dockerfile
@@ -27,13 +27,13 @@
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-FROM ubuntu:wily
-RUN apt-get update
-RUN apt-get -y install wget
+FROM ubuntu:15.10
+
+RUN apt-get update && apt-get -y install wget
RUN echo deb http://llvm.org/apt/wily/ llvm-toolchain-wily-3.8 main >> /etc/apt/sources.list
RUN echo deb-src http://llvm.org/apt/wily/ llvm-toolchain-wily-3.8 main >> /etc/apt/sources.list
RUN wget -O - http://llvm.org/apt/llvm-snapshot.gpg.key| apt-key add -
-RUN apt-get update
-RUN apt-get -y install clang-format-3.8
+RUN apt-get update && apt-get -y install clang-format-3.8
+
ADD clang_format_all_the_things.sh /
CMD ["echo 'Run with tools/distrib/clang_format_code.sh'"]
diff --git a/tools/dockerfile/grpc_clang_format/clang_format_all_the_things.sh b/tools/dockerfile/grpc_clang_format/clang_format_all_the_things.sh
index 462c65a..c6e4aab 100755
--- a/tools/dockerfile/grpc_clang_format/clang_format_all_the_things.sh
+++ b/tools/dockerfile/grpc_clang_format/clang_format_all_the_things.sh
@@ -44,7 +44,7 @@
do
for glob in $GLOB
do
- files="$files `find /local-code/$dir -name $glob -and -not -name *.generated.* -and -not -name *.pb.h -and -not -name *.pb.c -and -not -name *.pb.cc`"
+ files="$files `find ${CLANG_FORMAT_ROOT}/$dir -name $glob -and -not -name *.generated.* -and -not -name *.pb.h -and -not -name *.pb.c -and -not -name *.pb.cc`"
done
done
@@ -54,7 +54,7 @@
files=$(comm -12 <(echo $files | tr ' ' '\n' | sort -u) <(echo $CHANGED_FILES | tr ' ' '\n' | sort -u))
fi
-if [ "x$TEST" = "x" ]
+if [ "$TEST" == "" ]
then
echo $files | xargs $CLANG_FORMAT -i
else
diff --git a/tools/dockerfile/test/sanity/Dockerfile b/tools/dockerfile/test/sanity/Dockerfile
index 6b19ac8..811384f 100644
--- a/tools/dockerfile/test/sanity/Dockerfile
+++ b/tools/dockerfile/test/sanity/Dockerfile
@@ -97,17 +97,27 @@
# ./compile.sh without a local protoc dependency
# TODO(mattkwong): install dependencies to support latest Bazel version if newer
# version is needed
-RUN git clone https://github.com/bazelbuild/bazel.git /bazel && cd /bazel && git checkout tags/0.4.1 && ./compile.sh
+RUN git clone https://github.com/bazelbuild/bazel.git /bazel && \
+ cd /bazel && git checkout tags/0.4.1 && ./compile.sh
RUN ln -s /bazel/output/bazel /bin/
-#===================
-# Docker "inception"
-# Note this is quite the ugly hack.
-# This makes sure that the docker binary we inject has its dependencies.
-RUN curl https://get.docker.com/ | sh
-RUN apt-get remove --purge -y docker-engine
+RUN apt-get update && apt-get -y install wget
+RUN echo deb http://llvm.org/apt/wily/ llvm-toolchain-wily-3.8 main >> /etc/apt/sources.list
+RUN echo deb-src http://llvm.org/apt/wily/ llvm-toolchain-wily-3.8 main >> /etc/apt/sources.list
+RUN wget -O - http://llvm.org/apt/llvm-snapshot.gpg.key| apt-key add -
+RUN apt-get update && apt-get -y install clang-format-3.8
+
+# Prepare ccache
+RUN ln -s /usr/bin/ccache /usr/local/bin/gcc
+RUN ln -s /usr/bin/ccache /usr/local/bin/g++
+RUN ln -s /usr/bin/ccache /usr/local/bin/cc
+RUN ln -s /usr/bin/ccache /usr/local/bin/c++
+RUN ln -s /usr/bin/ccache /usr/local/bin/clang
+RUN ln -s /usr/bin/ccache /usr/local/bin/clang++
+
RUN mkdir /var/local/jenkins
+
# Define the default command.
CMD ["bash"]
diff --git a/tools/run_tests/dockerize/build_docker_and_run_tests.sh b/tools/run_tests/dockerize/build_docker_and_run_tests.sh
index c3219c5..b68ac89 100755
--- a/tools/run_tests/dockerize/build_docker_and_run_tests.sh
+++ b/tools/run_tests/dockerize/build_docker_and_run_tests.sh
@@ -77,8 +77,6 @@
-v /tmp/ccache:/tmp/ccache \
-v /tmp/npm-cache:/tmp/npm-cache \
-v /tmp/xdg-cache-home:/tmp/xdg-cache-home \
- -v /var/run/docker.sock:/var/run/docker.sock \
- -v $(which docker):/bin/docker \
-w /var/local/git/grpc \
--name=$CONTAINER_NAME \
$DOCKER_IMAGE_NAME \
diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py
index 1008c6b..9242741 100755
--- a/tools/run_tests/run_tests.py
+++ b/tools/run_tests/run_tests.py
@@ -820,8 +820,12 @@
def test_specs(self):
import yaml
with open('tools/run_tests/sanity/sanity_tests.yaml', 'r') as f:
+ environ={'TEST': 'true'}
+ if _is_use_docker_child():
+ environ['CLANG_FORMAT_SKIP_DOCKER'] = 'true'
return [self.config.job_spec(cmd['script'].split(),
- timeout_seconds=30*60, environ={'TEST': 'true'},
+ timeout_seconds=30*60,
+ environ=environ,
cpu_cost=cmd.get('cpu_cost', 1))
for cmd in yaml.load(f)]