diff options
Diffstat (limited to 'tools')
23 files changed, 565 insertions, 31 deletions
diff --git a/tools/dockerfile/grpc_interop_stress_cxx/Dockerfile b/tools/dockerfile/grpc_interop_stress_cxx/Dockerfile index 214747fd4a..556a26ee13 100644 --- a/tools/dockerfile/grpc_interop_stress_cxx/Dockerfile +++ b/tools/dockerfile/grpc_interop_stress_cxx/Dockerfile @@ -81,7 +81,7 @@ RUN pip install --upgrade google-api-python-client #================= -# Update clang to a version with improved tsan +# Update clang to a version with improved tsan and fuzzing capabilities RUN apt-get update && apt-get -y install python cmake && apt-get clean @@ -111,7 +111,7 @@ RUN cd llvm-build && cmake \ -DCMAKE_INSTALL_PREFIX:STRING=/usr \ -DLLVM_TARGETS_TO_BUILD:STRING=X86 \ ../llvm -RUN make -C llvm-build && make -C llvm-build install && rm -rf llvm-build +RUN make -C llvm-build -j 12 && make -C llvm-build install && rm -rf llvm-build # Define the default command. CMD ["bash"] diff --git a/tools/dockerfile/test/cxx_jessie_x64/Dockerfile b/tools/dockerfile/test/cxx_jessie_x64/Dockerfile index b848f233b7..6c7aece4ff 100644 --- a/tools/dockerfile/test/cxx_jessie_x64/Dockerfile +++ b/tools/dockerfile/test/cxx_jessie_x64/Dockerfile @@ -68,7 +68,7 @@ RUN apt-get update && apt-get install -y time && apt-get clean RUN apt-get update && apt-get -y install libgflags-dev libgtest-dev libc++-dev clang && apt-get clean #================= -# Update clang to a version with improved tsan +# Update clang to a version with improved tsan and fuzzing capabilities RUN apt-get update && apt-get -y install python cmake && apt-get clean @@ -98,7 +98,7 @@ RUN cd llvm-build && cmake \ -DCMAKE_INSTALL_PREFIX:STRING=/usr \ -DLLVM_TARGETS_TO_BUILD:STRING=X86 \ ../llvm -RUN make -C llvm-build && make -C llvm-build install && rm -rf llvm-build +RUN make -C llvm-build -j 12 && make -C llvm-build install && rm -rf llvm-build # Prepare ccache RUN ln -s /usr/bin/ccache /usr/local/bin/gcc diff --git a/tools/dockerfile/test/fuzzer/Dockerfile b/tools/dockerfile/test/fuzzer/Dockerfile new file mode 100644 index 0000000000..7b35c009e3 --- /dev/null +++ b/tools/dockerfile/test/fuzzer/Dockerfile @@ -0,0 +1,123 @@ +# Copyright 2015-2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +FROM debian:jessie + +# Install Git and basic packages. +RUN apt-get update && apt-get install -y \ + autoconf \ + autotools-dev \ + build-essential \ + bzip2 \ + ccache \ + curl \ + gcc \ + gcc-multilib \ + git \ + golang \ + gyp \ + lcov \ + libc6 \ + libc6-dbg \ + libc6-dev \ + libgtest-dev \ + libtool \ + make \ + perl \ + strace \ + python-dev \ + python-setuptools \ + python-yaml \ + telnet \ + unzip \ + wget \ + zip && apt-get clean + +#================ +# Build profiling +RUN apt-get update && apt-get install -y time && apt-get clean + +#================= +# C++ dependencies +RUN apt-get update && apt-get -y install libgflags-dev libgtest-dev libc++-dev clang && apt-get clean + +#================= +# Update clang to a version with improved tsan and fuzzing capabilities + +RUN apt-get update && apt-get -y install python cmake && apt-get clean + +RUN git clone -n -b release_38 http://llvm.org/git/llvm.git && \ + cd llvm && git checkout ad57503 && cd .. +RUN git clone -n -b release_38 http://llvm.org/git/clang.git && \ + cd clang && git checkout ad2c56e && cd .. +RUN git clone -n -b release_38 http://llvm.org/git/compiler-rt.git && \ + cd compiler-rt && git checkout 3176922 && cd .. +RUN git clone -n -b release_38 \ + http://llvm.org/git/clang-tools-extra.git && cd clang-tools-extra && \ + git checkout c288525 && cd .. +RUN git clone -n -b release_38 http://llvm.org/git/libcxx.git && \ + cd libcxx && git checkout fda3549 && cd .. +RUN git clone -n -b release_38 http://llvm.org/git/libcxxabi.git && \ + cd libcxxabi && git checkout 8d4e51d && cd .. + +RUN mv clang llvm/tools +RUN mv compiler-rt llvm/projects +RUN mv clang-tools-extra llvm/tools/clang/tools +RUN mv libcxx llvm/projects +RUN mv libcxxabi llvm/projects + +RUN mkdir llvm-build +RUN cd llvm-build && cmake \ + -DCMAKE_BUILD_TYPE:STRING=Release \ + -DCMAKE_INSTALL_PREFIX:STRING=/usr \ + -DLLVM_TARGETS_TO_BUILD:STRING=X86 \ + ../llvm +RUN make -C llvm-build -j 12 && make -C llvm-build install && rm -rf llvm-build + +# Prepare ccache +RUN ln -s /usr/bin/ccache /usr/local/bin/gcc +RUN ln -s /usr/bin/ccache /usr/local/bin/g++ +RUN ln -s /usr/bin/ccache /usr/local/bin/cc +RUN ln -s /usr/bin/ccache /usr/local/bin/c++ +RUN ln -s /usr/bin/ccache /usr/local/bin/clang +RUN ln -s /usr/bin/ccache /usr/local/bin/clang++ + +#====================== +# Zookeeper dependencies +# TODO(jtattermusch): is zookeeper still needed? +RUN apt-get install -y libzookeeper-mt-dev + +RUN mkdir /var/local/jenkins + +RUN clang++ -c -g -O2 -std=c++11 llvm/lib/Fuzzer/*.cpp -IFuzzer +RUN ar ruv libFuzzer.a Fuzzer*.o +RUN mv libFuzzer.a /usr/lib +RUN rm -f Fuzzer*.o +# Define the default command. +CMD ["bash"] diff --git a/tools/fuzzer/build_and_run_fuzzer.sh b/tools/fuzzer/build_and_run_fuzzer.sh new file mode 100755 index 0000000000..bbe68884ad --- /dev/null +++ b/tools/fuzzer/build_and_run_fuzzer.sh @@ -0,0 +1,37 @@ +#!/bin/bash +# Copyright 2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# + +set -ex + +cd $(dirname $0)/../.. +make CONFIG=$config $1 -j3 +mkdir -p fuzzer_output +. tools/fuzzer/runners/$1.sh diff --git a/tools/fuzzer/runners/hpack_parser_fuzzer_test.sh b/tools/fuzzer/runners/hpack_parser_fuzzer_test.sh new file mode 100644 index 0000000000..a91cf6d458 --- /dev/null +++ b/tools/fuzzer/runners/hpack_parser_fuzzer_test.sh @@ -0,0 +1,38 @@ +#!/bin/bash +# Copyright 2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# + +flags="-max_total_time=3600 -jobs=3 -workers=3" +if [ "$config" == "asan-trace-cmp" ] +then + flags="-use_traces=1 $flags" +fi + +bins/$config/hpack_parser_fuzzer_test $flags fuzzer_output test/core/transport/chttp2/hpack_parser_corpus diff --git a/tools/fuzzer/runners/http_fuzzer_test.sh b/tools/fuzzer/runners/http_fuzzer_test.sh new file mode 100644 index 0000000000..47adad0cb7 --- /dev/null +++ b/tools/fuzzer/runners/http_fuzzer_test.sh @@ -0,0 +1,38 @@ +#!/bin/bash +# Copyright 2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# + +flags="-max_total_time=3600 -jobs=3 -workers=3" +if [ "$config" == "asan-trace-cmp" ] +then + flags="-use_traces=1 $flags" +fi + +bins/$config/http_fuzzer_test $flags fuzzer_output test/core/http/corpus diff --git a/tools/fuzzer/runners/json_fuzzer_test.sh b/tools/fuzzer/runners/json_fuzzer_test.sh new file mode 100644 index 0000000000..9c5cba3baf --- /dev/null +++ b/tools/fuzzer/runners/json_fuzzer_test.sh @@ -0,0 +1,38 @@ +#!/bin/bash +# Copyright 2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# + +flags="-max_total_time=3600 -jobs=3 -workers=3" +if [ "$config" == "asan-trace-cmp" ] +then + flags="-use_traces=1 $flags" +fi + +bins/$config/json_fuzzer_test $flags fuzzer_output test/core/json/corpus diff --git a/tools/fuzzer/runners/uri_fuzzer_test.sh b/tools/fuzzer/runners/uri_fuzzer_test.sh new file mode 100644 index 0000000000..58f20d021c --- /dev/null +++ b/tools/fuzzer/runners/uri_fuzzer_test.sh @@ -0,0 +1,38 @@ +#!/bin/bash +# Copyright 2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# + +flags="-max_total_time=3600 -jobs=3 -workers=3" +if [ "$config" == "asan-trace-cmp" ] +then + flags="-use_traces=1 $flags" +fi + +bins/$config/uri_fuzzer_test $flags fuzzer_output test/core/client_config/uri_corpus diff --git a/tools/gce/create_interop_worker.sh b/tools/gce/create_interop_worker.sh new file mode 100755 index 0000000000..3c49c6102a --- /dev/null +++ b/tools/gce/create_interop_worker.sh @@ -0,0 +1,64 @@ +#!/bin/bash +# Copyright 2015-2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# Creates an interop worker on GCE. +# IMPORTANT: After this script finishes, there are still some manual +# steps needed there are hard to automatize. +# See go/grpc-jenkins-setup for followup instructions. + +set -ex + +cd $(dirname $0) + +CLOUD_PROJECT=grpc-testing +ZONE=us-east1-a # canary gateway is reachable from this zone + +INSTANCE_NAME="${1:-grpc-canary-interop2}" + +gcloud compute instances create $INSTANCE_NAME \ + --project="$CLOUD_PROJECT" \ + --zone "$ZONE" \ + --machine-type n1-standard-16 \ + --image ubuntu-15-10 \ + --boot-disk-size 1000 \ + --scopes https://www.googleapis.com/auth/xapi.zoo + +echo 'Created GCE instance, waiting 60 seconds for it to come online.' +sleep 60 + +gcloud compute copy-files \ + --project="$CLOUD_PROJECT" \ + --zone "$ZONE" \ + jenkins_master.pub linux_worker_init.sh ${INSTANCE_NAME}:~ + +gcloud compute ssh \ + --project="$CLOUD_PROJECT" \ + --zone "$ZONE" \ + $INSTANCE_NAME --command "./linux_worker_init.sh" diff --git a/tools/gource/gource.sh b/tools/gource/gource.sh new file mode 100755 index 0000000000..167a256b84 --- /dev/null +++ b/tools/gource/gource.sh @@ -0,0 +1,31 @@ +#!/bin/bash +# Copyright 2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +gource -c 4 -s 0.1 --max-file-lag 0.05 --max-files 0 -e 0.01 --hide filenames,dirnames --file-filter 'doc/ref|vsprojects/vcxproj' $* diff --git a/tools/jenkins/docker_run_tests.sh b/tools/jenkins/docker_run_tests.sh index 8d6c42cd9b..90a2bd8e36 100755 --- a/tools/jenkins/docker_run_tests.sh +++ b/tools/jenkins/docker_run_tests.sh @@ -34,7 +34,7 @@ set -e export CONFIG=$config -export ASAN_SYMBOLIZER_PATH=/usr/bin/llvm-symbolizer-3.5 +export ASAN_SYMBOLIZER_PATH=/usr/bin/llvm-symbolizer # Ensure that programs depending on current-user-ownership of cache directories # are satisfied (it's being mounted from outside the image). diff --git a/tools/jenkins/run_fuzzer.sh b/tools/jenkins/run_fuzzer.sh new file mode 100755 index 0000000000..3d6da99762 --- /dev/null +++ b/tools/jenkins/run_fuzzer.sh @@ -0,0 +1,43 @@ +#!/bin/bash +# Copyright 2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# Builds and runs a fuzzer (specified by the first command line argument) + +set -ex + +export RUN_COMMAND="tools/fuzzer/build_and_run_fuzzer.sh $1" +export DOCKER_RUN_SCRIPT=tools/jenkins/docker_run.sh +export DOCKERFILE_DIR=tools/dockerfile/test/fuzzer +export OUTPUT_DIR=fuzzer_output + +tools/jenkins/build_and_run_docker.sh \ + -e RUN_COMMAND="$RUN_COMMAND" \ + -e OUTPUT_DIR="$OUTPUT_DIR" \ + -e config="$config" diff --git a/tools/run_tests/build_artifact_node.sh b/tools/run_tests/build_artifact_node.sh index 6aa4824538..ef3476a038 100755 --- a/tools/run_tests/build_artifact_node.sh +++ b/tools/run_tests/build_artifact_node.sh @@ -30,9 +30,9 @@ NODE_TARGET_ARCH=$1 source ~/.nvm/nvm.sh -set -ex nvm use 4 +set -ex cd $(dirname $0)/../.. diff --git a/tools/run_tests/build_artifact_python.sh b/tools/run_tests/build_artifact_python.sh index 7ba04d7546..1f23f9fade 100755 --- a/tools/run_tests/build_artifact_python.sh +++ b/tools/run_tests/build_artifact_python.sh @@ -35,15 +35,18 @@ cd $(dirname $0)/../.. if [ "$SKIP_PIP_INSTALL" == "" ] then pip install --upgrade six - pip install --upgrade setuptools + # There's a bug in newer versions of setuptools (see + # https://bitbucket.org/pypa/setuptools/issues/503/pkg_resources_vendorpackagingrequirementsi) + pip install --upgrade 'setuptools==18' pip install -rrequirements.txt fi +export GRPC_PYTHON_USE_CUSTOM_BDIST=0 +export GRPC_PYTHON_BUILD_WITH_CYTHON=1 + # Build the source distribution first because MANIFEST.in cannot override # exclusion of built shared objects among package resources (for some # inexplicable reason). -GRPC_PYTHON_USE_CUSTOM_BDIST=0 \ -GRPC_PYTHON_BUILD_WITH_CYTHON=1 \ ${SETARCH_CMD} python setup.py \ sdist @@ -51,15 +54,11 @@ ${SETARCH_CMD} python setup.py \ # and thus ought to be run in a shell command separate of others. Further, it # trashes the actual bdist_wheel output, so it should be run first so that # bdist_wheel may be run unmolested. -GRPC_PYTHON_USE_CUSTOM_BDIST=0 \ -GRPC_PYTHON_BUILD_WITH_CYTHON=1 \ ${SETARCH_CMD} python setup.py \ build_tagged_ext # Wheel has a bug where directories don't get excluded. # https://bitbucket.org/pypa/wheel/issues/99/cannot-exclude-directory -GRPC_PYTHON_USE_CUSTOM_BDIST=0 \ -GRPC_PYTHON_BUILD_WITH_CYTHON=1 \ ${SETARCH_CMD} python setup.py \ bdist_wheel diff --git a/tools/run_tests/build_node.sh b/tools/run_tests/build_node.sh index 9c4af07185..cbe0e31d2e 100755 --- a/tools/run_tests/build_node.sh +++ b/tools/run_tests/build_node.sh @@ -31,9 +31,9 @@ NODE_VERSION=$1 source ~/.nvm/nvm.sh -set -ex nvm use $NODE_VERSION +set -ex CONFIG=${CONFIG:-opt} diff --git a/tools/run_tests/build_package_node.sh b/tools/run_tests/build_package_node.sh index aca90a3750..540c826311 100755 --- a/tools/run_tests/build_package_node.sh +++ b/tools/run_tests/build_package_node.sh @@ -29,9 +29,9 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. source ~/.nvm/nvm.sh -set -ex nvm use 4 +set -ex cd $(dirname $0)/../.. diff --git a/tools/run_tests/build_python.sh b/tools/run_tests/build_python.sh index 79a148faf1..fe84300c7b 100755 --- a/tools/run_tests/build_python.sh +++ b/tools/run_tests/build_python.sh @@ -40,6 +40,7 @@ export PATH=$ROOT/bins/$CONFIG:$ROOT/bins/$CONFIG/protobuf:$PATH export CFLAGS="-I$ROOT/include -std=gnu99" export LDFLAGS="-L$ROOT/libs/$CONFIG" export GRPC_PYTHON_BUILD_WITH_CYTHON=1 +export GRPC_PYTHON_USE_PRECOMPILED_BINARIES=0 if [ "$CONFIG" = "gcov" ] then diff --git a/tools/run_tests/configs.json b/tools/run_tests/configs.json index a858170d87..325e9aa929 100644 --- a/tools/run_tests/configs.json +++ b/tools/run_tests/configs.json @@ -3,6 +3,14 @@ "config": "opt" }, { + "config": "asan-trace-cmp", + "environ": { + "ASAN_OPTIONS": "detect_leaks=1:color=always", + "LSAN_OPTIONS": "suppressions=tools/lsan_suppressions.txt:report_objects=1" + }, + "timeout_multiplier": 3 + }, + { "config": "dbg" }, { diff --git a/tools/run_tests/pre_build_node.sh b/tools/run_tests/pre_build_node.sh index 11f46d60fc..1f55df0b7b 100755 --- a/tools/run_tests/pre_build_node.sh +++ b/tools/run_tests/pre_build_node.sh @@ -31,9 +31,9 @@ NODE_VERSION=$1 source ~/.nvm/nvm.sh -set -ex nvm use $NODE_VERSION +set -ex export GRPC_CONFIG=${CONFIG:-opt} diff --git a/tools/run_tests/run_node.sh b/tools/run_tests/run_node.sh index d33890068d..b94dc3ec62 100755 --- a/tools/run_tests/run_node.sh +++ b/tools/run_tests/run_node.sh @@ -30,9 +30,9 @@ NODE_VERSION=$1 source ~/.nvm/nvm.sh -set -ex nvm use $NODE_VERSION +set -ex CONFIG=${CONFIG:-opt} diff --git a/tools/run_tests/run_python.sh b/tools/run_tests/run_python.sh index d4b7250cbb..e3be14e7c7 100755 --- a/tools/run_tests/run_python.sh +++ b/tools/run_tests/run_python.sh @@ -40,6 +40,7 @@ export PATH=$ROOT/bins/$CONFIG:$ROOT/bins/$CONFIG/protobuf:$PATH export CFLAGS="-I$ROOT/include -std=c89" export LDFLAGS="-L$ROOT/libs/$CONFIG" export GRPC_PYTHON_BUILD_WITH_CYTHON=1 +export GRPC_PYTHON_USE_PRECOMPILED_BINARIES=0 if [ "$CONFIG" = "gcov" ] then diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index a9438045aa..71bb7b12db 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -120,7 +120,12 @@ def get_c_tests(travis, test_lang) : def _check_compiler(compiler, supported_compilers): if compiler not in supported_compilers: - raise Exception('Compiler %s not supported.' % compiler) + raise Exception('Compiler %s not supported (on this platform).' % compiler) + + +def _check_arch(arch, supported_archs): + if arch not in supported_archs: + raise Exception('Architecture %s not supported.' % arch) def _is_use_docker_child(): @@ -464,7 +469,20 @@ class CSharpLanguage(object): def configure(self, config, args): self.config = config self.args = args - _check_compiler(self.args.compiler, ['default']) + if self.platform == 'windows': + # Explicitly choosing between x86 and x64 arch doesn't work yet + _check_arch(self.args.arch, ['default']) + self._make_options = [_windows_toolset_option(self.args.compiler), + _windows_arch_option(self.args.arch)] + else: + _check_compiler(self.args.compiler, ['default']) + if self.platform == 'mac': + # On Mac, official distribution of mono is 32bit. + # TODO(jtattermusch): EMBED_ZLIB=true currently breaks the mac build + self._make_options = ['EMBED_OPENSSL=true', + 'CFLAGS=-m32', 'LDFLAGS=-m32'] + else: + self._make_options = ['EMBED_OPENSSL=true', 'EMBED_ZLIB=true'] def test_specs(self): with open('src/csharp/tests.json') as f: @@ -511,23 +529,16 @@ class CSharpLanguage(object): return [['tools/run_tests/pre_build_csharp.sh']] def make_targets(self): - # For Windows, this target doesn't really build anything, - # everything is build by buildall script later. - if self.platform == 'windows': - return [] - else: - return ['grpc_csharp_ext'] + return ['grpc_csharp_ext'] def make_options(self): - if self.platform == 'mac': - # On Mac, official distribution of mono is 32bit. - return ['CFLAGS=-arch i386', 'LDFLAGS=-arch i386'] - else: - return [] + return self._make_options; def build_steps(self): if self.platform == 'windows': - return [['src\\csharp\\buildall.bat']] + return [[_windows_build_bat(self.args.compiler), + 'src/csharp/Grpc.sln', + '/p:Configuration=%s' % _MSBUILD_CONFIG[self.config.build_config]]] else: return [['tools/run_tests/build_csharp.sh']] diff --git a/tools/run_tests/sources_and_headers.json b/tools/run_tests/sources_and_headers.json index e1bfa8719b..d7c9839d5a 100644 --- a/tools/run_tests/sources_and_headers.json +++ b/tools/run_tests/sources_and_headers.json @@ -891,6 +891,22 @@ ], "headers": [], "language": "c", + "name": "hpack_parser_fuzzer_test", + "src": [ + "test/core/transport/chttp2/hpack_parser_fuzzer_test.c" + ], + "third_party": false, + "type": "target" + }, + { + "deps": [ + "gpr", + "gpr_test_util", + "grpc", + "grpc_test_util" + ], + "headers": [], + "language": "c", "name": "hpack_parser_test", "src": [ "test/core/transport/chttp2/hpack_parser_test.c" @@ -923,6 +939,22 @@ ], "headers": [], "language": "c", + "name": "http_fuzzer_test", + "src": [ + "test/core/http/fuzzer.c" + ], + "third_party": false, + "type": "target" + }, + { + "deps": [ + "gpr", + "gpr_test_util", + "grpc", + "grpc_test_util" + ], + "headers": [], + "language": "c", "name": "http_parser_test", "src": [ "test/core/http/parser_test.c" @@ -1013,6 +1045,22 @@ { "deps": [ "gpr", + "gpr_test_util", + "grpc", + "grpc_test_util" + ], + "headers": [], + "language": "c", + "name": "json_fuzzer_test", + "src": [ + "test/core/json/fuzzer.c" + ], + "third_party": false, + "type": "target" + }, + { + "deps": [ + "gpr", "grpc" ], "headers": [], @@ -1544,6 +1592,22 @@ ], "headers": [], "language": "c", + "name": "uri_fuzzer_test", + "src": [ + "test/core/client_config/uri_fuzzer_test.c" + ], + "third_party": false, + "type": "target" + }, + { + "deps": [ + "gpr", + "gpr_test_util", + "grpc", + "grpc_test_util" + ], + "headers": [], + "language": "c", "name": "uri_parser_test", "src": [ "test/core/client_config/uri_parser_test.c" |