diff options
Diffstat (limited to 'tools')
-rwxr-xr-x | tools/buildgen/generate_projects.py | 3 | ||||
-rw-r--r-- | tools/dockerfile/grpc_artifact_linux_x64/Dockerfile (renamed from tools/jenkins/grpc_artifact_linux_x64/Dockerfile) | 0 | ||||
-rw-r--r-- | tools/dockerfile/grpc_artifact_linux_x86/Dockerfile (renamed from tools/jenkins/grpc_artifact_linux_x86/Dockerfile) | 0 | ||||
-rw-r--r-- | tools/dockerfile/grpc_interop_csharp/Dockerfile (renamed from tools/jenkins/grpc_interop_csharp/Dockerfile) | 2 | ||||
-rwxr-xr-x | tools/dockerfile/grpc_interop_csharp/build_interop.sh (renamed from tools/jenkins/grpc_interop_csharp/build_interop.sh) | 0 | ||||
-rw-r--r-- | tools/dockerfile/grpc_interop_cxx/Dockerfile (renamed from tools/jenkins/grpc_interop_stress_cxx/Dockerfile) | 0 | ||||
-rwxr-xr-x | tools/dockerfile/grpc_interop_cxx/build_interop.sh (renamed from tools/jenkins/grpc_interop_cxx/build_interop.sh) | 2 | ||||
-rw-r--r-- | tools/dockerfile/grpc_interop_go/Dockerfile (renamed from tools/jenkins/grpc_interop_http2/Dockerfile) | 2 | ||||
-rwxr-xr-x | tools/dockerfile/grpc_interop_go/build_interop.sh (renamed from tools/jenkins/grpc_interop_go/build_interop.sh) | 2 | ||||
-rw-r--r-- | tools/dockerfile/grpc_interop_http2/Dockerfile (renamed from tools/jenkins/grpc_interop_go/Dockerfile) | 2 | ||||
-rwxr-xr-x | tools/dockerfile/grpc_interop_http2/build_interop.sh (renamed from tools/jenkins/grpc_interop_http2/build_interop.sh) | 2 | ||||
-rw-r--r-- | tools/dockerfile/grpc_interop_java/Dockerfile (renamed from tools/jenkins/grpc_interop_java/Dockerfile) | 2 | ||||
-rwxr-xr-x | tools/dockerfile/grpc_interop_java/build_interop.sh (renamed from tools/jenkins/grpc_interop_java/build_interop.sh) | 2 | ||||
-rw-r--r-- | tools/dockerfile/grpc_interop_node/Dockerfile (renamed from tools/jenkins/grpc_interop_node/Dockerfile) | 2 | ||||
-rwxr-xr-x | tools/dockerfile/grpc_interop_node/build_interop.sh (renamed from tools/jenkins/grpc_interop_node/build_interop.sh) | 4 | ||||
-rw-r--r-- | tools/dockerfile/grpc_interop_php/Dockerfile (renamed from tools/jenkins/grpc_interop_php/Dockerfile) | 2 | ||||
-rwxr-xr-x | tools/dockerfile/grpc_interop_php/build_interop.sh (renamed from tools/jenkins/grpc_interop_php/build_interop.sh) | 2 | ||||
-rw-r--r-- | tools/dockerfile/grpc_interop_python/Dockerfile (renamed from tools/jenkins/grpc_interop_python/Dockerfile) | 2 | ||||
-rwxr-xr-x | tools/dockerfile/grpc_interop_python/build_interop.sh (renamed from tools/jenkins/grpc_interop_python/build_interop.sh) | 2 | ||||
-rw-r--r-- | tools/dockerfile/grpc_interop_ruby/Dockerfile (renamed from tools/jenkins/grpc_interop_ruby/Dockerfile) | 2 | ||||
-rwxr-xr-x | tools/dockerfile/grpc_interop_ruby/build_interop.sh (renamed from tools/jenkins/grpc_interop_ruby/build_interop.sh) | 2 | ||||
-rw-r--r-- | tools/dockerfile/grpc_interop_stress_cxx/Dockerfile (renamed from tools/jenkins/grpc_interop_cxx/Dockerfile) | 2 | ||||
-rwxr-xr-x | tools/dockerfile/grpc_interop_stress_cxx/build_interop_stress.sh (renamed from tools/jenkins/grpc_interop_stress_cxx/build_interop_stress.sh) | 0 | ||||
-rw-r--r-- | tools/dockerfile/grpc_linuxbrew/Dockerfile (renamed from tools/jenkins/grpc_linuxbrew/Dockerfile) | 2 | ||||
-rw-r--r-- | tools/dockerfile/grpc_sanity/Dockerfile | 79 | ||||
-rw-r--r-- | tools/dockerfile/grpc_tests_multilang_x64/Dockerfile (renamed from tools/jenkins/grpc_jenkins_slave/Dockerfile) | 15 | ||||
-rw-r--r-- | tools/dockerfile/grpc_tests_multilang_x86/Dockerfile (renamed from tools/jenkins/grpc_jenkins_slave_32bits/Dockerfile) | 8 | ||||
-rw-r--r-- | tools/doxygen/Doxyfile.c++ | 2 | ||||
-rw-r--r-- | tools/doxygen/Doxyfile.c++.internal | 2 | ||||
-rw-r--r-- | tools/doxygen/Doxyfile.core | 2 | ||||
-rw-r--r-- | tools/doxygen/Doxyfile.core.internal | 2 | ||||
-rwxr-xr-x | tools/jenkins/build_docker_and_run_tests.sh | 12 | ||||
-rwxr-xr-x | tools/jenkins/build_interop_image.sh | 6 | ||||
-rwxr-xr-x | tools/jenkins/build_interop_stress_image.sh | 6 | ||||
-rwxr-xr-x | tools/jenkins/docker_run_tests.sh | 10 | ||||
-rwxr-xr-x | tools/jenkins/run_distribution.sh | 6 | ||||
-rwxr-xr-x | tools/jenkins/run_jenkins.sh | 44 | ||||
-rwxr-xr-x | tools/jenkins/run_portability.sh | 21 | ||||
-rwxr-xr-x | tools/run_tests/build_artifacts.py | 21 | ||||
-rwxr-xr-x | tools/run_tests/build_node.sh | 4 | ||||
-rw-r--r-- | tools/run_tests/configs.json | 2 | ||||
-rwxr-xr-x | tools/run_tests/jobset.py | 2 | ||||
-rwxr-xr-x | tools/run_tests/post_tests_ruby.sh | 4 | ||||
-rwxr-xr-x | tools/run_tests/run_node.sh | 4 | ||||
-rwxr-xr-x | tools/run_tests/run_tests.py | 171 | ||||
-rwxr-xr-x | tools/run_tests/sanity/check_cache_mk.sh (renamed from tools/run_tests/check_cache_mk.sh) | 0 | ||||
-rwxr-xr-x | tools/run_tests/sanity/check_sources_and_headers.py (renamed from tools/run_tests/check_sources_and_headers.py) | 66 | ||||
-rwxr-xr-x | tools/run_tests/sanity/check_submodules.sh (renamed from tools/run_tests/check_submodules.sh) | 2 | ||||
-rw-r--r-- | tools/run_tests/sanity/sanity_tests.yaml | 9 | ||||
-rw-r--r-- | tools/run_tests/sanity_tests.yaml | 9 | ||||
-rw-r--r-- | tools/run_tests/tests.json | 59 | ||||
-rw-r--r-- | tools/tsan_suppressions.txt | 2 |
52 files changed, 389 insertions, 222 deletions
diff --git a/tools/buildgen/generate_projects.py b/tools/buildgen/generate_projects.py index 083a97874d..965dd292af 100755 --- a/tools/buildgen/generate_projects.py +++ b/tools/buildgen/generate_projects.py @@ -47,6 +47,7 @@ os.chdir(os.path.join(os.path.dirname(sys.argv[0]), '..', '..')) argp = argparse.ArgumentParser() argp.add_argument('json', nargs='+') argp.add_argument('--templates', nargs='+', default=[]) +argp.add_argument('--jobs', '-j', default=multiprocessing.cpu_count(), type=int) args = argp.parse_args() json = args.json @@ -87,7 +88,7 @@ for template in templates: cmd.append(root + '/' + f) jobs.append(jobset.JobSpec(cmd, shortname=out, timeout_seconds=None)) -jobset.run(jobs, maxjobs=multiprocessing.cpu_count()) +jobset.run(jobs, maxjobs=args.jobs) if test is not None: for s, g in test.iteritems(): diff --git a/tools/jenkins/grpc_artifact_linux_x64/Dockerfile b/tools/dockerfile/grpc_artifact_linux_x64/Dockerfile index de40247a73..de40247a73 100644 --- a/tools/jenkins/grpc_artifact_linux_x64/Dockerfile +++ b/tools/dockerfile/grpc_artifact_linux_x64/Dockerfile diff --git a/tools/jenkins/grpc_artifact_linux_x86/Dockerfile b/tools/dockerfile/grpc_artifact_linux_x86/Dockerfile index 774b4523a1..774b4523a1 100644 --- a/tools/jenkins/grpc_artifact_linux_x86/Dockerfile +++ b/tools/dockerfile/grpc_artifact_linux_x86/Dockerfile diff --git a/tools/jenkins/grpc_interop_csharp/Dockerfile b/tools/dockerfile/grpc_interop_csharp/Dockerfile index 3789cd3203..c3c79487f7 100644 --- a/tools/jenkins/grpc_interop_csharp/Dockerfile +++ b/tools/dockerfile/grpc_interop_csharp/Dockerfile @@ -1,4 +1,4 @@ -# Copyright 2015, Google Inc. +# Copyright 2015-2016, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without diff --git a/tools/jenkins/grpc_interop_csharp/build_interop.sh b/tools/dockerfile/grpc_interop_csharp/build_interop.sh index e87fe2f692..e87fe2f692 100755 --- a/tools/jenkins/grpc_interop_csharp/build_interop.sh +++ b/tools/dockerfile/grpc_interop_csharp/build_interop.sh diff --git a/tools/jenkins/grpc_interop_stress_cxx/Dockerfile b/tools/dockerfile/grpc_interop_cxx/Dockerfile index 58a8c32e34..58a8c32e34 100644 --- a/tools/jenkins/grpc_interop_stress_cxx/Dockerfile +++ b/tools/dockerfile/grpc_interop_cxx/Dockerfile diff --git a/tools/jenkins/grpc_interop_cxx/build_interop.sh b/tools/dockerfile/grpc_interop_cxx/build_interop.sh index 1c0828d23a..903c31b3e7 100755 --- a/tools/jenkins/grpc_interop_cxx/build_interop.sh +++ b/tools/dockerfile/grpc_interop_cxx/build_interop.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2015, Google Inc. +# Copyright 2015-2016, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without diff --git a/tools/jenkins/grpc_interop_http2/Dockerfile b/tools/dockerfile/grpc_interop_go/Dockerfile index bb60f09f24..f83e18c8a3 100644 --- a/tools/jenkins/grpc_interop_http2/Dockerfile +++ b/tools/dockerfile/grpc_interop_go/Dockerfile @@ -1,4 +1,4 @@ -# Copyright 2015, Google Inc. +# Copyright 2015-2016, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without diff --git a/tools/jenkins/grpc_interop_go/build_interop.sh b/tools/dockerfile/grpc_interop_go/build_interop.sh index e891549342..224b0040c0 100755 --- a/tools/jenkins/grpc_interop_go/build_interop.sh +++ b/tools/dockerfile/grpc_interop_go/build_interop.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2015, Google Inc. +# Copyright 2015-2016, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without diff --git a/tools/jenkins/grpc_interop_go/Dockerfile b/tools/dockerfile/grpc_interop_http2/Dockerfile index bb60f09f24..f83e18c8a3 100644 --- a/tools/jenkins/grpc_interop_go/Dockerfile +++ b/tools/dockerfile/grpc_interop_http2/Dockerfile @@ -1,4 +1,4 @@ -# Copyright 2015, Google Inc. +# Copyright 2015-2016, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without diff --git a/tools/jenkins/grpc_interop_http2/build_interop.sh b/tools/dockerfile/grpc_interop_http2/build_interop.sh index 46ddaf929a..5f701e34c0 100755 --- a/tools/jenkins/grpc_interop_http2/build_interop.sh +++ b/tools/dockerfile/grpc_interop_http2/build_interop.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2015, Google Inc. +# Copyright 2015-2016, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without diff --git a/tools/jenkins/grpc_interop_java/Dockerfile b/tools/dockerfile/grpc_interop_java/Dockerfile index 1ec24a95e3..5ba5324631 100644 --- a/tools/jenkins/grpc_interop_java/Dockerfile +++ b/tools/dockerfile/grpc_interop_java/Dockerfile @@ -1,4 +1,4 @@ -# Copyright 2015, Google Inc. +# Copyright 2015-2016, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without diff --git a/tools/jenkins/grpc_interop_java/build_interop.sh b/tools/dockerfile/grpc_interop_java/build_interop.sh index 9997c63308..9c9591abd8 100755 --- a/tools/jenkins/grpc_interop_java/build_interop.sh +++ b/tools/dockerfile/grpc_interop_java/build_interop.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2015, Google Inc. +# Copyright 2015-2016, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without diff --git a/tools/jenkins/grpc_interop_node/Dockerfile b/tools/dockerfile/grpc_interop_node/Dockerfile index db5aff844d..3f0f918ad9 100644 --- a/tools/jenkins/grpc_interop_node/Dockerfile +++ b/tools/dockerfile/grpc_interop_node/Dockerfile @@ -1,4 +1,4 @@ -# Copyright 2015, Google Inc. +# Copyright 2015-2016, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without diff --git a/tools/jenkins/grpc_interop_node/build_interop.sh b/tools/dockerfile/grpc_interop_node/build_interop.sh index 3b69715c9a..526dd61fdf 100755 --- a/tools/jenkins/grpc_interop_node/build_interop.sh +++ b/tools/dockerfile/grpc_interop_node/build_interop.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2015, Google Inc. +# Copyright 2015-2016, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -45,4 +45,4 @@ make install-certs # build Node interop client & server npm install -g node-gyp -(npm install && node-gyp rebuild) +npm install --unsafe-perm --build-from-source diff --git a/tools/jenkins/grpc_interop_php/Dockerfile b/tools/dockerfile/grpc_interop_php/Dockerfile index cf3e79176a..4f5b3fcb77 100644 --- a/tools/jenkins/grpc_interop_php/Dockerfile +++ b/tools/dockerfile/grpc_interop_php/Dockerfile @@ -1,4 +1,4 @@ -# Copyright 2015, Google Inc. +# Copyright 2015-2016, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without diff --git a/tools/jenkins/grpc_interop_php/build_interop.sh b/tools/dockerfile/grpc_interop_php/build_interop.sh index 87262f1d62..2bffbd8938 100755 --- a/tools/jenkins/grpc_interop_php/build_interop.sh +++ b/tools/dockerfile/grpc_interop_php/build_interop.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2015, Google Inc. +# Copyright 2015-2016, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without diff --git a/tools/jenkins/grpc_interop_python/Dockerfile b/tools/dockerfile/grpc_interop_python/Dockerfile index 047604b1b7..ef4432d260 100644 --- a/tools/jenkins/grpc_interop_python/Dockerfile +++ b/tools/dockerfile/grpc_interop_python/Dockerfile @@ -1,4 +1,4 @@ -# Copyright 2015, Google Inc. +# Copyright 2015-2016, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without diff --git a/tools/jenkins/grpc_interop_python/build_interop.sh b/tools/dockerfile/grpc_interop_python/build_interop.sh index 39c93677d8..203b012aed 100755 --- a/tools/jenkins/grpc_interop_python/build_interop.sh +++ b/tools/dockerfile/grpc_interop_python/build_interop.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2015, Google Inc. +# Copyright 2015-2016, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without diff --git a/tools/jenkins/grpc_interop_ruby/Dockerfile b/tools/dockerfile/grpc_interop_ruby/Dockerfile index ff201fa291..b3383afe79 100644 --- a/tools/jenkins/grpc_interop_ruby/Dockerfile +++ b/tools/dockerfile/grpc_interop_ruby/Dockerfile @@ -1,4 +1,4 @@ -# Copyright 2015, Google Inc. +# Copyright 2015-2016, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without diff --git a/tools/jenkins/grpc_interop_ruby/build_interop.sh b/tools/dockerfile/grpc_interop_ruby/build_interop.sh index c5023f5c1d..04288bf1aa 100755 --- a/tools/jenkins/grpc_interop_ruby/build_interop.sh +++ b/tools/dockerfile/grpc_interop_ruby/build_interop.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2015, Google Inc. +# Copyright 2015-2016, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without diff --git a/tools/jenkins/grpc_interop_cxx/Dockerfile b/tools/dockerfile/grpc_interop_stress_cxx/Dockerfile index 1fa1907533..58a8c32e34 100644 --- a/tools/jenkins/grpc_interop_cxx/Dockerfile +++ b/tools/dockerfile/grpc_interop_stress_cxx/Dockerfile @@ -1,4 +1,4 @@ -# Copyright 2015, Google Inc. +# Copyright 2015-2016, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without diff --git a/tools/jenkins/grpc_interop_stress_cxx/build_interop_stress.sh b/tools/dockerfile/grpc_interop_stress_cxx/build_interop_stress.sh index 6ed3ccb3fa..6ed3ccb3fa 100755 --- a/tools/jenkins/grpc_interop_stress_cxx/build_interop_stress.sh +++ b/tools/dockerfile/grpc_interop_stress_cxx/build_interop_stress.sh diff --git a/tools/jenkins/grpc_linuxbrew/Dockerfile b/tools/dockerfile/grpc_linuxbrew/Dockerfile index 848489e091..8c179da6c8 100644 --- a/tools/jenkins/grpc_linuxbrew/Dockerfile +++ b/tools/dockerfile/grpc_linuxbrew/Dockerfile @@ -1,4 +1,4 @@ -# Copyright 2015, Google Inc. +# Copyright 2015-2016, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without diff --git a/tools/dockerfile/grpc_sanity/Dockerfile b/tools/dockerfile/grpc_sanity/Dockerfile new file mode 100644 index 0000000000..6f7797d32a --- /dev/null +++ b/tools/dockerfile/grpc_sanity/Dockerfile @@ -0,0 +1,79 @@ +# Copyright 2015-2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# Dockerfile for running gRPC sanity tests + +FROM debian:jessie + +# Install Git and basic packages. +RUN apt-get update && apt-get install -y \ + autoconf \ + autotools-dev \ + build-essential \ + bzip2 \ + ccache \ + curl \ + gcc \ + gcc-multilib \ + git \ + golang \ + gyp \ + lcov \ + libc6 \ + libc6-dbg \ + libc6-dev \ + libgtest-dev \ + libtool \ + make \ + perl \ + strace \ + python-dev \ + python-setuptools \ + python-yaml \ + telnet \ + unzip \ + wget \ + zip && apt-get clean + +################## +# Sanity test dependencies +RUN apt-get update && apt-get install -y python-pip +RUN pip install simplejson mako + +################## +# Docker "inception". +# Note this is quite the ugly hack. +# This makes sure that the docker binary we inject has its dependencies. +RUN curl https://get.docker.com/ | sh +RUN apt-get remove --purge -y docker-engine + +RUN mkdir /var/local/jenkins + +# Define the default command. +CMD ["bash"] diff --git a/tools/jenkins/grpc_jenkins_slave/Dockerfile b/tools/dockerfile/grpc_tests_multilang_x64/Dockerfile index 48541b00ad..7785f3d99a 100644 --- a/tools/jenkins/grpc_jenkins_slave/Dockerfile +++ b/tools/dockerfile/grpc_tests_multilang_x64/Dockerfile @@ -27,8 +27,7 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# A work-in-progress Dockerfile that allows running gRPC test suites -# inside a docker container. +# Dockerfile for running gRPC test suites inside a docker container. FROM debian:jessie @@ -95,8 +94,6 @@ RUN apt-get update && apt-get -y dist-upgrade && apt-get install -y \ RUN cd /var/local && wget www.nuget.org/NuGet.exe ENV NUGET mono /var/local/NuGet.exe -# TODO(jtattermusch): add dependencies for other languages - ################## # Node dependencies @@ -154,18 +151,8 @@ RUN apt-get update && apt-get install -y \ ################## # Zookeeper dependencies - -# Install dependencies - RUN apt-get install -y libzookeeper-mt-dev -################## -# Docker "inception". -# Note this is quite the ugly hack. -# This makes sure that the docker binary we inject has its dependencies. -RUN curl https://get.docker.com/ | sh -RUN apt-get remove --purge -y docker-engine - RUN mkdir /var/local/jenkins # Define the default command. diff --git a/tools/jenkins/grpc_jenkins_slave_32bits/Dockerfile b/tools/dockerfile/grpc_tests_multilang_x86/Dockerfile index 2323f23387..c5cd45ea08 100644 --- a/tools/jenkins/grpc_jenkins_slave_32bits/Dockerfile +++ b/tools/dockerfile/grpc_tests_multilang_x86/Dockerfile @@ -27,8 +27,7 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# A work-in-progress Dockerfile that allows running gRPC test suites -# inside a docker container. +# Dockerfile for running gRPC test suites inside a docker container. FROM 32bit/debian:jessie @@ -95,8 +94,6 @@ RUN apt-get update && apt-get -y dist-upgrade && apt-get install -y \ RUN cd /var/local && wget www.nuget.org/NuGet.exe ENV NUGET mono /var/local/NuGet.exe -# TODO(jtattermusch): add dependencies for other languages - ################## # Node dependencies @@ -153,9 +150,6 @@ RUN apt-get update && apt-get install -y \ ################## # Zookeeper dependencies - -# Install dependencies - RUN apt-get install -y libzookeeper-mt-dev diff --git a/tools/doxygen/Doxyfile.c++ b/tools/doxygen/Doxyfile.c++ index 500d110ad0..ad0ad65cb1 100644 --- a/tools/doxygen/Doxyfile.c++ +++ b/tools/doxygen/Doxyfile.c++ @@ -40,7 +40,7 @@ PROJECT_NAME = "GRPC C++" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 0.12.0.0 +PROJECT_NUMBER = 0.13.0.0 # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal index ba1dec0d38..1d4b0cec7b 100644 --- a/tools/doxygen/Doxyfile.c++.internal +++ b/tools/doxygen/Doxyfile.c++.internal @@ -40,7 +40,7 @@ PROJECT_NAME = "GRPC C++" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 0.12.0.0 +PROJECT_NUMBER = 0.13.0.0 # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/tools/doxygen/Doxyfile.core b/tools/doxygen/Doxyfile.core index 3a1d097fb9..db3db4e200 100644 --- a/tools/doxygen/Doxyfile.core +++ b/tools/doxygen/Doxyfile.core @@ -40,7 +40,7 @@ PROJECT_NAME = "GRPC Core" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 0.12.0.0 +PROJECT_NUMBER = 0.13.0.0 # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index 5eea5a3846..699af45928 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -40,7 +40,7 @@ PROJECT_NAME = "GRPC Core" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 0.12.0.0 +PROJECT_NUMBER = 0.13.0.0 # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/tools/jenkins/build_docker_and_run_tests.sh b/tools/jenkins/build_docker_and_run_tests.sh index 562cfcb6a6..58163bb583 100755 --- a/tools/jenkins/build_docker_and_run_tests.sh +++ b/tools/jenkins/build_docker_and_run_tests.sh @@ -47,11 +47,15 @@ mkdir -p /tmp/xdg-cache-home # Create a local branch so the child Docker script won't complain git branch -f jenkins-docker -# Use image name based on Dockerfile checksum -DOCKER_IMAGE_NAME=grpc_jenkins_slave${docker_suffix}_`sha1sum tools/jenkins/grpc_jenkins_slave/Dockerfile | cut -f1 -d\ ` +# Inputs +# DOCKERFILE_DIR - Directory in which Dockerfile file is located. +# DOCKER_RUN_SCRIPT - Script to run under docker (relative to grpc repo root) + +# Use image name based on Dockerfile location checksum +DOCKER_IMAGE_NAME=$(basename $DOCKERFILE_DIR)_$(sha1sum $DOCKERFILE_DIR/Dockerfile | cut -f1 -d\ ) # Make sure docker image has been built. Should be instantaneous if so. -docker build -t $DOCKER_IMAGE_NAME tools/jenkins/grpc_jenkins_slave$docker_suffix +docker build -t $DOCKER_IMAGE_NAME $DOCKERFILE_DIR # Choose random name for docker container CONTAINER_NAME="run_tests_$(uuidgen)" @@ -76,7 +80,7 @@ docker run \ -w /var/local/git/grpc \ --name=$CONTAINER_NAME \ $DOCKER_IMAGE_NAME \ - bash -l /var/local/jenkins/grpc/tools/jenkins/docker_run_tests.sh || DOCKER_FAILED="true" + bash -l "/var/local/jenkins/grpc/$DOCKER_RUN_SCRIPT" || DOCKER_FAILED="true" if [ "$XML_REPORT" != "" ] then diff --git a/tools/jenkins/build_interop_image.sh b/tools/jenkins/build_interop_image.sh index 73937068a4..28d5f31d4f 100755 --- a/tools/jenkins/build_interop_image.sh +++ b/tools/jenkins/build_interop_image.sh @@ -71,10 +71,10 @@ then fi # Use image name based on Dockerfile checksum -BASE_IMAGE=${BASE_NAME}_base:`sha1sum tools/jenkins/$BASE_NAME/Dockerfile | cut -f1 -d\ ` +BASE_IMAGE=${BASE_NAME}_base:`sha1sum tools/dockerfile/$BASE_NAME/Dockerfile | cut -f1 -d\ ` # Make sure base docker image has been built. Should be instantaneous if so. -docker build -t $BASE_IMAGE --force-rm=true tools/jenkins/$BASE_NAME || exit $? +docker build -t $BASE_IMAGE --force-rm=true tools/dockerfile/$BASE_NAME || exit $? # Create a local branch so the child Docker script won't complain git branch -f jenkins-docker @@ -92,7 +92,7 @@ CONTAINER_NAME="build_${BASE_NAME}_$(uuidgen)" -v /tmp/ccache:/tmp/ccache \ --name=$CONTAINER_NAME \ $BASE_IMAGE \ - bash -l /var/local/jenkins/grpc/tools/jenkins/$BASE_NAME/build_interop.sh \ + bash -l /var/local/jenkins/grpc/tools/dockerfile/$BASE_NAME/build_interop.sh \ && docker commit $CONTAINER_NAME $INTEROP_IMAGE \ && echo "Successfully built image $INTEROP_IMAGE") EXITCODE=$? diff --git a/tools/jenkins/build_interop_stress_image.sh b/tools/jenkins/build_interop_stress_image.sh index 395eaa32d9..92f2dab5e3 100755 --- a/tools/jenkins/build_interop_stress_image.sh +++ b/tools/jenkins/build_interop_stress_image.sh @@ -55,10 +55,10 @@ then fi # Use image name based on Dockerfile checksum -BASE_IMAGE=${BASE_NAME}_base:`sha1sum tools/jenkins/$BASE_NAME/Dockerfile | cut -f1 -d\ ` +BASE_IMAGE=${BASE_NAME}_base:`sha1sum tools/dockerfile/$BASE_NAME/Dockerfile | cut -f1 -d\ ` # Make sure base docker image has been built. Should be instantaneous if so. -docker build -t $BASE_IMAGE --force-rm=true tools/jenkins/$BASE_NAME || exit $? +docker build -t $BASE_IMAGE --force-rm=true tools/dockerfile/$BASE_NAME || exit $? # Create a local branch so the child Docker script won't complain git branch -f jenkins-docker @@ -75,7 +75,7 @@ CONTAINER_NAME="build_${BASE_NAME}_$(uuidgen)" -v /tmp/ccache:/tmp/ccache \ --name=$CONTAINER_NAME \ $BASE_IMAGE \ - bash -l /var/local/jenkins/grpc/tools/jenkins/$BASE_NAME/build_interop_stress.sh \ + bash -l /var/local/jenkins/grpc/tools/dockerfile/$BASE_NAME/build_interop_stress.sh \ && docker commit $CONTAINER_NAME $INTEROP_IMAGE \ && echo "Successfully built image $INTEROP_IMAGE") EXITCODE=$? diff --git a/tools/jenkins/docker_run_tests.sh b/tools/jenkins/docker_run_tests.sh index 148a0f5e08..26e5585d2d 100755 --- a/tools/jenkins/docker_run_tests.sh +++ b/tools/jenkins/docker_run_tests.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2015, Google Inc. +# Copyright 2015-2016, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -43,8 +43,12 @@ chown `whoami` $XDG_CACHE_HOME mkdir -p /var/local/git git clone --recursive /var/local/jenkins/grpc /var/local/git/grpc -nvm use 0.12 -rvm use ruby-2.1 +nvm use 0.12 || true + +if [ -x "$(command -v rvm)" ] +then + rvm use ruby-2.1 +fi mkdir -p reports diff --git a/tools/jenkins/run_distribution.sh b/tools/jenkins/run_distribution.sh index 64c60f1502..446ce16703 100755 --- a/tools/jenkins/run_distribution.sh +++ b/tools/jenkins/run_distribution.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2015, Google Inc. +# Copyright 2015-2016, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -47,11 +47,11 @@ if [ "$platform" == "linux" ]; then if [ "$dist_channel" == "homebrew" ]; then - sha1=$(sha1sum tools/jenkins/grpc_linuxbrew/Dockerfile | cut -f1 -d\ ) + sha1=$(sha1sum tools/dockerfile/grpc_linuxbrew/Dockerfile | cut -f1 -d\ ) DOCKER_IMAGE_NAME=grpc_linuxbrew_$sha1 # build docker image, contains all pre-requisites - docker build -t $DOCKER_IMAGE_NAME tools/jenkins/grpc_linuxbrew + docker build -t $DOCKER_IMAGE_NAME tools/dockerfile/grpc_linuxbrew # run per-language homebrew installation script docker run --rm=true $DOCKER_IMAGE_NAME bash -l \ diff --git a/tools/jenkins/run_jenkins.sh b/tools/jenkins/run_jenkins.sh index 49b2fa55c2..4cb31e601a 100755 --- a/tools/jenkins/run_jenkins.sh +++ b/tools/jenkins/run_jenkins.sh @@ -39,51 +39,17 @@ # NOTE: No empty lines should appear in this file before igncr is set! set -ex -o igncr || set -ex -# Grabbing the machine's architecture -arch=`uname -m` - -case $platform in - i386) - arch="i386" - platform="linux" - docker_suffix=_32bits - ;; -esac - if [ "$platform" == "linux" ] then - echo "building $language on Linux" - - ./tools/run_tests/run_tests.py --use_docker -t -l $language -c $config -x report.xml -j 3 $@ || TESTS_FAILED="true" - -elif [ "$platform" == "windows" ] -then - echo "building $language on Windows" - - # Prevent msbuild from picking up "platform" env variable, which would break the build - unset platform - - python tools/run_tests/run_tests.py -t -l $language -c $config -x report.xml -j 3 $@ || TESTS_FAILED="true" - -elif [ "$platform" == "macos" ] -then - echo "building $language on MacOS" - - # Prevent msbuild from picking up "platform" env variable, which would break the build - unset platform - - ./tools/run_tests/run_tests.py -t -l $language -c $config -x report.xml -j 3 $@ || TESTS_FAILED="true" - + USE_DOCKER_MAYBE="--use_docker" elif [ "$platform" == "freebsd" ] then - echo "building $language on FreeBSD" + export MAKE=gmake +fi - MAKE=gmake ./tools/run_tests/run_tests.py -t -l $language -c $config -x report.xml -j 3 $@ || TESTS_FAILED="true" +unset platform # variable named 'platform' breaks the windows build -else - echo "Unknown platform $platform" - exit 1 -fi +python tools/run_tests/run_tests.py $USE_DOCKER_MAYBE -t -l $language -c $config -x report.xml -j 2 $@ || TESTS_FAILED="true" if [ ! -e reports/index.html ] then diff --git a/tools/jenkins/run_portability.sh b/tools/jenkins/run_portability.sh index afce4ad743..6f15da73f4 100755 --- a/tools/jenkins/run_portability.sh +++ b/tools/jenkins/run_portability.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Copyright 2015, Google Inc. +# Copyright 2015-2016, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -41,6 +41,14 @@ set -ex -o igncr || set -ex echo "building $scenario" +# If scenario has _bo suffix, add --build_only flag. +# Short suffix name had to been chosen due to path length limit on Windows. +if [ "$scenario" != "${scenario%_bo}" ] +then + scenario="${scenario%_bo}" + BUILD_ONLY_MAYBE="--build_only" +fi + parts=($(echo $scenario | tr '_' ' ')) # split scenario into parts curr_platform=${parts[0]} # variable named 'platform' breaks the windows build @@ -48,13 +56,10 @@ curr_arch=${parts[1]} curr_compiler=${parts[2]} config='dbg' -maybe_build_only='--build_only' -if [ "$curr_platform" == "windows" ] +if [ "$curr_platform" == "linux" ] then - win_arch="windows_${curr_arch}" - python tools/run_tests/run_tests.py -t -l $language -c $config --arch ${win_arch} --compiler ${curr_compiler} ${maybe_build_only} -x report.xml $@ -else - echo "Unsupported scenario." - exit 1 + USE_DOCKER_MAYBE="--use_docker" fi + +python tools/run_tests/run_tests.py $USE_DOCKER_MAYBE $BUILD_ONLY_MAYBE -t -l $language -c $config --arch ${curr_arch} --compiler ${curr_compiler} -x report.xml -j 3 $@ diff --git a/tools/run_tests/build_artifacts.py b/tools/run_tests/build_artifacts.py index 0fd5bc63f3..0337f1b3d7 100755 --- a/tools/run_tests/build_artifacts.py +++ b/tools/run_tests/build_artifacts.py @@ -129,18 +129,19 @@ class CSharpExtArtifact: '/p:PlatformToolset=v120', '/p:Platform=%s' % msbuild_platform], shell=True) - if self.platform == 'linux': - environ = {'CONFIG': 'opt'} - return create_docker_jobspec(self.name, - 'tools/jenkins/grpc_artifact_linux_%s' % self.arch, - 'tools/run_tests/build_artifact_csharp.sh') else: - environ = {'CONFIG': 'opt'} - if self.platform == 'macos': + environ = {'CONFIG': 'opt', + 'EMBED_OPENSSL': 'true', + 'EMBED_ZLIB': 'true'} + if self.platform == 'linux': + return create_docker_jobspec(self.name, + 'tools/dockerfile/grpc_artifact_linux_%s' % self.arch, + 'tools/run_tests/build_artifact_csharp.sh') + else: environ.update(macos_arch_env(self.arch)) - return create_jobspec(self.name, - ['tools/run_tests/build_artifact_csharp.sh'], - environ=environ) + return create_jobspec(self.name, + ['tools/run_tests/build_artifact_csharp.sh'], + environ=environ) def __str__(self): return self.name diff --git a/tools/run_tests/build_node.sh b/tools/run_tests/build_node.sh index faa7b624b8..8f2ab4413a 100755 --- a/tools/run_tests/build_node.sh +++ b/tools/run_tests/build_node.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright 2015, Google Inc. +# Copyright 2015-2016, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -36,4 +36,4 @@ CONFIG=${CONFIG:-opt} # change to grpc repo root cd $(dirname $0)/../.. -npm install --unsafe-perm +npm install --unsafe-perm --build-from-source diff --git a/tools/run_tests/configs.json b/tools/run_tests/configs.json index b21793d9cc..9d7b8a3c72 100644 --- a/tools/run_tests/configs.json +++ b/tools/run_tests/configs.json @@ -59,7 +59,7 @@ }, { "config": "msan", - "timeout_multiplier": 1.5 + "timeout_multiplier": 2 }, { "config": "mutrace" diff --git a/tools/run_tests/jobset.py b/tools/run_tests/jobset.py index beeb99c0ae..adf178bb3c 100755 --- a/tools/run_tests/jobset.py +++ b/tools/run_tests/jobset.py @@ -360,7 +360,7 @@ class Jobset(object): if self.cancelled(): return False current_cpu_cost = self.cpu_cost() if current_cpu_cost == 0: break - if current_cpu_cost + spec.cpu_cost < self._maxjobs: break + if current_cpu_cost + spec.cpu_cost <= self._maxjobs: break self.reap() if self.cancelled(): return False if spec.hash_targets: diff --git a/tools/run_tests/post_tests_ruby.sh b/tools/run_tests/post_tests_ruby.sh index 66a9fbc534..1a02e566c7 100755 --- a/tools/run_tests/post_tests_ruby.sh +++ b/tools/run_tests/post_tests_ruby.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2015, Google Inc. +# Copyright 2015-2016, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -43,4 +43,4 @@ genhtml $tmp2 --output-directory $out rm $tmp2 rm $tmp1 -cp -rv $root/src/ruby/coverage $root/reports/ruby +cp -rv $root/coverage $root/reports/ruby diff --git a/tools/run_tests/run_node.sh b/tools/run_tests/run_node.sh index fff579fab8..f93c9c30cb 100755 --- a/tools/run_tests/run_node.sh +++ b/tools/run_tests/run_node.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2015, Google Inc. +# Copyright 2015-2016, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -51,5 +51,5 @@ then echo '<html><head><meta http-equiv="refresh" content="0;URL=lcov-report/index.html"></head></html>' > \ ../reports/node_coverage/index.html else - JUNIT_REPORT_PATH=src/node/reports.xml JUNIT_REPORT_STACK=1 ./node_modules/.bin/mocha --reporter mocha-jenkins-reporter src/node/test || true + JUNIT_REPORT_PATH=src/node/reports.xml JUNIT_REPORT_STACK=1 ./node_modules/.bin/mocha --reporter mocha-jenkins-reporter src/node/test fi diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index f8b01021c8..fbc74014e9 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -151,9 +151,9 @@ class CLanguage(object): def make_targets(self, test_regex): if platform_string() != 'windows' and test_regex != '.*': # use the regex to minimize the number of things to build - return [target['name'] + return [os.path.basename(target['name']) for target in get_c_tests(False, self.test_lang) - if re.search(test_regex, target['name'])] + if re.search(test_regex, '/' + target['name'])] if platform_string() == 'windows': # don't build tools on windows just yet return ['buildtests_%s' % self.make_target] @@ -183,6 +183,9 @@ class CLanguage(object): def supports_multi_config(self): return True + def dockerfile_dir(self, config, arch): + return None + def __str__(self): return self.make_target @@ -215,6 +218,9 @@ class NodeLanguage(object): def supports_multi_config(self): return False + def dockerfile_dir(self, config, arch): + return None + def __str__(self): return 'node' @@ -246,6 +252,9 @@ class PhpLanguage(object): def supports_multi_config(self): return False + def dockerfile_dir(self, config, arch): + return None + def __str__(self): return 'php' @@ -299,6 +308,9 @@ class PythonLanguage(object): def supports_multi_config(self): return False + def dockerfile_dir(self, config, arch): + return None + def __str__(self): return 'python' @@ -330,6 +342,9 @@ class RubyLanguage(object): def supports_multi_config(self): return False + def dockerfile_dir(self, config, arch): + return None + def __str__(self): return 'ruby' @@ -412,6 +427,9 @@ class CSharpLanguage(object): def supports_multi_config(self): return False + def dockerfile_dir(self, config, arch): + return None + def __str__(self): return 'csharp' @@ -443,6 +461,9 @@ class ObjCLanguage(object): def supports_multi_config(self): return False + def dockerfile_dir(self, config, arch): + return None + def __str__(self): return 'objc' @@ -451,8 +472,10 @@ class Sanity(object): def test_specs(self, config, args): import yaml - with open('tools/run_tests/sanity_tests.yaml', 'r') as f: - return [config.job_spec([cmd['script']], None, timeout_seconds=None, environ={'TEST': 'true'}, cpu_cost=cmd.get('cpu_cost', 1)) + with open('tools/run_tests/sanity/sanity_tests.yaml', 'r') as f: + return [config.job_spec(cmd['script'].split(), None, + timeout_seconds=None, environ={'TEST': 'true'}, + cpu_cost=cmd.get('cpu_cost', 1)) for cmd in yaml.load(f)] def pre_build_steps(self): @@ -476,6 +499,9 @@ class Sanity(object): def supports_multi_config(self): return False + def dockerfile_dir(self, config, arch): + return 'tools/dockerfile/grpc_sanity' + def __str__(self): return 'sanity' @@ -506,6 +532,9 @@ class Build(object): def supports_multi_config(self): return True + def dockerfile_dir(self, config, arch): + return None + def __str__(self): return self.make_target @@ -538,15 +567,37 @@ _WINDOWS_CONFIG = { def _windows_arch_option(arch): """Returns msbuild cmdline option for selected architecture.""" - if arch == 'default' or arch == 'windows_x86': + if arch == 'default' or arch == 'x86': return '/p:Platform=Win32' - elif arch == 'windows_x64': + elif arch == 'x64': return '/p:Platform=x64' else: - print 'Architecture %s not supported on current platform.' % arch + print 'Architecture %s not supported.' % arch sys.exit(1) - + +def _check_arch_option(arch): + """Checks that architecture option is valid.""" + if platform_string() == 'windows': + _windows_arch_option(arch) + elif platform_string() == 'linux': + # On linux, we need to be running under docker with the right architecture. + runtime_arch = platform.architecture()[0] + if arch == 'default': + return + elif runtime_arch == '64bit' and arch == 'x64': + return + elif runtime_arch == '32bit' and arch == 'x86': + return + else: + print 'Architecture %s does not match current runtime architecture.' % arch + sys.exit(1) + else: + if args.arch != 'default': + print 'Architecture %s not supported on current platform.' % args.arch + sys.exit(1) + + def _windows_build_bat(compiler): """Returns name of build.bat for selected compiler.""" if compiler == 'default' or compiler == 'vs2013': @@ -558,8 +609,8 @@ def _windows_build_bat(compiler): else: print 'Compiler %s not supported.' % compiler sys.exit(1) - - + + def _windows_toolset_option(compiler): """Returns msbuild PlatformToolset for selected compiler.""" if compiler == 'default' or compiler == 'vs2013': @@ -571,7 +622,21 @@ def _windows_toolset_option(compiler): else: print 'Compiler %s not supported.' % compiler sys.exit(1) - + + +def _get_dockerfile_dir(language, cfg, arch): + """Returns dockerfile to use""" + custom = language.dockerfile_dir(cfg, arch) + if custom: + return custom + else: + if arch == 'default' or arch == 'x64': + return 'tools/dockerfile/grpc_tests_multilang_x64' + elif arch == 'x86': + return 'tools/dockerfile/grpc_tests_multilang_x86' + else: + print 'Architecture %s not supported with current settings.' % arch + sys.exit(1) def runs_per_test_type(arg_str): """Auxilary function to parse the "runs_per_test" flag. @@ -638,7 +703,7 @@ argp.add_argument('--allow_flakes', const=True, help='Allow flaky tests to show as passing (re-runs failed tests up to five times)') argp.add_argument('--arch', - choices=['default', 'windows_x86', 'windows_x64'], + choices=['default', 'x86', 'x64'], default='default', help='Selects architecture to target. For some platforms "default" is the only supported choice.') argp.add_argument('--compiler', @@ -662,36 +727,6 @@ args = argp.parse_args() jobset.measure_cpu_costs = args.measure_cpu_costs -if args.use_docker: - if not args.travis: - print 'Seen --use_docker flag, will run tests under docker.' - print - print 'IMPORTANT: The changes you are testing need to be locally committed' - print 'because only the committed changes in the current branch will be' - print 'copied to the docker environment.' - time.sleep(5) - - child_argv = [ arg for arg in sys.argv if not arg == '--use_docker' ] - run_tests_cmd = 'tools/run_tests/run_tests.py %s' % ' '.join(child_argv[1:]) - - # TODO(jtattermusch): revisit if we need special handling for arch here - # set arch command prefix in case we are working with different arch. - arch_env = os.getenv('arch') - if arch_env: - run_test_cmd = 'arch %s %s' % (arch_env, run_test_cmd) - - env = os.environ.copy() - env['RUN_TESTS_COMMAND'] = run_tests_cmd - if args.xml_report: - env['XML_REPORT'] = args.xml_report - if not args.travis: - env['TTY_FLAG'] = '-t' # enables Ctrl-C when not on Jenkins. - - subprocess.check_call(['tools/jenkins/build_docker_and_run_tests.sh'], - shell=True, - env=env) - sys.exit(0) - # update submodules if necessary need_to_regenerate_projects = False for spec in args.update_submodules: @@ -755,16 +790,46 @@ if any(language.make_options() for language in languages): else: language_make_options = next(iter(languages)).make_options() -if platform_string() != 'windows': - if args.arch != 'default': - print 'Architecture %s not supported on current platform.' % args.arch - sys.exit(1) - if args.compiler != 'default': +if len(languages) != 1 or len(build_configs) != 1: + print 'Multi-language and multi-config testing is not supported.' + sys.exit(1) + +if args.use_docker: + if not args.travis: + print 'Seen --use_docker flag, will run tests under docker.' + print + print 'IMPORTANT: The changes you are testing need to be locally committed' + print 'because only the committed changes in the current branch will be' + print 'copied to the docker environment.' + time.sleep(5) + + child_argv = [ arg for arg in sys.argv if not arg == '--use_docker' ] + run_tests_cmd = 'python tools/run_tests/run_tests.py %s' % ' '.join(child_argv[1:]) + + env = os.environ.copy() + env['RUN_TESTS_COMMAND'] = run_tests_cmd + env['DOCKERFILE_DIR'] = _get_dockerfile_dir(next(iter(languages)), + next(iter(build_configs)), + args.arch) + env['DOCKER_RUN_SCRIPT'] = 'tools/jenkins/docker_run_tests.sh' + if args.xml_report: + env['XML_REPORT'] = args.xml_report + if not args.travis: + env['TTY_FLAG'] = '-t' # enables Ctrl-C when not on Jenkins. + + subprocess.check_call(['tools/jenkins/build_docker_and_run_tests.sh'], + shell=True, + env=env) + sys.exit(0) + +if platform_string() != 'windows' and args.compiler != 'default': print 'Compiler %s not supported on current platform.' % args.compiler sys.exit(1) -if platform_string() == 'windows': - def make_jobspec(cfg, targets, makefile='Makefile'): +_check_arch_option(args.arch) + +def make_jobspec(cfg, targets, makefile='Makefile'): + if platform_string() == 'windows': extra_args = [] # better do parallel compilation # empirically /m:2 gives the best performance/price and should prevent @@ -782,8 +847,7 @@ if platform_string() == 'windows': language_make_options, shell=True, timeout_seconds=None) for target in targets] -else: - def make_jobspec(cfg, targets, makefile='Makefile'): + else: if targets: return [jobset.JobSpec([os.getenv('MAKE', 'make'), '-f', makefile, @@ -796,6 +860,7 @@ else: timeout_seconds=None)] else: return [] + make_targets = {} for l in languages: makefile = l.makefile_name() @@ -994,13 +1059,15 @@ def _build_and_run( check_cancelled, newline_on_success, cache, xml_report=None, build_only=False): """Do one pass of building & running tests.""" # build latest sequentially - num_failures, _ = jobset.run( + num_failures, resultset = jobset.run( build_steps, maxjobs=1, stop_on_failure=True, newline_on_success=newline_on_success, travis=args.travis) if num_failures: return [BuildAndRunError.BUILD] if build_only: + if xml_report: + report_utils.render_junit_xml_report(resultset, xml_report) return [] # start antagonists diff --git a/tools/run_tests/check_cache_mk.sh b/tools/run_tests/sanity/check_cache_mk.sh index b738d6a965..b738d6a965 100755 --- a/tools/run_tests/check_cache_mk.sh +++ b/tools/run_tests/sanity/check_cache_mk.sh diff --git a/tools/run_tests/check_sources_and_headers.py b/tools/run_tests/sanity/check_sources_and_headers.py index 50574f44b0..3974af0032 100755 --- a/tools/run_tests/check_sources_and_headers.py +++ b/tools/run_tests/sanity/check_sources_and_headers.py @@ -33,9 +33,9 @@ import os import re import sys -root = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '../..')) +root = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '../../..')) with open(os.path.join(root, 'tools', 'run_tests', 'sources_and_headers.json')) as f: - js = json.loads(f.read()) + js = json.loads(f.read()) re_inc1 = re.compile(r'^#\s*include\s*"([^"]*)"') assert re_inc1.match('#include "foo"').group(1) == 'foo' @@ -43,41 +43,41 @@ re_inc2 = re.compile(r'^#\s*include\s*<((grpc|grpc\+\+)/[^"]*)>') assert re_inc2.match('#include <grpc++/foo>').group(1) == 'grpc++/foo' def get_target(name): - for target in js: - if target['name'] == name: - return target - assert False, 'no target %s' % name + for target in js: + if target['name'] == name: + return target + assert False, 'no target %s' % name def target_has_header(target, name): -# print target['name'], name - if name in target['headers']: - return True - for dep in target['deps']: - if target_has_header(get_target(dep), name): - return True - if name == 'src/core/profiling/stap_probes.h': - return True - return False + # print target['name'], name + if name in target['headers']: + return True + for dep in target['deps']: + if target_has_header(get_target(dep), name): + return True + if name == 'src/core/profiling/stap_probes.h': + return True + return False errors = 0 for target in js: - for fn in target['src']: - with open(os.path.join(root, fn)) as f: - src = f.read().splitlines() - for line in src: - m = re_inc1.match(line) - if m: - if not target_has_header(target, m.group(1)): - print ( - 'target %s (%s) does not name header %s as a dependency' % ( - target['name'], fn, m.group(1))) - errors += 1 - m = re_inc2.match(line) - if m: - if not target_has_header(target, 'include/' + m.group(1)): - print ( - 'target %s (%s) does not name header %s as a dependency' % ( - target['name'], fn, m.group(1))) - errors += 1 + for fn in target['src']: + with open(os.path.join(root, fn)) as f: + src = f.read().splitlines() + for line in src: + m = re_inc1.match(line) + if m: + if not target_has_header(target, m.group(1)): + print ( + 'target %s (%s) does not name header %s as a dependency' % ( + target['name'], fn, m.group(1))) + errors += 1 + m = re_inc2.match(line) + if m: + if not target_has_header(target, 'include/' + m.group(1)): + print ( + 'target %s (%s) does not name header %s as a dependency' % ( + target['name'], fn, m.group(1))) + errors += 1 assert errors == 0 diff --git a/tools/run_tests/check_submodules.sh b/tools/run_tests/sanity/check_submodules.sh index b4ca4fa3ce..f49230e49a 100755 --- a/tools/run_tests/check_submodules.sh +++ b/tools/run_tests/sanity/check_submodules.sh @@ -34,7 +34,7 @@ set -e export TEST=true -cd `dirname $0`/../.. +cd `dirname $0`/../../.. submodules=`mktemp /tmp/submXXXXXX` want_submodules=`mktemp /tmp/submXXXXXX` diff --git a/tools/run_tests/sanity/sanity_tests.yaml b/tools/run_tests/sanity/sanity_tests.yaml new file mode 100644 index 0000000000..809e6ce645 --- /dev/null +++ b/tools/run_tests/sanity/sanity_tests.yaml @@ -0,0 +1,9 @@ +# a set of tests that are run in parallel for sanity tests +- script: tools/run_tests/sanity/check_sources_and_headers.py +- script: tools/run_tests/sanity/check_submodules.sh +- script: tools/run_tests/sanity/check_cache_mk.sh +- script: tools/buildgen/generate_projects.sh -j 3 + cpu_cost: 3 +- script: tools/distrib/check_copyright.py +- script: tools/distrib/clang_format_code.sh +- script: tools/distrib/check_trailing_newlines.sh diff --git a/tools/run_tests/sanity_tests.yaml b/tools/run_tests/sanity_tests.yaml deleted file mode 100644 index 160acde565..0000000000 --- a/tools/run_tests/sanity_tests.yaml +++ /dev/null @@ -1,9 +0,0 @@ -# a set of tests that are run in parallel for sanity tests -- script: tools/run_tests/check_sources_and_headers.py -- script: tools/run_tests/check_submodules.sh -- script: tools/run_tests/check_cache_mk.sh -- script: tools/buildgen/generate_projects.sh - cpu_cost: 100 -- script: tools/distrib/check_copyright.py -- script: tools/distrib/clang_format_code.sh -- script: tools/distrib/check_trailing_newlines.sh diff --git a/tools/run_tests/tests.json b/tools/run_tests/tests.json index 8c76b3f134..1604c8bbde 100644 --- a/tools/run_tests/tests.json +++ b/tools/run_tests/tests.json @@ -2433,6 +2433,7 @@ "windows" ], "cpu_cost": 1.0, + "defaults": "boringssl", "exclude_configs": [ "asan" ], @@ -2456,6 +2457,7 @@ "windows" ], "cpu_cost": 1.0, + "defaults": "boringssl", "exclude_configs": [ "asan" ], @@ -2479,6 +2481,7 @@ "windows" ], "cpu_cost": 1.0, + "defaults": "boringssl", "exclude_configs": [ "asan" ], @@ -2502,6 +2505,7 @@ "windows" ], "cpu_cost": 1.0, + "defaults": "boringssl", "exclude_configs": [ "asan" ], @@ -2525,6 +2529,7 @@ "windows" ], "cpu_cost": 1.0, + "defaults": "boringssl", "exclude_configs": [ "asan" ], @@ -2551,6 +2556,7 @@ "windows" ], "cpu_cost": 1.0, + "defaults": "boringssl", "exclude_configs": [ "asan" ], @@ -2577,6 +2583,7 @@ "windows" ], "cpu_cost": 1.0, + "defaults": "boringssl", "exclude_configs": [ "asan" ], @@ -2603,6 +2610,7 @@ "windows" ], "cpu_cost": 1.0, + "defaults": "boringssl", "exclude_configs": [ "asan" ], @@ -2629,6 +2637,7 @@ "windows" ], "cpu_cost": 1.0, + "defaults": "boringssl", "exclude_configs": [ "asan" ], @@ -2655,6 +2664,7 @@ "windows" ], "cpu_cost": 1.0, + "defaults": "boringssl", "exclude_configs": [ "asan" ], @@ -2681,6 +2691,7 @@ "windows" ], "cpu_cost": 1.0, + "defaults": "boringssl", "exclude_configs": [ "asan" ], @@ -2707,6 +2718,7 @@ "windows" ], "cpu_cost": 1.0, + "defaults": "boringssl", "exclude_configs": [ "asan" ], @@ -2733,6 +2745,7 @@ "windows" ], "cpu_cost": 1.0, + "defaults": "boringssl", "exclude_configs": [ "asan" ], @@ -2759,6 +2772,7 @@ "windows" ], "cpu_cost": 1.0, + "defaults": "boringssl", "exclude_configs": [ "asan" ], @@ -2785,6 +2799,7 @@ "windows" ], "cpu_cost": 1.0, + "defaults": "boringssl", "exclude_configs": [ "asan" ], @@ -2811,6 +2826,7 @@ "windows" ], "cpu_cost": 1.0, + "defaults": "boringssl", "exclude_configs": [ "asan" ], @@ -2837,6 +2853,7 @@ "windows" ], "cpu_cost": 1.0, + "defaults": "boringssl", "exclude_configs": [ "asan" ], @@ -2863,6 +2880,7 @@ "windows" ], "cpu_cost": 1.0, + "defaults": "boringssl", "exclude_configs": [ "asan" ], @@ -2889,6 +2907,7 @@ "windows" ], "cpu_cost": 1.0, + "defaults": "boringssl", "exclude_configs": [ "asan" ], @@ -2915,6 +2934,7 @@ "windows" ], "cpu_cost": 1.0, + "defaults": "boringssl", "exclude_configs": [ "asan" ], @@ -2941,6 +2961,7 @@ "windows" ], "cpu_cost": 1.0, + "defaults": "boringssl", "exclude_configs": [ "asan" ], @@ -2967,6 +2988,7 @@ "windows" ], "cpu_cost": 1.0, + "defaults": "boringssl", "exclude_configs": [ "asan" ], @@ -2993,6 +3015,7 @@ "windows" ], "cpu_cost": 1.0, + "defaults": "boringssl", "exclude_configs": [ "asan" ], @@ -3019,6 +3042,7 @@ "windows" ], "cpu_cost": 1.0, + "defaults": "boringssl", "exclude_configs": [ "asan" ], @@ -3045,6 +3069,7 @@ "windows" ], "cpu_cost": 1.0, + "defaults": "boringssl", "exclude_configs": [ "asan" ], @@ -3071,6 +3096,7 @@ "windows" ], "cpu_cost": 1.0, + "defaults": "boringssl", "exclude_configs": [ "asan" ], @@ -3097,6 +3123,7 @@ "windows" ], "cpu_cost": 1.0, + "defaults": "boringssl", "exclude_configs": [ "asan" ], @@ -3123,6 +3150,7 @@ "windows" ], "cpu_cost": 1.0, + "defaults": "boringssl", "exclude_configs": [ "asan" ], @@ -3149,6 +3177,7 @@ "windows" ], "cpu_cost": 1.0, + "defaults": "boringssl", "exclude_configs": [ "asan" ], @@ -3174,6 +3203,7 @@ "windows" ], "cpu_cost": 1.0, + "defaults": "boringssl", "exclude_configs": [ "asan" ], @@ -3197,6 +3227,7 @@ "windows" ], "cpu_cost": 1.0, + "defaults": "boringssl", "exclude_configs": [ "asan" ], @@ -3220,6 +3251,7 @@ "windows" ], "cpu_cost": 1.0, + "defaults": "boringssl", "exclude_configs": [ "asan" ], @@ -3245,6 +3277,7 @@ "windows" ], "cpu_cost": 1.0, + "defaults": "boringssl", "exclude_configs": [ "asan" ], @@ -3268,6 +3301,7 @@ "windows" ], "cpu_cost": 1.0, + "defaults": "boringssl", "exclude_configs": [ "asan" ], @@ -3291,6 +3325,7 @@ "windows" ], "cpu_cost": 1.0, + "defaults": "boringssl", "exclude_configs": [ "asan" ], @@ -3314,6 +3349,7 @@ "windows" ], "cpu_cost": 1.0, + "defaults": "boringssl", "exclude_configs": [ "asan" ], @@ -3337,6 +3373,7 @@ "windows" ], "cpu_cost": 1.0, + "defaults": "boringssl", "exclude_configs": [ "asan" ], @@ -3360,6 +3397,7 @@ "windows" ], "cpu_cost": 1.0, + "defaults": "boringssl", "exclude_configs": [ "asan" ], @@ -3383,6 +3421,7 @@ "windows" ], "cpu_cost": 1.0, + "defaults": "boringssl", "exclude_configs": [ "asan" ], @@ -3406,6 +3445,7 @@ "windows" ], "cpu_cost": 1.0, + "defaults": "boringssl", "exclude_configs": [ "asan" ], @@ -3429,6 +3469,7 @@ "windows" ], "cpu_cost": 1.0, + "defaults": "boringssl", "exclude_configs": [ "asan" ], @@ -3452,6 +3493,7 @@ "windows" ], "cpu_cost": 1.0, + "defaults": "boringssl", "exclude_configs": [ "asan" ], @@ -3477,6 +3519,7 @@ "windows" ], "cpu_cost": 1.0, + "defaults": "boringssl", "exclude_configs": [ "asan" ], @@ -3500,6 +3543,7 @@ "windows" ], "cpu_cost": 1.0, + "defaults": "boringssl", "exclude_configs": [ "asan" ], @@ -3523,6 +3567,7 @@ "windows" ], "cpu_cost": 1.0, + "defaults": "boringssl", "exclude_configs": [ "asan" ], @@ -3548,6 +3593,7 @@ "windows" ], "cpu_cost": 1.0, + "defaults": "boringssl", "exclude_configs": [ "asan" ], @@ -3571,6 +3617,7 @@ "windows" ], "cpu_cost": 1.0, + "defaults": "boringssl", "exclude_configs": [ "asan" ], @@ -3594,6 +3641,7 @@ "windows" ], "cpu_cost": 1.0, + "defaults": "boringssl", "exclude_configs": [ "asan" ], @@ -3617,6 +3665,7 @@ "windows" ], "cpu_cost": 1.0, + "defaults": "boringssl", "exclude_configs": [ "asan" ], @@ -3640,6 +3689,7 @@ "windows" ], "cpu_cost": 1.0, + "defaults": "boringssl", "exclude_configs": [ "asan" ], @@ -3665,6 +3715,7 @@ "windows" ], "cpu_cost": 1.0, + "defaults": "boringssl", "exclude_configs": [ "asan" ], @@ -3688,6 +3739,7 @@ "windows" ], "cpu_cost": 1.0, + "defaults": "boringssl", "exclude_configs": [ "asan" ], @@ -3711,6 +3763,7 @@ "windows" ], "cpu_cost": 1.0, + "defaults": "boringssl", "exclude_configs": [ "asan" ], @@ -3734,6 +3787,7 @@ "windows" ], "cpu_cost": 1.0, + "defaults": "boringssl", "exclude_configs": [ "asan" ], @@ -3757,6 +3811,7 @@ "windows" ], "cpu_cost": 1.0, + "defaults": "boringssl", "exclude_configs": [ "asan" ], @@ -3780,6 +3835,7 @@ "windows" ], "cpu_cost": 1.0, + "defaults": "boringssl", "exclude_configs": [ "asan" ], @@ -3803,6 +3859,7 @@ "windows" ], "cpu_cost": 1.0, + "defaults": "boringssl", "exclude_configs": [ "asan" ], @@ -3826,6 +3883,7 @@ "windows" ], "cpu_cost": 1.0, + "defaults": "boringssl", "exclude_configs": [ "asan" ], @@ -3849,6 +3907,7 @@ "windows" ], "cpu_cost": 1.0, + "defaults": "boringssl", "exclude_configs": [ "asan" ], diff --git a/tools/tsan_suppressions.txt b/tools/tsan_suppressions.txt index 65e7e2ec4b..09e68cd233 100644 --- a/tools/tsan_suppressions.txt +++ b/tools/tsan_suppressions.txt @@ -5,4 +5,4 @@ race:cleanse_ctr # https://www.mail-archive.com/openssl-dev@openssl.org/msg09019.html race:ssleay_rand_add race:ssleay_rand_bytes - +race:__sleep_for |