diff options
author | Craig Tiller <ctiller@google.com> | 2016-01-27 13:52:56 -0800 |
---|---|---|
committer | Craig Tiller <ctiller@google.com> | 2016-01-27 13:52:56 -0800 |
commit | 324426b654468ff7b6abd036e95f76238a4f8e53 (patch) | |
tree | 73b811d4d11a0bc1b41f529ee665ca2dc70517b1 /tools | |
parent | b0f275e4af3a18882110994713368eb288594b66 (diff) | |
parent | 26e4f5b1b6478809edc576646d3b4cd24562d549 (diff) |
Merge github.com:grpc/grpc into fix-c++
Diffstat (limited to 'tools')
-rwxr-xr-x | tools/buildgen/generate_projects.py | 3 | ||||
-rw-r--r-- | tools/dockerfile/grpc_artifact_linux_x64/Dockerfile (renamed from tools/jenkins/grpc_artifact_linux_x64/Dockerfile) | 0 | ||||
-rw-r--r-- | tools/dockerfile/grpc_artifact_linux_x86/Dockerfile (renamed from tools/jenkins/grpc_artifact_linux_x86/Dockerfile) | 0 | ||||
-rw-r--r-- | tools/dockerfile/grpc_interop_csharp/Dockerfile (renamed from tools/jenkins/grpc_interop_csharp/Dockerfile) | 2 | ||||
-rwxr-xr-x | tools/dockerfile/grpc_interop_csharp/build_interop.sh (renamed from tools/jenkins/grpc_interop_csharp/build_interop.sh) | 0 | ||||
-rw-r--r-- | tools/dockerfile/grpc_interop_cxx/Dockerfile (renamed from tools/jenkins/grpc_interop_stress_cxx/Dockerfile) | 0 | ||||
-rwxr-xr-x | tools/dockerfile/grpc_interop_cxx/build_interop.sh (renamed from tools/jenkins/grpc_interop_cxx/build_interop.sh) | 2 | ||||
-rw-r--r-- | tools/dockerfile/grpc_interop_go/Dockerfile (renamed from tools/jenkins/grpc_interop_http2/Dockerfile) | 2 | ||||
-rwxr-xr-x | tools/dockerfile/grpc_interop_go/build_interop.sh (renamed from tools/jenkins/grpc_interop_go/build_interop.sh) | 2 | ||||
-rw-r--r-- | tools/dockerfile/grpc_interop_http2/Dockerfile (renamed from tools/jenkins/grpc_interop_go/Dockerfile) | 2 | ||||
-rwxr-xr-x | tools/dockerfile/grpc_interop_http2/build_interop.sh (renamed from tools/jenkins/grpc_interop_http2/build_interop.sh) | 2 | ||||
-rw-r--r-- | tools/dockerfile/grpc_interop_java/Dockerfile (renamed from tools/jenkins/grpc_interop_java/Dockerfile) | 2 | ||||
-rwxr-xr-x | tools/dockerfile/grpc_interop_java/build_interop.sh (renamed from tools/jenkins/grpc_interop_java/build_interop.sh) | 2 | ||||
-rw-r--r-- | tools/dockerfile/grpc_interop_node/Dockerfile (renamed from tools/jenkins/grpc_interop_node/Dockerfile) | 2 | ||||
-rwxr-xr-x | tools/dockerfile/grpc_interop_node/build_interop.sh (renamed from tools/jenkins/grpc_interop_node/build_interop.sh) | 4 | ||||
-rw-r--r-- | tools/dockerfile/grpc_interop_php/Dockerfile (renamed from tools/jenkins/grpc_interop_php/Dockerfile) | 2 | ||||
-rwxr-xr-x | tools/dockerfile/grpc_interop_php/build_interop.sh (renamed from tools/jenkins/grpc_interop_php/build_interop.sh) | 2 | ||||
-rw-r--r-- | tools/dockerfile/grpc_interop_python/Dockerfile (renamed from tools/jenkins/grpc_interop_python/Dockerfile) | 2 | ||||
-rwxr-xr-x | tools/dockerfile/grpc_interop_python/build_interop.sh (renamed from tools/jenkins/grpc_interop_python/build_interop.sh) | 2 | ||||
-rw-r--r-- | tools/dockerfile/grpc_interop_ruby/Dockerfile (renamed from tools/jenkins/grpc_interop_ruby/Dockerfile) | 2 | ||||
-rwxr-xr-x | tools/dockerfile/grpc_interop_ruby/build_interop.sh (renamed from tools/jenkins/grpc_interop_ruby/build_interop.sh) | 2 | ||||
-rw-r--r-- | tools/dockerfile/grpc_interop_stress_cxx/Dockerfile (renamed from tools/jenkins/grpc_interop_cxx/Dockerfile) | 2 | ||||
-rwxr-xr-x | tools/dockerfile/grpc_interop_stress_cxx/build_interop_stress.sh (renamed from tools/jenkins/grpc_interop_stress_cxx/build_interop_stress.sh) | 0 | ||||
-rw-r--r-- | tools/dockerfile/grpc_jenkins_slave_x64/Dockerfile (renamed from tools/jenkins/grpc_jenkins_slave/Dockerfile) | 0 | ||||
-rw-r--r-- | tools/dockerfile/grpc_jenkins_slave_x86/Dockerfile (renamed from tools/jenkins/grpc_jenkins_slave_32bits/Dockerfile) | 0 | ||||
-rw-r--r-- | tools/dockerfile/grpc_linuxbrew/Dockerfile (renamed from tools/jenkins/grpc_linuxbrew/Dockerfile) | 2 | ||||
-rwxr-xr-x | tools/jenkins/build_docker_and_run_tests.sh | 12 | ||||
-rwxr-xr-x | tools/jenkins/build_interop_image.sh | 6 | ||||
-rwxr-xr-x | tools/jenkins/build_interop_stress_image.sh | 6 | ||||
-rwxr-xr-x | tools/jenkins/run_distribution.sh | 6 | ||||
-rwxr-xr-x | tools/jenkins/run_jenkins.sh | 44 | ||||
-rwxr-xr-x | tools/jenkins/run_portability.sh | 5 | ||||
-rwxr-xr-x | tools/run_tests/build_artifacts.py | 2 | ||||
-rwxr-xr-x | tools/run_tests/build_node.sh | 4 | ||||
-rwxr-xr-x | tools/run_tests/run_node.sh | 4 | ||||
-rwxr-xr-x | tools/run_tests/run_tests.py | 126 | ||||
-rw-r--r-- | tools/run_tests/sanity_tests.yaml | 4 | ||||
-rw-r--r-- | tools/tsan_suppressions.txt | 2 |
38 files changed, 131 insertions, 131 deletions
diff --git a/tools/buildgen/generate_projects.py b/tools/buildgen/generate_projects.py index 083a97874d..965dd292af 100755 --- a/tools/buildgen/generate_projects.py +++ b/tools/buildgen/generate_projects.py @@ -47,6 +47,7 @@ os.chdir(os.path.join(os.path.dirname(sys.argv[0]), '..', '..')) argp = argparse.ArgumentParser() argp.add_argument('json', nargs='+') argp.add_argument('--templates', nargs='+', default=[]) +argp.add_argument('--jobs', '-j', default=multiprocessing.cpu_count(), type=int) args = argp.parse_args() json = args.json @@ -87,7 +88,7 @@ for template in templates: cmd.append(root + '/' + f) jobs.append(jobset.JobSpec(cmd, shortname=out, timeout_seconds=None)) -jobset.run(jobs, maxjobs=multiprocessing.cpu_count()) +jobset.run(jobs, maxjobs=args.jobs) if test is not None: for s, g in test.iteritems(): diff --git a/tools/jenkins/grpc_artifact_linux_x64/Dockerfile b/tools/dockerfile/grpc_artifact_linux_x64/Dockerfile index de40247a73..de40247a73 100644 --- a/tools/jenkins/grpc_artifact_linux_x64/Dockerfile +++ b/tools/dockerfile/grpc_artifact_linux_x64/Dockerfile diff --git a/tools/jenkins/grpc_artifact_linux_x86/Dockerfile b/tools/dockerfile/grpc_artifact_linux_x86/Dockerfile index 774b4523a1..774b4523a1 100644 --- a/tools/jenkins/grpc_artifact_linux_x86/Dockerfile +++ b/tools/dockerfile/grpc_artifact_linux_x86/Dockerfile diff --git a/tools/jenkins/grpc_interop_csharp/Dockerfile b/tools/dockerfile/grpc_interop_csharp/Dockerfile index 3789cd3203..c3c79487f7 100644 --- a/tools/jenkins/grpc_interop_csharp/Dockerfile +++ b/tools/dockerfile/grpc_interop_csharp/Dockerfile @@ -1,4 +1,4 @@ -# Copyright 2015, Google Inc. +# Copyright 2015-2016, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without diff --git a/tools/jenkins/grpc_interop_csharp/build_interop.sh b/tools/dockerfile/grpc_interop_csharp/build_interop.sh index e87fe2f692..e87fe2f692 100755 --- a/tools/jenkins/grpc_interop_csharp/build_interop.sh +++ b/tools/dockerfile/grpc_interop_csharp/build_interop.sh diff --git a/tools/jenkins/grpc_interop_stress_cxx/Dockerfile b/tools/dockerfile/grpc_interop_cxx/Dockerfile index 58a8c32e34..58a8c32e34 100644 --- a/tools/jenkins/grpc_interop_stress_cxx/Dockerfile +++ b/tools/dockerfile/grpc_interop_cxx/Dockerfile diff --git a/tools/jenkins/grpc_interop_cxx/build_interop.sh b/tools/dockerfile/grpc_interop_cxx/build_interop.sh index 1c0828d23a..903c31b3e7 100755 --- a/tools/jenkins/grpc_interop_cxx/build_interop.sh +++ b/tools/dockerfile/grpc_interop_cxx/build_interop.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2015, Google Inc. +# Copyright 2015-2016, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without diff --git a/tools/jenkins/grpc_interop_http2/Dockerfile b/tools/dockerfile/grpc_interop_go/Dockerfile index bb60f09f24..f83e18c8a3 100644 --- a/tools/jenkins/grpc_interop_http2/Dockerfile +++ b/tools/dockerfile/grpc_interop_go/Dockerfile @@ -1,4 +1,4 @@ -# Copyright 2015, Google Inc. +# Copyright 2015-2016, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without diff --git a/tools/jenkins/grpc_interop_go/build_interop.sh b/tools/dockerfile/grpc_interop_go/build_interop.sh index e891549342..224b0040c0 100755 --- a/tools/jenkins/grpc_interop_go/build_interop.sh +++ b/tools/dockerfile/grpc_interop_go/build_interop.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2015, Google Inc. +# Copyright 2015-2016, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without diff --git a/tools/jenkins/grpc_interop_go/Dockerfile b/tools/dockerfile/grpc_interop_http2/Dockerfile index bb60f09f24..f83e18c8a3 100644 --- a/tools/jenkins/grpc_interop_go/Dockerfile +++ b/tools/dockerfile/grpc_interop_http2/Dockerfile @@ -1,4 +1,4 @@ -# Copyright 2015, Google Inc. +# Copyright 2015-2016, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without diff --git a/tools/jenkins/grpc_interop_http2/build_interop.sh b/tools/dockerfile/grpc_interop_http2/build_interop.sh index 46ddaf929a..5f701e34c0 100755 --- a/tools/jenkins/grpc_interop_http2/build_interop.sh +++ b/tools/dockerfile/grpc_interop_http2/build_interop.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2015, Google Inc. +# Copyright 2015-2016, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without diff --git a/tools/jenkins/grpc_interop_java/Dockerfile b/tools/dockerfile/grpc_interop_java/Dockerfile index 1ec24a95e3..5ba5324631 100644 --- a/tools/jenkins/grpc_interop_java/Dockerfile +++ b/tools/dockerfile/grpc_interop_java/Dockerfile @@ -1,4 +1,4 @@ -# Copyright 2015, Google Inc. +# Copyright 2015-2016, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without diff --git a/tools/jenkins/grpc_interop_java/build_interop.sh b/tools/dockerfile/grpc_interop_java/build_interop.sh index 9997c63308..9c9591abd8 100755 --- a/tools/jenkins/grpc_interop_java/build_interop.sh +++ b/tools/dockerfile/grpc_interop_java/build_interop.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2015, Google Inc. +# Copyright 2015-2016, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without diff --git a/tools/jenkins/grpc_interop_node/Dockerfile b/tools/dockerfile/grpc_interop_node/Dockerfile index db5aff844d..3f0f918ad9 100644 --- a/tools/jenkins/grpc_interop_node/Dockerfile +++ b/tools/dockerfile/grpc_interop_node/Dockerfile @@ -1,4 +1,4 @@ -# Copyright 2015, Google Inc. +# Copyright 2015-2016, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without diff --git a/tools/jenkins/grpc_interop_node/build_interop.sh b/tools/dockerfile/grpc_interop_node/build_interop.sh index 3b69715c9a..526dd61fdf 100755 --- a/tools/jenkins/grpc_interop_node/build_interop.sh +++ b/tools/dockerfile/grpc_interop_node/build_interop.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2015, Google Inc. +# Copyright 2015-2016, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -45,4 +45,4 @@ make install-certs # build Node interop client & server npm install -g node-gyp -(npm install && node-gyp rebuild) +npm install --unsafe-perm --build-from-source diff --git a/tools/jenkins/grpc_interop_php/Dockerfile b/tools/dockerfile/grpc_interop_php/Dockerfile index cf3e79176a..4f5b3fcb77 100644 --- a/tools/jenkins/grpc_interop_php/Dockerfile +++ b/tools/dockerfile/grpc_interop_php/Dockerfile @@ -1,4 +1,4 @@ -# Copyright 2015, Google Inc. +# Copyright 2015-2016, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without diff --git a/tools/jenkins/grpc_interop_php/build_interop.sh b/tools/dockerfile/grpc_interop_php/build_interop.sh index 87262f1d62..2bffbd8938 100755 --- a/tools/jenkins/grpc_interop_php/build_interop.sh +++ b/tools/dockerfile/grpc_interop_php/build_interop.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2015, Google Inc. +# Copyright 2015-2016, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without diff --git a/tools/jenkins/grpc_interop_python/Dockerfile b/tools/dockerfile/grpc_interop_python/Dockerfile index 047604b1b7..ef4432d260 100644 --- a/tools/jenkins/grpc_interop_python/Dockerfile +++ b/tools/dockerfile/grpc_interop_python/Dockerfile @@ -1,4 +1,4 @@ -# Copyright 2015, Google Inc. +# Copyright 2015-2016, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without diff --git a/tools/jenkins/grpc_interop_python/build_interop.sh b/tools/dockerfile/grpc_interop_python/build_interop.sh index 39c93677d8..203b012aed 100755 --- a/tools/jenkins/grpc_interop_python/build_interop.sh +++ b/tools/dockerfile/grpc_interop_python/build_interop.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2015, Google Inc. +# Copyright 2015-2016, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without diff --git a/tools/jenkins/grpc_interop_ruby/Dockerfile b/tools/dockerfile/grpc_interop_ruby/Dockerfile index ff201fa291..b3383afe79 100644 --- a/tools/jenkins/grpc_interop_ruby/Dockerfile +++ b/tools/dockerfile/grpc_interop_ruby/Dockerfile @@ -1,4 +1,4 @@ -# Copyright 2015, Google Inc. +# Copyright 2015-2016, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without diff --git a/tools/jenkins/grpc_interop_ruby/build_interop.sh b/tools/dockerfile/grpc_interop_ruby/build_interop.sh index c5023f5c1d..04288bf1aa 100755 --- a/tools/jenkins/grpc_interop_ruby/build_interop.sh +++ b/tools/dockerfile/grpc_interop_ruby/build_interop.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2015, Google Inc. +# Copyright 2015-2016, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without diff --git a/tools/jenkins/grpc_interop_cxx/Dockerfile b/tools/dockerfile/grpc_interop_stress_cxx/Dockerfile index 1fa1907533..58a8c32e34 100644 --- a/tools/jenkins/grpc_interop_cxx/Dockerfile +++ b/tools/dockerfile/grpc_interop_stress_cxx/Dockerfile @@ -1,4 +1,4 @@ -# Copyright 2015, Google Inc. +# Copyright 2015-2016, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without diff --git a/tools/jenkins/grpc_interop_stress_cxx/build_interop_stress.sh b/tools/dockerfile/grpc_interop_stress_cxx/build_interop_stress.sh index 6ed3ccb3fa..6ed3ccb3fa 100755 --- a/tools/jenkins/grpc_interop_stress_cxx/build_interop_stress.sh +++ b/tools/dockerfile/grpc_interop_stress_cxx/build_interop_stress.sh diff --git a/tools/jenkins/grpc_jenkins_slave/Dockerfile b/tools/dockerfile/grpc_jenkins_slave_x64/Dockerfile index 48541b00ad..48541b00ad 100644 --- a/tools/jenkins/grpc_jenkins_slave/Dockerfile +++ b/tools/dockerfile/grpc_jenkins_slave_x64/Dockerfile diff --git a/tools/jenkins/grpc_jenkins_slave_32bits/Dockerfile b/tools/dockerfile/grpc_jenkins_slave_x86/Dockerfile index 2323f23387..2323f23387 100644 --- a/tools/jenkins/grpc_jenkins_slave_32bits/Dockerfile +++ b/tools/dockerfile/grpc_jenkins_slave_x86/Dockerfile diff --git a/tools/jenkins/grpc_linuxbrew/Dockerfile b/tools/dockerfile/grpc_linuxbrew/Dockerfile index 848489e091..8c179da6c8 100644 --- a/tools/jenkins/grpc_linuxbrew/Dockerfile +++ b/tools/dockerfile/grpc_linuxbrew/Dockerfile @@ -1,4 +1,4 @@ -# Copyright 2015, Google Inc. +# Copyright 2015-2016, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without diff --git a/tools/jenkins/build_docker_and_run_tests.sh b/tools/jenkins/build_docker_and_run_tests.sh index 562cfcb6a6..58163bb583 100755 --- a/tools/jenkins/build_docker_and_run_tests.sh +++ b/tools/jenkins/build_docker_and_run_tests.sh @@ -47,11 +47,15 @@ mkdir -p /tmp/xdg-cache-home # Create a local branch so the child Docker script won't complain git branch -f jenkins-docker -# Use image name based on Dockerfile checksum -DOCKER_IMAGE_NAME=grpc_jenkins_slave${docker_suffix}_`sha1sum tools/jenkins/grpc_jenkins_slave/Dockerfile | cut -f1 -d\ ` +# Inputs +# DOCKERFILE_DIR - Directory in which Dockerfile file is located. +# DOCKER_RUN_SCRIPT - Script to run under docker (relative to grpc repo root) + +# Use image name based on Dockerfile location checksum +DOCKER_IMAGE_NAME=$(basename $DOCKERFILE_DIR)_$(sha1sum $DOCKERFILE_DIR/Dockerfile | cut -f1 -d\ ) # Make sure docker image has been built. Should be instantaneous if so. -docker build -t $DOCKER_IMAGE_NAME tools/jenkins/grpc_jenkins_slave$docker_suffix +docker build -t $DOCKER_IMAGE_NAME $DOCKERFILE_DIR # Choose random name for docker container CONTAINER_NAME="run_tests_$(uuidgen)" @@ -76,7 +80,7 @@ docker run \ -w /var/local/git/grpc \ --name=$CONTAINER_NAME \ $DOCKER_IMAGE_NAME \ - bash -l /var/local/jenkins/grpc/tools/jenkins/docker_run_tests.sh || DOCKER_FAILED="true" + bash -l "/var/local/jenkins/grpc/$DOCKER_RUN_SCRIPT" || DOCKER_FAILED="true" if [ "$XML_REPORT" != "" ] then diff --git a/tools/jenkins/build_interop_image.sh b/tools/jenkins/build_interop_image.sh index 73937068a4..28d5f31d4f 100755 --- a/tools/jenkins/build_interop_image.sh +++ b/tools/jenkins/build_interop_image.sh @@ -71,10 +71,10 @@ then fi # Use image name based on Dockerfile checksum -BASE_IMAGE=${BASE_NAME}_base:`sha1sum tools/jenkins/$BASE_NAME/Dockerfile | cut -f1 -d\ ` +BASE_IMAGE=${BASE_NAME}_base:`sha1sum tools/dockerfile/$BASE_NAME/Dockerfile | cut -f1 -d\ ` # Make sure base docker image has been built. Should be instantaneous if so. -docker build -t $BASE_IMAGE --force-rm=true tools/jenkins/$BASE_NAME || exit $? +docker build -t $BASE_IMAGE --force-rm=true tools/dockerfile/$BASE_NAME || exit $? # Create a local branch so the child Docker script won't complain git branch -f jenkins-docker @@ -92,7 +92,7 @@ CONTAINER_NAME="build_${BASE_NAME}_$(uuidgen)" -v /tmp/ccache:/tmp/ccache \ --name=$CONTAINER_NAME \ $BASE_IMAGE \ - bash -l /var/local/jenkins/grpc/tools/jenkins/$BASE_NAME/build_interop.sh \ + bash -l /var/local/jenkins/grpc/tools/dockerfile/$BASE_NAME/build_interop.sh \ && docker commit $CONTAINER_NAME $INTEROP_IMAGE \ && echo "Successfully built image $INTEROP_IMAGE") EXITCODE=$? diff --git a/tools/jenkins/build_interop_stress_image.sh b/tools/jenkins/build_interop_stress_image.sh index 395eaa32d9..92f2dab5e3 100755 --- a/tools/jenkins/build_interop_stress_image.sh +++ b/tools/jenkins/build_interop_stress_image.sh @@ -55,10 +55,10 @@ then fi # Use image name based on Dockerfile checksum -BASE_IMAGE=${BASE_NAME}_base:`sha1sum tools/jenkins/$BASE_NAME/Dockerfile | cut -f1 -d\ ` +BASE_IMAGE=${BASE_NAME}_base:`sha1sum tools/dockerfile/$BASE_NAME/Dockerfile | cut -f1 -d\ ` # Make sure base docker image has been built. Should be instantaneous if so. -docker build -t $BASE_IMAGE --force-rm=true tools/jenkins/$BASE_NAME || exit $? +docker build -t $BASE_IMAGE --force-rm=true tools/dockerfile/$BASE_NAME || exit $? # Create a local branch so the child Docker script won't complain git branch -f jenkins-docker @@ -75,7 +75,7 @@ CONTAINER_NAME="build_${BASE_NAME}_$(uuidgen)" -v /tmp/ccache:/tmp/ccache \ --name=$CONTAINER_NAME \ $BASE_IMAGE \ - bash -l /var/local/jenkins/grpc/tools/jenkins/$BASE_NAME/build_interop_stress.sh \ + bash -l /var/local/jenkins/grpc/tools/dockerfile/$BASE_NAME/build_interop_stress.sh \ && docker commit $CONTAINER_NAME $INTEROP_IMAGE \ && echo "Successfully built image $INTEROP_IMAGE") EXITCODE=$? diff --git a/tools/jenkins/run_distribution.sh b/tools/jenkins/run_distribution.sh index 64c60f1502..446ce16703 100755 --- a/tools/jenkins/run_distribution.sh +++ b/tools/jenkins/run_distribution.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2015, Google Inc. +# Copyright 2015-2016, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -47,11 +47,11 @@ if [ "$platform" == "linux" ]; then if [ "$dist_channel" == "homebrew" ]; then - sha1=$(sha1sum tools/jenkins/grpc_linuxbrew/Dockerfile | cut -f1 -d\ ) + sha1=$(sha1sum tools/dockerfile/grpc_linuxbrew/Dockerfile | cut -f1 -d\ ) DOCKER_IMAGE_NAME=grpc_linuxbrew_$sha1 # build docker image, contains all pre-requisites - docker build -t $DOCKER_IMAGE_NAME tools/jenkins/grpc_linuxbrew + docker build -t $DOCKER_IMAGE_NAME tools/dockerfile/grpc_linuxbrew # run per-language homebrew installation script docker run --rm=true $DOCKER_IMAGE_NAME bash -l \ diff --git a/tools/jenkins/run_jenkins.sh b/tools/jenkins/run_jenkins.sh index 49b2fa55c2..123b252719 100755 --- a/tools/jenkins/run_jenkins.sh +++ b/tools/jenkins/run_jenkins.sh @@ -39,51 +39,17 @@ # NOTE: No empty lines should appear in this file before igncr is set! set -ex -o igncr || set -ex -# Grabbing the machine's architecture -arch=`uname -m` - -case $platform in - i386) - arch="i386" - platform="linux" - docker_suffix=_32bits - ;; -esac - if [ "$platform" == "linux" ] then - echo "building $language on Linux" - - ./tools/run_tests/run_tests.py --use_docker -t -l $language -c $config -x report.xml -j 3 $@ || TESTS_FAILED="true" - -elif [ "$platform" == "windows" ] -then - echo "building $language on Windows" - - # Prevent msbuild from picking up "platform" env variable, which would break the build - unset platform - - python tools/run_tests/run_tests.py -t -l $language -c $config -x report.xml -j 3 $@ || TESTS_FAILED="true" - -elif [ "$platform" == "macos" ] -then - echo "building $language on MacOS" - - # Prevent msbuild from picking up "platform" env variable, which would break the build - unset platform - - ./tools/run_tests/run_tests.py -t -l $language -c $config -x report.xml -j 3 $@ || TESTS_FAILED="true" - + USE_DOCKER_MAYBE="--use_docker" elif [ "$platform" == "freebsd" ] then - echo "building $language on FreeBSD" + export MAKE=gmake +fi - MAKE=gmake ./tools/run_tests/run_tests.py -t -l $language -c $config -x report.xml -j 3 $@ || TESTS_FAILED="true" +unset platform # variable named 'platform' breaks the windows build -else - echo "Unknown platform $platform" - exit 1 -fi +python tools/run_tests/run_tests.py $USE_DOCKER_MAYBE -t -l $language -c $config -x report.xml -j 3 $@ || TESTS_FAILED="true" if [ ! -e reports/index.html ] then diff --git a/tools/jenkins/run_portability.sh b/tools/jenkins/run_portability.sh index afce4ad743..f76dbd3144 100755 --- a/tools/jenkins/run_portability.sh +++ b/tools/jenkins/run_portability.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Copyright 2015, Google Inc. +# Copyright 2015-2016, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -52,8 +52,7 @@ maybe_build_only='--build_only' if [ "$curr_platform" == "windows" ] then - win_arch="windows_${curr_arch}" - python tools/run_tests/run_tests.py -t -l $language -c $config --arch ${win_arch} --compiler ${curr_compiler} ${maybe_build_only} -x report.xml $@ + python tools/run_tests/run_tests.py -t -l $language -c $config --arch ${curr_arch} --compiler ${curr_compiler} ${maybe_build_only} -x report.xml $@ else echo "Unsupported scenario." exit 1 diff --git a/tools/run_tests/build_artifacts.py b/tools/run_tests/build_artifacts.py index ff9dd4735a..0337f1b3d7 100755 --- a/tools/run_tests/build_artifacts.py +++ b/tools/run_tests/build_artifacts.py @@ -135,7 +135,7 @@ class CSharpExtArtifact: 'EMBED_ZLIB': 'true'} if self.platform == 'linux': return create_docker_jobspec(self.name, - 'tools/jenkins/grpc_artifact_linux_%s' % self.arch, + 'tools/dockerfile/grpc_artifact_linux_%s' % self.arch, 'tools/run_tests/build_artifact_csharp.sh') else: environ.update(macos_arch_env(self.arch)) diff --git a/tools/run_tests/build_node.sh b/tools/run_tests/build_node.sh index faa7b624b8..8f2ab4413a 100755 --- a/tools/run_tests/build_node.sh +++ b/tools/run_tests/build_node.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright 2015, Google Inc. +# Copyright 2015-2016, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -36,4 +36,4 @@ CONFIG=${CONFIG:-opt} # change to grpc repo root cd $(dirname $0)/../.. -npm install --unsafe-perm +npm install --unsafe-perm --build-from-source diff --git a/tools/run_tests/run_node.sh b/tools/run_tests/run_node.sh index fff579fab8..f93c9c30cb 100755 --- a/tools/run_tests/run_node.sh +++ b/tools/run_tests/run_node.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2015, Google Inc. +# Copyright 2015-2016, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -51,5 +51,5 @@ then echo '<html><head><meta http-equiv="refresh" content="0;URL=lcov-report/index.html"></head></html>' > \ ../reports/node_coverage/index.html else - JUNIT_REPORT_PATH=src/node/reports.xml JUNIT_REPORT_STACK=1 ./node_modules/.bin/mocha --reporter mocha-jenkins-reporter src/node/test || true + JUNIT_REPORT_PATH=src/node/reports.xml JUNIT_REPORT_STACK=1 ./node_modules/.bin/mocha --reporter mocha-jenkins-reporter src/node/test fi diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index 3180e6bc3a..c8f5d6fc2f 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -452,7 +452,7 @@ class Sanity(object): def test_specs(self, config, args): import yaml with open('tools/run_tests/sanity_tests.yaml', 'r') as f: - return [config.job_spec([cmd['script']], None, timeout_seconds=None, environ={'TEST': 'true'}, cpu_cost=cmd.get('cpu_cost', 1)) + return [config.job_spec(cmd['script'].split(), None, timeout_seconds=None, environ={'TEST': 'true'}, cpu_cost=cmd.get('cpu_cost', 1)) for cmd in yaml.load(f)] def pre_build_steps(self): @@ -538,15 +538,37 @@ _WINDOWS_CONFIG = { def _windows_arch_option(arch): """Returns msbuild cmdline option for selected architecture.""" - if arch == 'default' or arch == 'windows_x86': + if arch == 'default' or arch == 'x86': return '/p:Platform=Win32' - elif arch == 'windows_x64': + elif arch == 'x64': return '/p:Platform=x64' else: - print 'Architecture %s not supported on current platform.' % arch + print 'Architecture %s not supported.' % arch sys.exit(1) - + +def _check_arch_option(arch): + """Checks that architecture option is valid.""" + if platform_string() == 'windows': + _windows_arch_option(arch) + elif platform_string() == 'linux': + # On linux, we need to be running under docker with the right architecture. + runtime_arch = platform.architecture()[0] + if arch == 'default': + return + elif runtime_arch == '64bit' and arch == 'x64': + return + elif runtime_arch == '32bit' and arch == 'x86': + return + else: + print 'Architecture %s does not match current runtime architecture.' % arch + sys.exit(1) + else: + if args.arch != 'default': + print 'Architecture %s not supported on current platform.' % args.arch + sys.exit(1) + + def _windows_build_bat(compiler): """Returns name of build.bat for selected compiler.""" if compiler == 'default' or compiler == 'vs2013': @@ -558,8 +580,8 @@ def _windows_build_bat(compiler): else: print 'Compiler %s not supported.' % compiler sys.exit(1) - - + + def _windows_toolset_option(compiler): """Returns msbuild PlatformToolset for selected compiler.""" if compiler == 'default' or compiler == 'vs2013': @@ -571,7 +593,17 @@ def _windows_toolset_option(compiler): else: print 'Compiler %s not supported.' % compiler sys.exit(1) - + + +def _get_dockerfile_dir(arch): + """Returns dockerfile to use""" + if arch == 'default' or arch == 'x64': + return 'tools/dockerfile/grpc_jenkins_slave_x64' + elif arch == 'x86': + return 'tools/dockerfile/grpc_jenkins_slave_x86' + else: + print 'Architecture %s not supported with current settings.' % arch + sys.exit(1) def runs_per_test_type(arg_str): """Auxilary function to parse the "runs_per_test" flag. @@ -638,7 +670,7 @@ argp.add_argument('--allow_flakes', const=True, help='Allow flaky tests to show as passing (re-runs failed tests up to five times)') argp.add_argument('--arch', - choices=['default', 'windows_x86', 'windows_x64'], + choices=['default', 'x86', 'x64'], default='default', help='Selects architecture to target. For some platforms "default" is the only supported choice.') argp.add_argument('--compiler', @@ -662,36 +694,6 @@ args = argp.parse_args() jobset.measure_cpu_costs = args.measure_cpu_costs -if args.use_docker: - if not args.travis: - print 'Seen --use_docker flag, will run tests under docker.' - print - print 'IMPORTANT: The changes you are testing need to be locally committed' - print 'because only the committed changes in the current branch will be' - print 'copied to the docker environment.' - time.sleep(5) - - child_argv = [ arg for arg in sys.argv if not arg == '--use_docker' ] - run_tests_cmd = 'tools/run_tests/run_tests.py %s' % ' '.join(child_argv[1:]) - - # TODO(jtattermusch): revisit if we need special handling for arch here - # set arch command prefix in case we are working with different arch. - arch_env = os.getenv('arch') - if arch_env: - run_test_cmd = 'arch %s %s' % (arch_env, run_test_cmd) - - env = os.environ.copy() - env['RUN_TESTS_COMMAND'] = run_tests_cmd - if args.xml_report: - env['XML_REPORT'] = args.xml_report - if not args.travis: - env['TTY_FLAG'] = '-t' # enables Ctrl-C when not on Jenkins. - - subprocess.check_call(['tools/jenkins/build_docker_and_run_tests.sh'], - shell=True, - env=env) - sys.exit(0) - # update submodules if necessary need_to_regenerate_projects = False for spec in args.update_submodules: @@ -755,16 +757,44 @@ if any(language.make_options() for language in languages): else: language_make_options = next(iter(languages)).make_options() -if platform_string() != 'windows': - if args.arch != 'default': - print 'Architecture %s not supported on current platform.' % args.arch - sys.exit(1) - if args.compiler != 'default': +if len(languages) != 1 or len(build_configs) != 1: + print 'Multi-language and multi-config testing is not supported.' + sys.exit(1) + +if args.use_docker: + if not args.travis: + print 'Seen --use_docker flag, will run tests under docker.' + print + print 'IMPORTANT: The changes you are testing need to be locally committed' + print 'because only the committed changes in the current branch will be' + print 'copied to the docker environment.' + time.sleep(5) + + child_argv = [ arg for arg in sys.argv if not arg == '--use_docker' ] + run_tests_cmd = 'tools/run_tests/run_tests.py %s' % ' '.join(child_argv[1:]) + + env = os.environ.copy() + env['RUN_TESTS_COMMAND'] = run_tests_cmd + env['DOCKERFILE_DIR'] = _get_dockerfile_dir(args.arch) + env['DOCKER_RUN_SCRIPT'] = 'tools/jenkins/docker_run_tests.sh' + if args.xml_report: + env['XML_REPORT'] = args.xml_report + if not args.travis: + env['TTY_FLAG'] = '-t' # enables Ctrl-C when not on Jenkins. + + subprocess.check_call(['tools/jenkins/build_docker_and_run_tests.sh'], + shell=True, + env=env) + sys.exit(0) + +if platform_string() != 'windows' and args.compiler != 'default': print 'Compiler %s not supported on current platform.' % args.compiler sys.exit(1) -if platform_string() == 'windows': - def make_jobspec(cfg, targets, makefile='Makefile'): +_check_arch_option(args.arch) + +def make_jobspec(cfg, targets, makefile='Makefile'): + if platform_string() == 'windows': extra_args = [] # better do parallel compilation # empirically /m:2 gives the best performance/price and should prevent @@ -782,8 +812,7 @@ if platform_string() == 'windows': language_make_options, shell=True, timeout_seconds=None) for target in targets] -else: - def make_jobspec(cfg, targets, makefile='Makefile'): + else: if targets: return [jobset.JobSpec([os.getenv('MAKE', 'make'), '-f', makefile, @@ -796,6 +825,7 @@ else: timeout_seconds=None)] else: return [] + make_targets = {} for l in languages: makefile = l.makefile_name() diff --git a/tools/run_tests/sanity_tests.yaml b/tools/run_tests/sanity_tests.yaml index 160acde565..757bcd9d34 100644 --- a/tools/run_tests/sanity_tests.yaml +++ b/tools/run_tests/sanity_tests.yaml @@ -2,8 +2,8 @@ - script: tools/run_tests/check_sources_and_headers.py - script: tools/run_tests/check_submodules.sh - script: tools/run_tests/check_cache_mk.sh -- script: tools/buildgen/generate_projects.sh - cpu_cost: 100 +- script: tools/buildgen/generate_projects.sh -j 3 + cpu_cost: 3 - script: tools/distrib/check_copyright.py - script: tools/distrib/clang_format_code.sh - script: tools/distrib/check_trailing_newlines.sh diff --git a/tools/tsan_suppressions.txt b/tools/tsan_suppressions.txt index 65e7e2ec4b..09e68cd233 100644 --- a/tools/tsan_suppressions.txt +++ b/tools/tsan_suppressions.txt @@ -5,4 +5,4 @@ race:cleanse_ctr # https://www.mail-archive.com/openssl-dev@openssl.org/msg09019.html race:ssleay_rand_add race:ssleay_rand_bytes - +race:__sleep_for |