diff options
Diffstat (limited to 'tools')
| -rw-r--r-- | tools/dockerfile/grpc_build_deb/Dockerfile | 29 | ||||
| -rw-r--r-- | tools/dockerfile/grpc_ruby_deb/Dockerfile | 29 | ||||
| -rwxr-xr-x | tools/gce_setup/cloud_prod_runner.sh | 13 | ||||
| -rwxr-xr-x | tools/gce_setup/grpc_docker.sh | 43 | ||||
| -rwxr-xr-x | tools/run_tests/jobset.py | 32 | ||||
| -rwxr-xr-x | tools/run_tests/run_tests.py | 5 |
6 files changed, 109 insertions, 42 deletions
diff --git a/tools/dockerfile/grpc_build_deb/Dockerfile b/tools/dockerfile/grpc_build_deb/Dockerfile index ad26fb35d0..6cba74e4c6 100644 --- a/tools/dockerfile/grpc_build_deb/Dockerfile +++ b/tools/dockerfile/grpc_build_deb/Dockerfile @@ -1,3 +1,32 @@ +# Copyright 2015, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + # Dockerfile to build Debian packages for gRPC C core. FROM grpc/base diff --git a/tools/dockerfile/grpc_ruby_deb/Dockerfile b/tools/dockerfile/grpc_ruby_deb/Dockerfile index 25ea2c54bd..679fa51f5d 100644 --- a/tools/dockerfile/grpc_ruby_deb/Dockerfile +++ b/tools/dockerfile/grpc_ruby_deb/Dockerfile @@ -1,3 +1,32 @@ +# Copyright 2015, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + # Dockerfile for gRPC Ruby, but using Debian packages for gRPC C core. FROM grpc/ruby_base diff --git a/tools/gce_setup/cloud_prod_runner.sh b/tools/gce_setup/cloud_prod_runner.sh index 52e9b5e2a3..4732f952c2 100755 --- a/tools/gce_setup/cloud_prod_runner.sh +++ b/tools/gce_setup/cloud_prod_runner.sh @@ -32,6 +32,7 @@ main() { source grpc_docker.sh test_cases=(large_unary empty_unary ping_pong client_streaming server_streaming service_account_creds compute_engine_creds) + auth_test_cases=(service_account_creds compute_engine_creds) clients=(cxx java go ruby node) for test_case in "${test_cases[@]}" do @@ -45,6 +46,18 @@ main() { fi done done + for test_case in "${auth_test_cases[@]}" + do + for client in "${clients[@]}" + do + if grpc_cloud_prod_auth_test $test_case grpc-docker-testclients $client + then + echo "$test_case $client $server passed" >> /tmp/cloud_prod_result.txt + else + echo "$test_case $client $server failed" >> /tmp/cloud_prod_result.txt + fi + done + done gsutil cp /tmp/cloud_prod_result.txt gs://stoked-keyword-656-output/cloud_prod_result.txt rm /tmp/cloud_prod_result.txt } diff --git a/tools/gce_setup/grpc_docker.sh b/tools/gce_setup/grpc_docker.sh index bbc138c6be..231625efb3 100755 --- a/tools/gce_setup/grpc_docker.sh +++ b/tools/gce_setup/grpc_docker.sh @@ -350,7 +350,7 @@ grpc_interop_test_flags() { echo "$FUNCNAME: missing arg: test_case" 1>&2 return 1 } - echo "--server_host=$server_ip --server_port=$port --test_case=$test_case" + echo "--server_host_override=foo.test.google.fr --server_host=$server_ip --server_port=$port --test_case=$test_case" } # checks the positional args and assigns them to variables visible in the caller @@ -503,7 +503,7 @@ grpc_cloud_prod_auth_test_args() { [[ -n $1 ]] && { # client_type case $1 in - cxx|go|java|nodejs|php|python|ruby) + cxx|go|java|node|php|python|ruby) grpc_gen_test_cmd+="_gen_$1_cmd" declare -F $grpc_gen_test_cmd >> /dev/null || { echo "-f: test_func for $1 => $grpc_gen_test_cmd is not defined" 1>&2 @@ -673,7 +673,7 @@ _grpc_launch_servers_args() { [[ -n $1 ]] && { servers="$@" } || { - servers="cxx java go node ruby" + servers="cxx java go node ruby python" echo "$FUNCNAME: no servers specified, will launch defaults '$servers'" } } @@ -795,16 +795,7 @@ grpc_interop_test() { echo " $ssh_cmd" echo "on $host" [[ $dry_run == 1 ]] && return 0 # don't run the command on a dry run - gcloud compute $project_opt ssh $zone_opt $host --command "$cmd" & - PID=$! - sleep 10 - echo "pid is $PID" - if ps -p $PID - then - kill $PID - return 1 - fi - + gcloud compute $project_opt ssh $zone_opt $host --command "$cmd" } # Runs a test command on a docker instance. @@ -850,16 +841,7 @@ grpc_cloud_prod_test() { echo " $ssh_cmd" echo "on $host" [[ $dry_run == 1 ]] && return 0 # don't run the command on a dry run - gcloud compute $project_opt ssh $zone_opt $host --command "$cmd" & - PID=$! - sleep 10 - echo "pid is $PID" - if ps -p $PID - then - kill $PID - return 1 - fi - + gcloud compute $project_opt ssh $zone_opt $host --command "$cmd" } # Runs a test command on a docker instance. @@ -945,7 +927,7 @@ grpc_cloud_prod_auth_service_account_creds_gen_go_cmd() { local test_script="cd src/google.golang.org/grpc/interop/client" local test_script+=" && go run client.go --use_tls=true" local gfe_flags=" --tls_ca_file=\"\" --tls_server_name=\"\" --server_port=443 --server_host=grpc-test.sandbox.google.com" - local added_gfe_flags=$(_grpc_svc_acc_test_flags) + local added_gfe_flags=$(_grpc_svc_acc_test_flags) local the_cmd="$cmd_prefix '$test_script $gfe_flags $added_gfe_flags $@'" echo $the_cmd } @@ -960,7 +942,7 @@ grpc_cloud_prod_auth_compute_engine_creds_gen_go_cmd() { local test_script="cd src/google.golang.org/grpc/interop/client" local test_script+=" && go run client.go --use_tls=true" local gfe_flags=" --tls_ca_file=\"\" --tls_server_name=\"\" --server_port=443 --server_host=grpc-test.sandbox.google.com" - local added_gfe_flags=$(_grpc_gce_test_flags) + local added_gfe_flags=$(_grpc_gce_test_flags) local the_cmd="$cmd_prefix '$test_script $gfe_flags $added_gfe_flags $@'" echo $the_cmd } @@ -1085,7 +1067,8 @@ grpc_interop_gen_node_cmd() { # flags= .... # generic flags to include the command # cmd=$($grpc_gen_test_cmd $flags) grpc_cloud_prod_gen_node_cmd() { - local cmd_prefix="sudo docker run grpc/node"; + local env_flag="-e SSL_CERT_FILE=/cacerts/roots.pem " + local cmd_prefix="sudo docker run $env_flag grpc/node"; local test_script="/usr/bin/nodejs /var/local/git/grpc/src/node/interop/interop_client.js --use_tls=true"; local gfe_flags=$(_grpc_prod_gfe_flags); local the_cmd="$cmd_prefix $test_script $gfe_flags $@"; @@ -1098,12 +1081,12 @@ grpc_cloud_prod_gen_node_cmd() { # flags= .... # generic flags to include the command # cmd=$($grpc_gen_test_cmd $flags) grpc_cloud_prod_auth_service_account_creds_gen_node_cmd() { - local cmd_prefix="sudo docker run grpc/node"; + local env_flag="-e SSL_CERT_FILE=/cacerts/roots.pem " + env_flag+="-e GOOGLE_APPLICATION_CREDENTIALS=/service_account/stubbyCloudTestingTest-7dd63462c60c.json " + local cmd_prefix="sudo docker run $env_flag grpc/node"; local test_script="/usr/bin/nodejs /var/local/git/grpc/src/node/interop/interop_client.js --use_tls=true"; local gfe_flags=$(_grpc_prod_gfe_flags); - local env_prefix="SSL_CERT_FILE=/cacerts/roots.pem" - env_prefix+=" GOOGLE_APPLICATION_CREDENTIALS=/service_account/stubbyCloudTestingTest-7dd63462c60c.json" - local the_cmd="$env_prefix $cmd_prefix $test_script $gfe_flags $@"; + local the_cmd="$cmd_prefix $test_script $gfe_flags $@"; echo $the_cmd } diff --git a/tools/run_tests/jobset.py b/tools/run_tests/jobset.py index df83b30516..569cb5bac2 100755 --- a/tools/run_tests/jobset.py +++ b/tools/run_tests/jobset.py @@ -33,6 +33,7 @@ import hashlib import multiprocessing import os import random +import signal import subprocess import sys import tempfile @@ -42,6 +43,12 @@ import time _DEFAULT_MAX_JOBS = 16 * multiprocessing.cpu_count() +# setup a signal handler so that signal.pause registers 'something' +# when a child finishes +# not using futures and threading to avoid a dependency on subprocess32 +signal.signal(signal.SIGCHLD, lambda unused_signum, unused_frame: None) + + def shuffle_iteratable(it): """Return an iterable that randomly walks it""" # take a random sampling from the passed in iterable @@ -94,16 +101,19 @@ _TAG_COLOR = { def message(tag, message, explanatory_text=None, do_newline=False): - sys.stdout.write('%s%s%s\x1b[%d;%dm%s\x1b[0m: %s%s' % ( - _BEGINNING_OF_LINE, - _CLEAR_LINE, - '\n%s' % explanatory_text if explanatory_text is not None else '', - _COLORS[_TAG_COLOR[tag]][1], - _COLORS[_TAG_COLOR[tag]][0], - tag, - message, - '\n' if do_newline or explanatory_text is not None else '')) - sys.stdout.flush() + try: + sys.stdout.write('%s%s%s\x1b[%d;%dm%s\x1b[0m: %s%s' % ( + _BEGINNING_OF_LINE, + _CLEAR_LINE, + '\n%s' % explanatory_text if explanatory_text is not None else '', + _COLORS[_TAG_COLOR[tag]][1], + _COLORS[_TAG_COLOR[tag]][0], + tag, + message, + '\n' if do_newline or explanatory_text is not None else '')) + sys.stdout.flush() + except: + pass def which(filename): @@ -232,7 +242,7 @@ class Jobset(object): if dead: return message('WAITING', '%d jobs running, %d complete, %d failed' % ( len(self._running), self._completed, self._failures)) - time.sleep(0.1) + signal.pause() def cancelled(self): """Poll for cancellation.""" diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index 64478b3753..7732466d6e 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -36,6 +36,7 @@ import itertools import json import multiprocessing import os +import re import sys import time @@ -168,6 +169,7 @@ argp.add_argument('-c', '--config', nargs='+', default=_DEFAULT) argp.add_argument('-n', '--runs_per_test', default=1, type=int) +argp.add_argument('-r', '--regex', default='.*', type=str) argp.add_argument('-j', '--jobs', default=1000, type=int) argp.add_argument('-f', '--forever', default=False, @@ -205,7 +207,8 @@ one_run = set( spec for config in run_configs for language in args.language - for spec in _LANGUAGES[language].test_specs(config)) + for spec in _LANGUAGES[language].test_specs(config) + if re.search(args.regex, spec.shortname)) runs_per_test = args.runs_per_test forever = args.forever |