diff options
Diffstat (limited to 'tools')
-rwxr-xr-x | tools/buildgen/plugins/generate_vsprojects.py | 3 | ||||
-rw-r--r-- | tools/dockerfile/grpc_dist_proto/Dockerfile | 76 | ||||
-rw-r--r-- | tools/dockerfile/grpc_dist_proto/version.txt | 1 | ||||
-rw-r--r-- | tools/dockerfile/grpc_python/Dockerfile | 2 | ||||
-rwxr-xr-x | tools/gce_setup/cloud_prod_runner.sh | 2 | ||||
-rwxr-xr-x | tools/gce_setup/grpc_docker.sh | 85 | ||||
-rwxr-xr-x | tools/gce_setup/interop_test_runner.sh | 2 | ||||
-rwxr-xr-x | tools/run_tests/build_python.sh | 2 | ||||
-rwxr-xr-x | tools/run_tests/jobset.py | 4 | ||||
-rwxr-xr-x | tools/run_tests/python_tests.json | 2 | ||||
-rwxr-xr-x | tools/run_tests/run_python.sh | 2 | ||||
-rwxr-xr-x | tools/run_tests/run_tests.py | 2 | ||||
-rw-r--r-- | tools/run_tests/tests.json | 5 |
13 files changed, 170 insertions, 18 deletions
diff --git a/tools/buildgen/plugins/generate_vsprojects.py b/tools/buildgen/plugins/generate_vsprojects.py index 6cbd74df66..f00c176214 100755 --- a/tools/buildgen/plugins/generate_vsprojects.py +++ b/tools/buildgen/plugins/generate_vsprojects.py @@ -57,9 +57,8 @@ def mako_plugin(dictionary): projects = [project for project in projects if project.get('vs_project_guid', None)] - # Exclude C++ projects for now projects = [project for project in projects - if not project['language'] == 'c++'] + if project['language'] != 'c++' or project['build'] == 'all'] project_dict = dict([(p['name'], p) for p in projects]) diff --git a/tools/dockerfile/grpc_dist_proto/Dockerfile b/tools/dockerfile/grpc_dist_proto/Dockerfile new file mode 100644 index 0000000000..b4ed3b6035 --- /dev/null +++ b/tools/dockerfile/grpc_dist_proto/Dockerfile @@ -0,0 +1,76 @@ +# Copyright 2015, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# Dockerfile to build protoc and plugins for inclusion in a release. +FROM grpc/base + +# Add the file containing the gRPC version +ADD version.txt version.txt + +# Install tools needed for building protoc. +RUN apt-get update && apt-get -y install libgflags-dev libgtest-dev + +# Get the protobuf source from GitHub. +RUN mkdir -p /var/local/git +RUN git clone https://github.com/google/protobuf.git /var/local/git/protobuf + +# Build the protobuf library statically and install to /tmp/protoc_static. +WORKDIR /var/local/git/protobuf +RUN ./autogen.sh && \ + ./configure --disable-shared --prefix=/tmp/protoc_static \ + LDFLAGS="-lgcc_eh -static-libgcc -static-libstdc++" && \ + make -j12 && make check && make install + +# Build the protobuf library dynamically and install to /usr/local. +WORKDIR /var/local/git/protobuf +RUN ./autogen.sh && \ + ./configure --prefix=/usr/local && \ + make -j12 && make check && make install + +# Build the grpc plugins. +RUN git clone https://github.com/google/grpc.git /var/local/git/grpc +WORKDIR /var/local/git/grpc +RUN LDFLAGS=-static make plugins + +# Create an archive containing all the generated binaries. +RUN mkdir /tmp/proto-bins_$(cat /version.txt)_linux-$(uname -m) +RUN cp -v bins/opt/* /tmp/proto-bins_$(cat /version.txt)_linux-$(uname -m) +RUN cp -v /tmp/protoc_static/bin/protoc /tmp/proto-bins_$(cat /version.txt)_linux-$(uname -m) +RUN cd /tmp && \ + tar -czf proto-bins_$(cat /version.txt)_linux-$(uname -m).tar.gz proto-bins_$(cat /version.txt)_linux-$(uname -m) + +# List the tar contents: provides a way to visually confirm that the contents +# are correct. +RUN echo 'proto-bins_$(cat /version.txt)_linux-tar-$(uname -m) contents:' && \ + tar -ztf /tmp/proto-bins_$(cat /version.txt)_linux-$(uname -m).tar.gz + + + + + diff --git a/tools/dockerfile/grpc_dist_proto/version.txt b/tools/dockerfile/grpc_dist_proto/version.txt new file mode 100644 index 0000000000..8f0916f768 --- /dev/null +++ b/tools/dockerfile/grpc_dist_proto/version.txt @@ -0,0 +1 @@ +0.5.0 diff --git a/tools/dockerfile/grpc_python/Dockerfile b/tools/dockerfile/grpc_python/Dockerfile index fd07e9cc6a..62ef785a31 100644 --- a/tools/dockerfile/grpc_python/Dockerfile +++ b/tools/dockerfile/grpc_python/Dockerfile @@ -54,7 +54,7 @@ RUN cd /var/local/git/grpc \ && python2.7 -B -m grpc._adapter._lonely_rear_link_test \ && python2.7 -B -m grpc._adapter._low_test \ && python2.7 -B -m grpc.early_adopter.implementations_test \ - && python2.7 -B -m grpc.framework.base.packets.implementations_test \ + && python2.7 -B -m grpc.framework.base.implementations_test \ && python2.7 -B -m grpc.framework.face.blocking_invocation_inline_service_test \ && python2.7 -B -m grpc.framework.face.event_invocation_synchronous_event_service_test \ && python2.7 -B -m grpc.framework.face.future_invocation_asynchronous_event_service_test \ diff --git a/tools/gce_setup/cloud_prod_runner.sh b/tools/gce_setup/cloud_prod_runner.sh index 3a9ae51b76..e236c921ec 100755 --- a/tools/gce_setup/cloud_prod_runner.sh +++ b/tools/gce_setup/cloud_prod_runner.sh @@ -36,7 +36,7 @@ echo $result_file_name main() { source grpc_docker.sh test_cases=(large_unary empty_unary ping_pong client_streaming server_streaming cancel_after_begin cancel_after_first_response) - auth_test_cases=(service_account_creds compute_engine_creds) + auth_test_cases=(service_account_creds compute_engine_creds jwt_token_creds) clients=(cxx java go ruby node csharp_mono) for test_case in "${test_cases[@]}" do diff --git a/tools/gce_setup/grpc_docker.sh b/tools/gce_setup/grpc_docker.sh index 3deef05ef3..497112ce39 100755 --- a/tools/gce_setup/grpc_docker.sh +++ b/tools/gce_setup/grpc_docker.sh @@ -560,7 +560,7 @@ grpc_sync_scripts() { _grpc_ensure_gcloud_ssh || return 1; # declare vars local so that they don't pollute the shell environment - # where they this func is used. + # where this func is used. local grpc_zone grpc_project dry_run # set by _grpc_set_project_and_zone local grpc_hosts grpc_gce_script_root @@ -600,7 +600,7 @@ grpc_sync_images() { _grpc_ensure_gcloud_ssh || return 1; # declare vars local so that they don't pollute the shell environment - # where they this func is used. + # where this func is used. local grpc_zone grpc_project dry_run # set by _grpc_set_project_and_zone local grpc_hosts @@ -645,7 +645,7 @@ _grpc_show_servers_args() { # Shows the grpc servers on the GCE instance <server_name> grpc_show_servers() { # declare vars local so that they don't pollute the shell environment - # where they this func is used. + # where this func is used. local grpc_zone grpc_project dry_run # set by _grpc_set_project_and_zone # set by _grpc_show_servers local host @@ -663,6 +663,58 @@ grpc_show_servers() { gcloud compute $project_opt ssh $zone_opt $host --command "$cmd" } +_grpc_build_proto_bins_args() { + [[ -n $1 ]] && { # host + host=$1 + shift + } || { + host='grpc-docker-builder' + } +} + +# grpc_build_proto_bins +# +# - rebuilds the dist_proto docker image +# * doing this builds the protoc and the ruby, python and cpp bins statically +# +# - runs a docker command that copies the built protos to the GCE host +# - copies the built protos to the local machine +grpc_build_proto_bins() { + _grpc_ensure_gcloud_ssh || return 1; + + # declare vars local so that they don't pollute the shell environment + # where this func is used. + local grpc_zone grpc_project dry_run # set by _grpc_set_project_and_zone + # set by _grpc_build_proto_bins_args + local host + + # set the project zone and check that all necessary args are provided + _grpc_set_project_and_zone -f _grpc_build_proto_bins_args "$@" || return 1 + gce_has_instance $grpc_project $host || return 1; + local project_opt="--project $grpc_project" + local zone_opt="--zone $grpc_zone" + + # rebuild the dist_proto image + local label='dist_proto' + grpc_update_image -- -h $host $label || return 1 + + # run a command to copy the generated archive to the docker host + local docker_prefix='sudo docker run -v /tmp:/tmp/proto_bins_out' + local tar_name='proto-bins*.tar.gz' + local cp_cmd="/bin/bash -c 'cp -v /tmp/$tar_name /tmp/proto_bins_out'" + local cmd="$docker_prefix grpc/$label $cp_cmd" + local ssh_cmd="bash -l -c \"$cmd\"" + echo "will run:" + echo " $ssh_cmd" + echo "on $host" + gcloud compute $project_opt ssh $zone_opt $host --command "$cmd" || return 1 + + # copy the tar.gz locally + local rmt_tar="$host:/tmp/$tar_name" + local local_copy="$(pwd)" + gcloud compute copy-files $rmt_tar $local_copy $project_opt $zone_opt || return 1 +} + _grpc_launch_servers_args() { [[ -n $1 ]] && { # host host=$1 @@ -690,7 +742,7 @@ _grpc_launch_servers_args() { # If no servers are specified, it launches all known servers grpc_launch_servers() { # declare vars local so that they don't pollute the shell environment - # where they this func is used. + # where this func is used. local grpc_zone grpc_project dry_run # set by _grpc_set_project_and_zone # set by _grpc_launch_servers_args local host servers @@ -811,7 +863,7 @@ test_runner() { grpc_interop_test() { _grpc_ensure_gcloud_ssh || return 1; # declare vars local so that they don't pollute the shell environment - # where they this func is used. + # where this func is used. local grpc_zone grpc_project dry_run # set by _grpc_set_project_and_zone # grpc_interop_test_args @@ -853,7 +905,7 @@ grpc_interop_test() { grpc_cloud_prod_test() { _grpc_ensure_gcloud_ssh || return 1; # declare vars local so that they don't pollute the shell environment - # where they this func is used. + # where this func is used. local grpc_zone grpc_project dry_run # set by _grpc_set_project_and_zone # grpc_cloud_prod_test_args @@ -892,7 +944,7 @@ grpc_cloud_prod_test() { grpc_cloud_prod_auth_test() { _grpc_ensure_gcloud_ssh || return 1; # declare vars local so that they don't pollute the shell environment - # where they this func is used. + # where this func is used. local grpc_zone grpc_project dry_run # set by _grpc_set_project_and_zone # grpc_cloud_prod_test_args @@ -1192,6 +1244,20 @@ grpc_cloud_prod_auth_compute_engine_creds_gen_cxx_cmd() { echo $the_cmd } +# constructs the full dockerized cpp jwt_token auth interop test cmd. +# +# call-seq: +# flags= .... # generic flags to include the command +# cmd=$($grpc_gen_test_cmd $flags) +grpc_cloud_prod_auth_jwt_token_creds_gen_cxx_cmd() { + local cmd_prefix="sudo docker run grpc/cxx"; + local test_script="/var/local/git/grpc/bins/opt/interop_client --enable_ssl --use_prod_roots"; + local gfe_flags=$(_grpc_prod_gfe_flags) + local added_gfe_flags=$(_grpc_jwt_token_test_flags) + local the_cmd="$cmd_prefix $test_script $gfe_flags $added_gfe_flags $@"; + echo $the_cmd +} + # constructs the full dockerized csharp-mono interop test cmd. # # call-seq: @@ -1230,6 +1296,11 @@ _grpc_svc_acc_test_flags() { echo " --service_account_key_file=/service_account/stubbyCloudTestingTest-7dd63462c60c.json --oauth_scope=https://www.googleapis.com/auth/xapi.zoo" } +# outputs the flags passed to the service account auth tests +_grpc_jwt_token_test_flags() { + echo " --service_account_key_file=/service_account/stubbyCloudTestingTest-7dd63462c60c.json" +} + # default credentials test flag _grpc_default_creds_test_flags() { echo " --oauth_scope=https://www.googleapis.com/auth/xapi.zoo" diff --git a/tools/gce_setup/interop_test_runner.sh b/tools/gce_setup/interop_test_runner.sh index 430ad09b8c..7f0b5bab1a 100755 --- a/tools/gce_setup/interop_test_runner.sh +++ b/tools/gce_setup/interop_test_runner.sh @@ -37,7 +37,7 @@ main() { source grpc_docker.sh test_cases=(large_unary empty_unary ping_pong client_streaming server_streaming cancel_after_begin cancel_after_first_response) clients=(cxx java go ruby node python csharp_mono) - servers=(cxx java go ruby node python) + servers=(cxx java go ruby node python csharp_mono) for test_case in "${test_cases[@]}" do for client in "${clients[@]}" diff --git a/tools/run_tests/build_python.sh b/tools/run_tests/build_python.sh index 0eba1c6377..b14597804f 100755 --- a/tools/run_tests/build_python.sh +++ b/tools/run_tests/build_python.sh @@ -38,5 +38,5 @@ rm -rf python2.7_virtual_environment virtualenv -p /usr/bin/python2.7 python2.7_virtual_environment source python2.7_virtual_environment/bin/activate pip install enum34==1.0.4 futures==2.2.0 protobuf==3.0.0-alpha-1 -CFLAGS=-I$root/include LDFLAGS=-L$root/libs/opt pip install src/python/src +CFLAGS=-I$root/include LDFLAGS=-L$root/libs/$CONFIG pip install src/python/src pip install src/python/interop diff --git a/tools/run_tests/jobset.py b/tools/run_tests/jobset.py index 26caf031c3..81cdd0e6e4 100755 --- a/tools/run_tests/jobset.py +++ b/tools/run_tests/jobset.py @@ -192,7 +192,7 @@ class Job(object): self._tempfile.seek(0) stdout = self._tempfile.read() message('FAILED', '%s [ret=%d]' % ( - self._spec.shortname, self._process.returncode), stdout) + self._spec.shortname, self._process.returncode), stdout, do_newline=True) else: self._state = _SUCCESS message('PASSED', '%s [time=%.1fsec]' % (self._spec.shortname, elapsed), @@ -200,7 +200,7 @@ class Job(object): if self._bin_hash: update_cache.finished(self._spec.identity(), self._bin_hash) elif self._state == _RUNNING and time.time() - self._start > 300: - message('TIMEOUT', self._spec.shortname, do_newline=self._travis) + message('TIMEOUT', self._spec.shortname, do_newline=True) self.kill() return self._state diff --git a/tools/run_tests/python_tests.json b/tools/run_tests/python_tests.json index ef483d9799..dff053784d 100755 --- a/tools/run_tests/python_tests.json +++ b/tools/run_tests/python_tests.json @@ -27,7 +27,7 @@ "module": "grpc.early_adopter.implementations_test" }, { - "module": "grpc.framework.base.packets.implementations_test" + "module": "grpc.framework.base.implementations_test" }, { "module": "grpc.framework.face.blocking_invocation_inline_service_test" diff --git a/tools/run_tests/run_python.sh b/tools/run_tests/run_python.sh index fa1497aee4..f0e091f47a 100755 --- a/tools/run_tests/run_python.sh +++ b/tools/run_tests/run_python.sh @@ -34,6 +34,6 @@ set -ex cd $(dirname $0)/../.. root=`pwd` -export LD_LIBRARY_PATH=$root/libs/opt +export LD_LIBRARY_PATH=$root/libs/$CONFIG source python2.7_virtual_environment/bin/activate python2.7 -B $* diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index 338f564948..6b8cd1111e 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -240,7 +240,7 @@ _CONFIGS = { } -_DEFAULT = ['dbg', 'opt'] +_DEFAULT = ['opt'] _LANGUAGES = { 'c++': CLanguage('cxx', 'c++'), 'c': CLanguage('c', 'c'), diff --git a/tools/run_tests/tests.json b/tools/run_tests/tests.json index 6f354452aa..355d5735cd 100644 --- a/tools/run_tests/tests.json +++ b/tools/run_tests/tests.json @@ -359,6 +359,11 @@ { "flaky": false, "language": "c++", + "name": "generic_end2end_test" + }, + { + "flaky": false, + "language": "c++", "name": "interop_test" }, { |