aboutsummaryrefslogtreecommitdiffhomepage
path: root/tools
diff options
context:
space:
mode:
Diffstat (limited to 'tools')
-rwxr-xr-xtools/clang-format/clang-format-all.sh9
-rw-r--r--tools/clang-format/config.sh11
-rw-r--r--tools/dockerfile/grpc_java/Dockerfile5
-rw-r--r--tools/dockerfile/grpc_java_base/Dockerfile18
-rw-r--r--tools/dockerfile/grpc_ruby/Dockerfile8
-rwxr-xr-xtools/gce_setup/cloud_prod_runner.sh24
-rwxr-xr-xtools/gce_setup/grpc_docker.sh144
-rwxr-xr-xtools/gce_setup/interop_test_runner.sh28
-rwxr-xr-xtools/gce_setup/shared_startup_funcs.sh38
-rwxr-xr-xtools/run_tests/build_node.sh7
-rwxr-xr-xtools/run_tests/build_php.sh8
-rwxr-xr-xtools/run_tests/run_lcov.sh2
-rwxr-xr-xtools/run_tests/run_node.sh10
-rwxr-xr-xtools/run_tests/run_tests.py24
14 files changed, 286 insertions, 50 deletions
diff --git a/tools/clang-format/clang-format-all.sh b/tools/clang-format/clang-format-all.sh
deleted file mode 100755
index 62228b0b01..0000000000
--- a/tools/clang-format/clang-format-all.sh
+++ /dev/null
@@ -1,9 +0,0 @@
-#!/bin/bash
-set -e
-source $(dirname $0)/config.sh
-cd $(dirname $0)/../..
-for dir in src test include
-do
- find $dir -name '*.c' -or -name '*.cc' -or -name '*.h' | xargs $CLANG_FORMAT -i
-done
-
diff --git a/tools/clang-format/config.sh b/tools/clang-format/config.sh
deleted file mode 100644
index 3adf2678e7..0000000000
--- a/tools/clang-format/config.sh
+++ /dev/null
@@ -1,11 +0,0 @@
-CLANG_FORMAT=clang-format-3.5
-
-set -ex
-
-if not hash $CLANG_FORMAT 2>/dev/null; then
- echo "$CLANG_FORMAT is needed but not installed"
- echo "perhaps try:"
- echo " sudo apt-get install $CLANG_FORMAT"
- exit 1
-fi
-
diff --git a/tools/dockerfile/grpc_java/Dockerfile b/tools/dockerfile/grpc_java/Dockerfile
index 78659dedeb..f234f514e6 100644
--- a/tools/dockerfile/grpc_java/Dockerfile
+++ b/tools/dockerfile/grpc_java/Dockerfile
@@ -1,9 +1,6 @@
# Dockerfile for the gRPC Java dev image
FROM grpc/java_base
-# Start the daemon that allows access to private git-on-borg repos
-RUN /var/local/git/gcompute-tools/git-cookie-authdaemon
-
RUN cd /var/local/git/grpc-java/lib/okhttp && \
mvn -pl okhttp -am install
RUN cd /var/local/git/grpc-java/lib/netty && \
@@ -13,4 +10,4 @@ RUN cd /var/local/git/grpc-java && \
mvn install
# Specify the default command such that the interop server runs on its known testing port
-CMD ["/var/local/git/grpc-java/run-test-server.sh", "--transport=HTTP2_NETTY_TLS", "--grpc_version=2", "--port=8030"]
+CMD ["/var/local/git/grpc-java/run-test-server.sh", "--use_tls=true", "--port=8030"]
diff --git a/tools/dockerfile/grpc_java_base/Dockerfile b/tools/dockerfile/grpc_java_base/Dockerfile
index 44fa52c0e8..3271d1b2c2 100644
--- a/tools/dockerfile/grpc_java_base/Dockerfile
+++ b/tools/dockerfile/grpc_java_base/Dockerfile
@@ -20,14 +20,24 @@ ENV M2_HOME /var/local/apache-maven-3.2.1
ENV PATH $PATH:$JAVA_HOME/bin:$M2_HOME/bin
ENV LD_LIBRARY_PATH /usr/local/lib
-# Start the daemon that allows access to the protected git-on-borg repos
-RUN /var/local/git/gcompute-tools/git-cookie-authdaemon
+# Install a GitHub SSH service credential that gives access to the GitHub repo while it's private
+# TODO: remove this once the repo is public
+ADD .ssh .ssh
+RUN chmod 600 .ssh/github.rsa
+RUN mkdir -p $HOME/.ssh && echo 'Host github.com' > $HOME/.ssh/config
+RUN echo " IdentityFile /.ssh/github.rsa" >> $HOME/.ssh/config
+RUN echo 'StrictHostKeyChecking no' >> $HOME/.ssh/config
-RUN git clone --recursive https://team.googlesource.com/one-platform-grpc-team/grpc-java /var/local/git/grpc-java
+# Get the protobuf source from GitHub and install it
+RUN git clone --recursive --branch v2.6.1 git@github.com:google/protobuf.git /var/local/git/protobuf
+RUN cd /var/local/git/protobuf && \
+ ./autogen.sh && \
+ ./configure --prefix=/usr && \
+ make -j12 && make check && make install && make clean
RUN cd /var/local/git/grpc-java/lib/okhttp && \
mvn -pl okhttp -am validate
RUN cd /var/local/git/grpc-java/lib/netty && \
mvn -pl codec-http2 -am validate
RUN cd /var/local/git/grpc-java && \
- mvn validate \ No newline at end of file
+ mvn validate
diff --git a/tools/dockerfile/grpc_ruby/Dockerfile b/tools/dockerfile/grpc_ruby/Dockerfile
index c677ceffff..f01f81d539 100644
--- a/tools/dockerfile/grpc_ruby/Dockerfile
+++ b/tools/dockerfile/grpc_ruby/Dockerfile
@@ -6,6 +6,9 @@ RUN cd /var/local/git/grpc \
&& git pull --recurse-submodules \
&& git submodule update --init --recursive
+# TODO: remove this, once make install is fixed
+RUN touch /var/local/git/grpc/include/grpc/support/string.h
+
# Build the C core.
RUN make install_c -C /var/local/git/grpc
@@ -18,5 +21,8 @@ RUN /bin/bash -l -c 'cd /var/local/git/grpc/src/ruby && bundle && rake compile:g
# - however, the interop server and client run OK, so this bug can be investigated
# RUN /bin/bash -l -c 'cd /var/local/git/grpc/src/ruby && bundle && rake'
+# Add a cacerts directory containing the Google root pem file, allowing the ruby client to access the production test instance
+ADD cacerts cacerts
+
# Specify the default command such that the interop server runs on its known testing port
-CMD ["/bin/bash", "-l", "-c", "ruby /var/local/git/grpc/src/ruby/bin/interop/interop_server.rb --port 8060"]
+CMD ["/bin/bash", "-l", "-c", "ruby /var/local/git/grpc/src/ruby/bin/interop/interop_server.rb --use_tls --port 8060"]
diff --git a/tools/gce_setup/cloud_prod_runner.sh b/tools/gce_setup/cloud_prod_runner.sh
new file mode 100755
index 0000000000..0c1163ad7d
--- /dev/null
+++ b/tools/gce_setup/cloud_prod_runner.sh
@@ -0,0 +1,24 @@
+#!/bin/bash
+
+main() {
+ source grpc_docker.sh
+ test_cases=(large_unary empty_unary client_streaming server_streaming)
+ clients=(cxx java go ruby)
+ for test_case in "${test_cases[@]}"
+ do
+ for client in "${clients[@]}"
+ do
+ if grpc_cloud_prod_test $test_case grpc-docker-testclients $client
+ then
+ echo "$test_case $client $server passed" >> /tmp/cloud_prod_result.txt
+ else
+ echo "$test_case $client $server failed" >> /tmp/cloud_prod_result.txt
+ fi
+ done
+ done
+ gsutil cp /tmp/cloud_prod_result.txt gs://stoked-keyword-656-output/cloud_prod_result.txt
+ rm /tmp/cloud_prod_result.txt
+}
+
+set -x
+main "$@"
diff --git a/tools/gce_setup/grpc_docker.sh b/tools/gce_setup/grpc_docker.sh
index 145685305c..16d6ef40e3 100755
--- a/tools/gce_setup/grpc_docker.sh
+++ b/tools/gce_setup/grpc_docker.sh
@@ -393,6 +393,53 @@ grpc_interop_test_args() {
}
}
+# checks the positional args and assigns them to variables visible in the caller
+#
+# these are the positional args passed to grpc_cloud_prod_test after option flags
+# are removed
+#
+# three args are expected, in order
+# - test_case
+# - host <the gce docker instance on which to run the test>
+# - client to run
+grpc_cloud_prod_test_args() {
+ [[ -n $1 ]] && { # test_case
+ test_case=$1
+ shift
+ } || {
+ echo "$FUNCNAME: missing arg: test_case" 1>&2
+ return 1
+ }
+
+ [[ -n $1 ]] && { # host
+ host=$1
+ shift
+ } || {
+ echo "$FUNCNAME: missing arg: host" 1>&2
+ return 1
+ }
+
+ [[ -n $1 ]] && { # client_type
+ case $1 in
+ cxx|go|java|nodejs|php|python|ruby)
+ grpc_gen_test_cmd="grpc_cloud_prod_gen_$1_cmd"
+ declare -F $grpc_gen_test_cmd >> /dev/null || {
+ echo "-f: test_func for $1 => $grpc_gen_test_cmd is not defined" 1>&2
+ return 2
+ }
+ shift
+ ;;
+ *)
+ echo "bad client_type: $1" 1>&2
+ return 1
+ ;;
+ esac
+ } || {
+ echo "$FUNCNAME: missing arg: client_type" 1>&2
+ return 1
+ }
+}
+
_grpc_sync_scripts_args() {
grpc_gce_script_root='tools/gce_setup'
@@ -622,6 +669,52 @@ grpc_interop_test() {
gcloud compute $project_opt ssh $zone_opt $host --command "$cmd"
}
+# Runs a test command on a docker instance.
+#
+# call-seq:
+# grpc_cloud_prod_test <test_name> <host> <client_type>
+#
+# requirements:
+# host is a GCE instance running docker with access to the gRPC docker images
+# test_name is one of the named gRPC tests [http://go/grpc_interop_tests]
+# client_type is one of [cxx,go,java,php,python,ruby]
+#
+# it assumes:
+# that each grpc-imp has a docker image named grpc/<imp>, e.g, grpc/java
+# a test is run using $ docker run 'path/to/interop_test_bin --flags'
+# the required images are available on <host>
+#
+# each client_type should have an associated bash func:
+# grpc_cloud_prod_gen_<client_type>_cmd
+# the func provides the dockerized commmand for running client_type's test.
+# If no such func is available, tests for that client type cannot be run.
+grpc_cloud_prod_test() {
+ _grpc_ensure_gcloud_ssh || return 1;
+ # declare vars local so that they don't pollute the shell environment
+ # where they this func is used.
+
+ local grpc_zone grpc_project dry_run # set by _grpc_set_project_and_zone
+ # grpc_cloud_prod_test_args
+ local test_case host grpc_gen_test_cmd
+
+ # set the project zone and check that all necessary args are provided
+ _grpc_set_project_and_zone -f grpc_cloud_prod_test_args "$@" || return 1
+ gce_has_instance $grpc_project $host || return 1;
+
+ local test_case_flag=" --test_case=$test_case"
+ cmd=$($grpc_gen_test_cmd $test_case_flag)
+ [[ -n $cmd ]] || return 1
+
+ local project_opt="--project $grpc_project"
+ local zone_opt="--zone $grpc_zone"
+ local ssh_cmd="bash -l -c \"$cmd\""
+ echo "will run:"
+ echo " $ssh_cmd"
+ echo "on $host"
+ [[ $dry_run == 1 ]] && return 0 # don't run the command on a dry run
+ gcloud compute $project_opt ssh $zone_opt $host --command "$cmd"
+}
+
# constructs the full dockerized ruby interop test cmd.
#
# call-seq:
@@ -630,10 +723,27 @@ grpc_interop_test() {
grpc_interop_gen_ruby_cmd() {
local cmd_prefix="sudo docker run grpc/ruby bin/bash -l -c"
local test_script="/var/local/git/grpc/src/ruby/bin/interop/interop_client.rb"
- local the_cmd="$cmd_prefix 'ruby $test_script $@'"
+ local the_cmd="$cmd_prefix 'ruby $test_script --use_test_ca --use_tls $@'"
+ echo $the_cmd
+}
+
+
+# constructs the full dockerized java interop test cmd.
+#
+# call-seq:
+# flags= .... # generic flags to include the command
+# cmd=$($grpc_gen_test_cmd $flags)
+grpc_cloud_prod_gen_ruby_cmd() {
+ local cmd_prefix="sudo docker run grpc/ruby bin/bash -l -c"
+ local test_script="/var/local/git/grpc/src/ruby/bin/interop/interop_client.rb"
+ local test_script+=" --use_tls"
+ local gfe_flags=" --server_port=443 --server_host=grpc-test.sandbox.google.com --server_host_override=grpc-test.sandbox.google.com"
+ local env_prefix="SSL_CERT_FILE=/cacerts/roots.pem"
+ local the_cmd="$cmd_prefix '$env_prefix ruby $test_script $gfe_flags $@'"
echo $the_cmd
}
+
# constructs the full dockerized Go interop test cmd.
#
# call-seq:
@@ -655,11 +765,25 @@ grpc_interop_gen_go_cmd() {
grpc_interop_gen_java_cmd() {
local cmd_prefix="sudo docker run grpc/java";
local test_script="/var/local/git/grpc-java/run-test-client.sh";
- local test_script+=" --transport=NETTY_TLS --grpc_version=2"
+ local test_script+=" --server_host_override=foo.test.google.fr --use_test_ca=true --use_tls=true"
local the_cmd="$cmd_prefix $test_script $@";
echo $the_cmd
}
+# constructs the full dockerized java interop test cmd.
+#
+# call-seq:
+# flags= .... # generic flags to include the command
+# cmd=$($grpc_gen_test_cmd $flags)
+grpc_cloud_prod_gen_java_cmd() {
+ local cmd_prefix="sudo docker run grpc/java";
+ local test_script="/var/local/git/grpc-java/run-test-client.sh";
+ local test_script+=" --use_tls=true"
+ local gfe_flags=" --server_port=443 --server_host=grpc-test.sandbox.google.com --server_host_override=grpc-test.sandbox.google.com"
+ local the_cmd="$cmd_prefix $test_script $gfe_flags $@";
+ echo $the_cmd
+}
+
# constructs the full dockerized php interop test cmd.
#
# TODO(mlumish): update this to use the script once that's on git-on-borg
@@ -683,10 +807,24 @@ grpc_interop_gen_php_cmd() {
# flags= .... # generic flags to include the command
# cmd=$($grpc_gen_test_cmd $flags)
grpc_interop_gen_cxx_cmd() {
- local cmd_prefix="sudo docker run grpc/cxx";
+ local cmd_prefix="sudo docker run grpc/cxx";
local test_script="/var/local/git/grpc/bins/opt/interop_client --enable_ssl";
local the_cmd="$cmd_prefix $test_script $@";
echo $the_cmd
}
+# constructs the full dockerized cpp interop test cmd.
+#
+#
+# call-seq:
+# flags= .... # generic flags to include the command
+# cmd=$($grpc_gen_test_cmd $flags)
+grpc_cloud_prod_gen_cxx_cmd() {
+ local cmd_prefix="sudo docker run grpc/cxx";
+ local test_script="/var/local/git/grpc/bins/opt/interop_client --enable_ssl";
+ local gfe_flags=" --use_prod_roots --server_port=443 --server_host=grpc-test.sandbox.google.com --server_host_override=grpc-test.sandbox.google.com"
+ local the_cmd="$cmd_prefix $test_script $gfe_flags $@";
+ echo $the_cmd
+}
+
# TODO(grpc-team): add grpc_interop_gen_xxx_cmd for python|cxx|nodejs
diff --git a/tools/gce_setup/interop_test_runner.sh b/tools/gce_setup/interop_test_runner.sh
new file mode 100755
index 0000000000..1c0d82095c
--- /dev/null
+++ b/tools/gce_setup/interop_test_runner.sh
@@ -0,0 +1,28 @@
+#!/bin/bash
+
+main() {
+ source grpc_docker.sh
+ test_cases=(large_unary empty_unary ping_pong client_streaming server_streaming)
+ clients=(cxx java go ruby)
+ servers=(cxx java go ruby)
+ for test_case in "${test_cases[@]}"
+ do
+ for client in "${clients[@]}"
+ do
+ for server in "${servers[@]}"
+ do
+ if grpc_interop_test $test_case grpc-docker-testclients $client grpc-docker-server $server
+ then
+ echo "$test_case $client $server passed" >> /tmp/interop_result.txt
+ else
+ echo "$test_case $client $server failed" >> /tmp/interop_result.txt
+ fi
+ done
+ done
+ done
+ gsutil cp /tmp/interop_result.txt gs://stoked-keyword-656-output/interop_result.txt
+ rm /tmp/interop_result.txt
+}
+
+set -x
+main "$@"
diff --git a/tools/gce_setup/shared_startup_funcs.sh b/tools/gce_setup/shared_startup_funcs.sh
index f1dbca9a2e..69f6ba8cc0 100755
--- a/tools/gce_setup/shared_startup_funcs.sh
+++ b/tools/gce_setup/shared_startup_funcs.sh
@@ -405,14 +405,18 @@ grpc_dockerfile_install() {
# For specific base images, sync the ssh key into the .ssh dir in the dockerfile context
[[ $image_label == "grpc/base" ]] && {
- grpc_docker_sync_github_key $dockerfile_dir/.ssh 'base_ssh_key'|| return 1;
+ grpc_docker_sync_github_key $dockerfile_dir/.ssh 'base_ssh_key' || return 1;
}
[[ $image_label == "grpc/go" ]] && {
- grpc_docker_sync_github_key $dockerfile_dir/.ssh 'go_ssh_key'|| return 1;
+ grpc_docker_sync_github_key $dockerfile_dir/.ssh 'go_ssh_key' || return 1;
}
[[ $image_label == "grpc/java_base" ]] && {
- grpc_docker_sync_github_key $dockerfile_dir/.ssh 'java_base_ssh_key'|| return 1;
+ grpc_docker_sync_github_key $dockerfile_dir/.ssh 'java_base_ssh_key' || return 1;
}
+ [[ $image_label == "grpc/ruby" ]] && {
+ grpc_docker_sync_roots_pem $dockerfile_dir/cacerts || return 1;
+ }
+
# TODO(temiola): maybe make cache/no-cache a func option?
sudo docker build $cache_opt -t $image_label $dockerfile_dir || {
@@ -471,3 +475,31 @@ grpc_docker_sync_github_key() {
}
gsutil cp $src $gcs_key_path $local_key_path
}
+
+# grpc_docker_sync_roots_pem.
+#
+# Copies the root pems from GCS to the target dir
+#
+# call-seq:
+# grpc_docker_sync_roots_pem <target_dir>
+grpc_docker_sync_roots_pem() {
+ local target_dir=$1
+ [[ -n $target_dir ]] || { echo "$FUNCNAME: missing arg: target_dir" >&2; return 1; }
+
+ # determine the admin root; the parent of the dockerfile root,
+ local gs_dockerfile_root=$(load_metadata "attributes/gs_dockerfile_root")
+ [[ -n $gs_dockerfile_root ]] || {
+ echo "$FUNCNAME: missing metadata: gs_dockerfile_root" >&2
+ return 1
+ }
+ local gcs_admin_root=$(dirname $gs_dockerfile_root)
+
+ # cp the file from gsutil to a known local area
+ local gcs_certs_path=$gcs_admin_root/cacerts/roots.pem
+ local local_certs_path=$target_dir/roots.pem
+ mkdir -p $target_dir || {
+ echo "$FUNCNAME: could not create dir: $target_dir" 1>&2
+ return 1
+ }
+ gsutil cp $src $gcs_certs_path $local_certs_path
+}
diff --git a/tools/run_tests/build_node.sh b/tools/run_tests/build_node.sh
index 600b1bde8c..4b092982b2 100755
--- a/tools/run_tests/build_node.sh
+++ b/tools/run_tests/build_node.sh
@@ -2,19 +2,18 @@
set -ex
+CONFIG=${CONFIG:-opt}
+
# change to grpc repo root
cd $(dirname $0)/../..
# tells npm install to look for files in that directory
export GRPC_ROOT=`pwd`
# tells npm install the subdirectory with library files
-export GRPC_LIB_SUBDIR=libs/opt
+export GRPC_LIB_SUBDIR=libs/$CONFIG
# tells npm install not to use default locations
export GRPC_NO_INSTALL=yes
-# build the c libraries
-make -j static_c
-
cd src/node
npm install
diff --git a/tools/run_tests/build_php.sh b/tools/run_tests/build_php.sh
index 6841656bdb..0a8d0c7492 100755
--- a/tools/run_tests/build_php.sh
+++ b/tools/run_tests/build_php.sh
@@ -2,14 +2,13 @@
set -ex
+CONFIG=${CONFIG:-opt}
+
# change to grpc repo root
cd $(dirname $0)/../..
root=`pwd`
-export GRPC_LIB_SUBDIR=libs/opt
-
-# make the libraries
-make -j static_c
+export GRPC_LIB_SUBDIR=libs/$CONFIG
# build php
cd src/php
@@ -18,4 +17,3 @@ cd ext/grpc
phpize
./configure --enable-grpc=$root
make
-
diff --git a/tools/run_tests/run_lcov.sh b/tools/run_tests/run_lcov.sh
index 6f22b0e8a4..068213a3d2 100755
--- a/tools/run_tests/run_lcov.sh
+++ b/tools/run_tests/run_lcov.sh
@@ -7,7 +7,7 @@ out=`realpath ${1:-coverage}`
root=`realpath $(dirname $0)/../..`
tmp=`mktemp`
cd $root
-tools/run_tests/run_tests.py -c gcov
+tools/run_tests/run_tests.py -c gcov -l c c++
lcov --capture --directory . --output-file $tmp
genhtml $tmp --output-directory $out
rm $tmp
diff --git a/tools/run_tests/run_node.sh b/tools/run_tests/run_node.sh
new file mode 100755
index 0000000000..0056295949
--- /dev/null
+++ b/tools/run_tests/run_node.sh
@@ -0,0 +1,10 @@
+#!/bin/bash
+
+set -ex
+
+# change to grpc repo root
+cd $(dirname $0)/../..
+
+root=`pwd`
+
+$root/src/node/node_modules/mocha/bin/mocha $root/src/node/test
diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py
index da849f04cb..b7248e524b 100755
--- a/tools/run_tests/run_tests.py
+++ b/tools/run_tests/run_tests.py
@@ -46,8 +46,8 @@ class CLanguage(object):
self.make_target = make_target
with open('tools/run_tests/tests.json') as f:
js = json.load(f)
- self.binaries = [tgt['name']
- for tgt in js
+ self.binaries = [tgt['name']
+ for tgt in js
if tgt['language'] == test_lang]
def test_binaries(self, config):
@@ -59,6 +59,19 @@ class CLanguage(object):
def build_steps(self):
return []
+class NodeLanguage(object):
+
+ def __init__(self):
+ self.allow_hashing = False
+
+ def test_binaries(self, config):
+ return ['tools/run_tests/run_node.sh']
+
+ def make_targets(self):
+ return ['static_c']
+
+ def build_steps(self):
+ return [['tools/run_tests/build_node.sh']]
class PhpLanguage(object):
@@ -69,7 +82,7 @@ class PhpLanguage(object):
return ['src/php/bin/run_tests.sh']
def make_targets(self):
- return []
+ return ['static_c']
def build_steps(self):
return [['tools/run_tests/build_php.sh']]
@@ -107,6 +120,7 @@ _DEFAULT = ['dbg', 'opt']
_LANGUAGES = {
'c++': CLanguage('cxx', 'c++'),
'c': CLanguage('c', 'c'),
+ 'node': NodeLanguage(),
'php': PhpLanguage(),
'python': PythonLanguage(),
}
@@ -190,8 +204,8 @@ class TestCache(object):
def _build_and_run(check_cancelled, newline_on_success, cache):
"""Do one pass of building & running tests."""
- # build latest, sharing cpu between the various makes
- if not jobset.run(build_steps):
+ # build latest sequentially
+ if not jobset.run(build_steps, maxjobs=1):
return 1
# run all the tests
generated by cgit v1.2.3 (git 2.39.1) at 2024-12-25 09:31:57 +0000