From 8a3ca244b789a36edfcfaa7cf1c5f4a5ece1427a Mon Sep 17 00:00:00 2001 From: Tim Emiola Date: Wed, 21 Jan 2015 16:09:11 -0800 Subject: bugfix: corrects a bug in grpc_sync_images --- tools/gce_setup/grpc_docker.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'tools') diff --git a/tools/gce_setup/grpc_docker.sh b/tools/gce_setup/grpc_docker.sh index bf776126b5..314ccb0ca8 100755 --- a/tools/gce_setup/grpc_docker.sh +++ b/tools/gce_setup/grpc_docker.sh @@ -412,7 +412,6 @@ grpc_sync_images() { # declare vars local so that they don't pollute the shell environment # where they this func is used. local grpc_zone grpc_project dry_run # set by _grpc_set_project_and_zone - # set by grpc_sync_images local grpc_hosts # set the project zone and check that all necessary args are provided @@ -425,7 +424,7 @@ grpc_sync_images() { local host for host in $grpc_hosts do - gce_has_instance $grpc_project $h || return 1; + gce_has_instance $grpc_project $host || return 1; local ssh_cmd="bash -l -c \"$cmd\"" echo "will run:" echo " $ssh_cmd" -- cgit v1.2.3 From 48054c0a19a3cd24a213f7b89ad1c31f3f8c293b Mon Sep 17 00:00:00 2001 From: Tim Emiola Date: Thu, 22 Jan 2015 12:15:39 -0800 Subject: Minor bugfixes; adds independent command for support scripts on GCE machines --- tools/gce_setup/grpc_docker.sh | 64 +++++++++++++++++++++++++++++++-- tools/gce_setup/shared_startup_funcs.sh | 18 +++++++--- 2 files changed, 75 insertions(+), 7 deletions(-) (limited to 'tools') diff --git a/tools/gce_setup/grpc_docker.sh b/tools/gce_setup/grpc_docker.sh index 314ccb0ca8..a19915594a 100755 --- a/tools/gce_setup/grpc_docker.sh +++ b/tools/gce_setup/grpc_docker.sh @@ -86,6 +86,7 @@ grpc_add_docker_user() { } _grpc_update_image_args() { + echo "image_args $@" # default the host, root storage uri and docker file root grpc_gs_root='gs://tmp-grpc-dev/admin/' grpc_dockerfile_root='tools/dockerfile' @@ -95,7 +96,7 @@ _grpc_update_image_args() { # see if -p or -z is used to override the the project or zone local OPTIND local OPTARG - while getopts :r:d:h name + while getopts :r:d:h: name do case $name in d) grpc_dockerfile_root=$OPTARG ;; @@ -261,7 +262,7 @@ _grpc_set_project_and_zone() { local OPTIND local OPTARG local arg_func - while getopts :p:z:f:n name + while getopts :np:z:f: name do case $name in f) declare -F $OPTARG >> /dev/null && { @@ -392,6 +393,65 @@ grpc_interop_test_args() { } } +_grpc_sync_scripts_args() { + grpc_gce_script_root='tools/gce_setup' + + local OPTIND + local OPTARG + while getopts :s: name + do + case $name in + s) grpc_gce_script_root=$OPTARG ;; + :) continue ;; # ignore -s without args, just use the defaults + \?) echo "-$OPTARG: unknown flag; it's ignored" 1>&2; continue ;; + esac + done + shift $((OPTIND-1)) + + [[ -d $grpc_gce_script_root ]] || { + echo "Could not locate gce script dir: $grpc_gce_script_root" 1>&2 + return 1 + } + + [[ $# -lt 1 ]] && { + echo "$FUNCNAME: missing arg: host1 [host2 ... hostN]" 1>&2 + return 1 + } + grpc_hosts="$@" +} + +# Updates the latest version of the support scripts on some hosts. +# +# call-seq; +# grpc_sync_scripts , .. +# +# Updates the GCE docker instance +grpc_sync_scripts() { + _grpc_ensure_gcloud_ssh || return 1; + + # declare vars local so that they don't pollute the shell environment + # where they this func is used. + local grpc_zone grpc_project dry_run # set by _grpc_set_project_and_zone + local grpc_hosts grpc_gce_script_root + + # set the project zone and check that all necessary args are provided + _grpc_set_project_and_zone -f _grpc_sync_scripts_args "$@" || return 1 + + local func_lib="shared_startup_funcs.sh" + local gce_func_lib="/var/local/startup_scripts/$func_lib" + local project_opt="--project $grpc_project" + local zone_opt="--zone $grpc_zone" + local host + for host in $grpc_hosts + do + gce_has_instance $grpc_project $host || return 1; + # Update the remote copy of the GCE func library. + local src_func_lib="$grpc_gce_script_root/$func_lib" + local rmt_func_lib="$host:$gce_func_lib" + gcloud compute copy-files $src_func_lib $rmt_func_lib $project_opt $zone_opt || return 1 + done +} + grpc_sync_images_args() { [[ $# -lt 1 ]] && { echo "$FUNCNAME: missing arg: host1 [host2 ... hostN]" 1>&2 diff --git a/tools/gce_setup/shared_startup_funcs.sh b/tools/gce_setup/shared_startup_funcs.sh index 9c747466a9..f9a6aafb09 100755 --- a/tools/gce_setup/shared_startup_funcs.sh +++ b/tools/gce_setup/shared_startup_funcs.sh @@ -367,7 +367,7 @@ grpc_docker_launch_registry() { grpc_docker_pull_known() { local addr=$1 [[ -n $addr ]] || addr="0.0.0.0:5000" - local known="base cxx php_base php ruby_base ruby java_base java" + local known="base cxx php_base php ruby_base ruby java_base java go" echo "... pulling docker images for '$known'" for i in $known do @@ -402,10 +402,15 @@ grpc_dockerfile_install() { [[ -d $dockerfile_dir ]] || { echo "$FUNCNAME: not a valid dir: $dockerfile_dir"; return 1; } - # For grpc/base, sync the ssh key into the .ssh dir in the dockerfile context - + # For specific base images, sync the ssh key into the .ssh dir in the dockerfile context [[ $image_label == "grpc/base" ]] && { - grpc_docker_sync_github_key $dockerfile_dir/.ssh || return 1; + grpc_docker_sync_github_key $dockerfile_dir/.ssh 'base_ssh_key'|| return 1; + } + [[ $image_label == "grpc/go" ]] && { + grpc_docker_sync_github_key $dockerfile_dir/.ssh 'go_ssh_key'|| return 1; + } + [[ $image_label == "grpc/java_base" ]] && { + grpc_docker_sync_github_key $dockerfile_dir/.ssh 'java_base_ssh_key'|| return 1; } # TODO(temiola): maybe make cache/no-cache a func option? @@ -445,6 +450,9 @@ grpc_docker_sync_github_key() { local target_dir=$1 [[ -n $target_dir ]] || { echo "$FUNCNAME: missing arg: target_dir" >&2; return 1; } + local key_file=$2 + [[ -n $key_file ]] || { echo "$FUNCNAME: missing arg: key_file" >&2; return 1; } + # determine the admin root; the parent of the dockerfile root, local gs_dockerfile_root=$(load_metadata "attributes/gs_dockerfile_root") [[ -n $gs_dockerfile_root ]] || { @@ -454,7 +462,7 @@ grpc_docker_sync_github_key() { local gcs_admin_root=$(dirname $gs_dockerfile_root) # cp the file from gsutil to a known local area - local gcs_key_path=$gcs_admin_root/github/ssh_key + local gcs_key_path=$gcs_admin_root/github/$key_file local local_key_path=$target_dir/github.rsa mkdir -p $target_dir || { echo "$FUNCNAME: could not create dir: $target_dir" 1>&2 -- cgit v1.2.3 From 840615ed276c7b94db70e817a4dde729278684e4 Mon Sep 17 00:00:00 2001 From: Nathaniel Manista Date: Thu, 22 Jan 2015 20:31:47 +0000 Subject: Add Python to run-tests. --- .gitignore | 3 +++ INSTALL | 2 +- tools/run_tests/build_python.sh | 10 ++++++++++ tools/run_tests/run_python.sh | 10 ++++++++++ tools/run_tests/run_tests.py | 18 +++++++++++++++++- 5 files changed, 41 insertions(+), 2 deletions(-) create mode 100755 tools/run_tests/build_python.sh create mode 100755 tools/run_tests/run_python.sh (limited to 'tools') diff --git a/.gitignore b/.gitignore index 63332d1b16..002e3e661c 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,9 @@ gens libs objs +# Python virtual environment (pre-3.4 only) +python2.7_virtual_environment + # gcov coverage data coverage *.gcno diff --git a/INSTALL b/INSTALL index 98c20f5898..48511aff7d 100644 --- a/INSTALL +++ b/INSTALL @@ -17,7 +17,7 @@ A typical unix installation won't require any more steps than running: You don't need anything else than GNU Make and gcc. Under a Debian or Ubuntu system, this should boil down to the following package: - # apt-get install build-essential + # apt-get install build-essential python-all-dev python-virtualenv ******************************* diff --git a/tools/run_tests/build_python.sh b/tools/run_tests/build_python.sh new file mode 100755 index 0000000000..6899ac7fe3 --- /dev/null +++ b/tools/run_tests/build_python.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +set -ex + +# change to grpc repo root +cd $(dirname $0)/../.. + +root=`pwd` +virtualenv python2.7_virtual_environment +python2.7_virtual_environment/bin/pip install enum34==1.0.4 futures==2.2.0 diff --git a/tools/run_tests/run_python.sh b/tools/run_tests/run_python.sh new file mode 100755 index 0000000000..0d5ed0238d --- /dev/null +++ b/tools/run_tests/run_python.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +set -ex + +# change to grpc repo root +cd $(dirname $0)/../.. + +root=`pwd` +python2.7_virtual_environment/bin/python2.7 -B -m unittest discover -s src/python -p '*.py' +python3.4 -B -m unittest discover -s src/python -p '*.py' diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index 15c523731b..da849f04cb 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -75,6 +75,21 @@ class PhpLanguage(object): return [['tools/run_tests/build_php.sh']] +class PythonLanguage(object): + + def __init__(self): + self.allow_hashing = False + + def test_binaries(self, config): + return ['tools/run_tests/run_python.sh'] + + def make_targets(self): + return[] + + def build_steps(self): + return [['tools/run_tests/build_python.sh']] + + # different configurations we can run under _CONFIGS = { 'dbg': SimpleConfig('dbg'), @@ -92,7 +107,8 @@ _DEFAULT = ['dbg', 'opt'] _LANGUAGES = { 'c++': CLanguage('cxx', 'c++'), 'c': CLanguage('c', 'c'), - 'php': PhpLanguage() + 'php': PhpLanguage(), + 'python': PythonLanguage(), } # parse command line -- cgit v1.2.3 From fb6d7fd5720773da1d68f05c2d58cd16bdaf7018 Mon Sep 17 00:00:00 2001 From: Tim Emiola Date: Thu, 22 Jan 2015 12:16:12 -0800 Subject: Adds Dockerfile for Go --- tools/dockerfile/grpc_go/Dockerfile | 27 +++++++++++++++++++++++++++ tools/dockerfile/grpc_go/README.md | 4 ++++ 2 files changed, 31 insertions(+) create mode 100644 tools/dockerfile/grpc_go/Dockerfile create mode 100644 tools/dockerfile/grpc_go/README.md (limited to 'tools') diff --git a/tools/dockerfile/grpc_go/Dockerfile b/tools/dockerfile/grpc_go/Dockerfile new file mode 100644 index 0000000000..ab463b2a00 --- /dev/null +++ b/tools/dockerfile/grpc_go/Dockerfile @@ -0,0 +1,27 @@ +# Dockerfile for gRPC Go +FROM golang:1.4 + +# Install SSH to that Go source can be pulled securely. +RUN apt-get update && apt-get install -y ssh + +# Install a GitHub SSH service credential that gives access to the GitHub repo while it's private +# +# TODO: remove this once the repo is public +ADD .ssh .ssh +RUN chmod 600 /.ssh/github.rsa +RUN mkdir -p $HOME/.ssh && echo 'Host github.com' > $HOME/.ssh/config +RUN echo " IdentityFile /.ssh/github.rsa" >> $HOME/.ssh/config +RUN echo 'StrictHostKeyChecking no' >> $HOME/.ssh/config + +# Force go get to use the GitHub ssh url instead of https, and use the SSH creds +RUN git config --global url."git@github.com:".insteadOf "https://github.com/" + +# Get the source from GitHub +RUN go get github.com/google/grpc-go + +# Build the interop client and server +RUN cd src/github.com/google/grpc-go/interop/client && go install +RUN cd src/github.com/google/grpc-go/interop/server && go install + +# Specify the default command such that the interop server runs on its known testing port +CMD ["/bin/bash", "-c 'cd src/github.com/google/grpc-go/interop/server && go run server.go --use_tls=true --port=8020'"] diff --git a/tools/dockerfile/grpc_go/README.md b/tools/dockerfile/grpc_go/README.md new file mode 100644 index 0000000000..0d6ad3e391 --- /dev/null +++ b/tools/dockerfile/grpc_go/README.md @@ -0,0 +1,4 @@ +GRPC Go Dockerfile +================== + +Dockerfile for gRPC Go development, testing and deployment. -- cgit v1.2.3 From c8ee19b9052981321992ae0bab7980422211c162 Mon Sep 17 00:00:00 2001 From: Tim Emiola Date: Thu, 22 Jan 2015 17:11:57 -0800 Subject: Adds a bash func that runs the interop test client command --- tools/gce_setup/grpc_docker.sh | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) (limited to 'tools') diff --git a/tools/gce_setup/grpc_docker.sh b/tools/gce_setup/grpc_docker.sh index a19915594a..bfa4c808be 100755 --- a/tools/gce_setup/grpc_docker.sh +++ b/tools/gce_setup/grpc_docker.sh @@ -634,6 +634,18 @@ grpc_interop_gen_ruby_cmd() { echo $the_cmd } +# constructs the full dockerized Go interop test cmd. +# +# call-seq: +# flags= .... # generic flags to include the command +# cmd=$($grpc_gen_test_cmd $flags) +grpc_interop_gen_go_cmd() { + local cmd_prefix="sudo docker run grpc/go bin/bash -c"; + local test_script="cd /go/src/github.com/google/grpc-go/interop/client"; + local test_script+=" && go run client.go --use_tls=true"; + local the_cmd="$cmd_prefix '$test_script $@ 1>&2'"; +} + # constructs the full dockerized java interop test cmd. # # call-seq: @@ -664,4 +676,4 @@ grpc_interop_gen_php_cmd() { } -# TODO(grpc-team): add grpc_interop_gen_xxx_cmd for python|cxx|nodejs|go +# TODO(grpc-team): add grpc_interop_gen_xxx_cmd for python|cxx|nodejs -- cgit v1.2.3 From 1a8582f18e1e52151b30845543a2a6b534651fcd Mon Sep 17 00:00:00 2001 From: Tim Emiola Date: Thu, 22 Jan 2015 17:12:44 -0800 Subject: removes noisy output when syncing --- tools/gce_setup/shared_startup_funcs.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'tools') diff --git a/tools/gce_setup/shared_startup_funcs.sh b/tools/gce_setup/shared_startup_funcs.sh index f9a6aafb09..f1dbca9a2e 100755 --- a/tools/gce_setup/shared_startup_funcs.sh +++ b/tools/gce_setup/shared_startup_funcs.sh @@ -371,7 +371,8 @@ grpc_docker_pull_known() { echo "... pulling docker images for '$known'" for i in $known do - sudo docker pull ${addr}/grpc/$i \ + echo "<--- grpc/$i" + sudo docker pull ${addr}/grpc/$i > /dev/null 2>&1 \ && sudo docker tag ${addr}/grpc/$i grpc/$i || { # log and continue echo "docker op error: could not pull ${addr}/grpc/$i" -- cgit v1.2.3