Merge github.com:google/grpc into async-api-new

4 files changed, 237 insertions, 64 deletions

diff --git a/tools/dockerfile/grpc_ruby/Dockerfile b/tools/dockerfile/grpc_ruby/Dockerfile
index c84548c880..47972e7eff 100644
--- a/tools/dockerfile/grpc_ruby/Dockerfile
+++ b/tools/dockerfile/grpc_ruby/Dockerfile
@@ -6,17 +6,19 @@ RUN cd /var/local/git/grpc \
   && git pull --recurse-submodules \
   && git submodule update --init --recursive
 
-# TODO: remove this, once make install is fixed
-RUN touch /var/local/git/grpc/include/grpc/support/string.h
-
 # Build the C core.
 RUN make install_c -C /var/local/git/grpc
 
 # Build ruby gRPC and run its tests
 RUN /bin/bash -l -c 'cd /var/local/git/grpc/src/ruby && bundle && rake'
 
-# Add a cacerts directory containing the Google root pem file, allowing the ruby client to access the production test instance
+# Add a cacerts directory containing the Google root pem file, allowing the
+# ruby client to access the production test instance
 ADD cacerts cacerts
 
-# Specify the default command such that the interop server runs on its known testing port
+# Add a service_account directory containing the auth creds file
+ADD service_account service_account
+
+# Specify the default command such that the interop server runs on its known
+# testing port
 CMD ["/bin/bash", "-l", "-c", "ruby /var/local/git/grpc/src/ruby/bin/interop/interop_server.rb --use_tls --port 8060"]
diff --git a/tools/gce_setup/grpc_docker.sh b/tools/gce_setup/grpc_docker.sh
index 2e7ed97a87..2ac75f3cc5 100755
--- a/tools/gce_setup/grpc_docker.sh
+++ b/tools/gce_setup/grpc_docker.sh
@@ -590,7 +590,7 @@ grpc_sync_images() {
   done
 }
 
-grpc_launch_server_args() {
+_grpc_show_servers_args() {
   [[ -n $1 ]] && {  # host
     host=$1
     shift
@@ -598,57 +598,104 @@ grpc_launch_server_args() {
     echo "$FUNCNAME: missing arg: host" 1>&2
     return 1
   }
-
-  [[ -n $1 ]] && {  # server_type
-    case $1 in
-      cxx)    grpc_port=8010 ;;
-      go)     grpc_port=8020 ;;
-      java)   grpc_port=8030 ;;
-      node)   grpc_port=8040 ;;
-      python) grpc_port=8050 ;;
-      ruby)   grpc_port=8060 ;;
-      *) echo "bad server_type: $1" 1>&2; return 1 ;;
-    esac
-    docker_label="grpc/$1"
-    docker_name="grpc_interop_$1"
-    shift
-  } || {
-    echo "$FUNCNAME: missing arg: server_type" 1>&2
-    return 1
-  }
 }
 
-# Launches a server on a docker instance.
+
+# Shows servers on a docker instance.
 #
 # call-seq;
-#   grpc_launch_server <server_name> <server_type>
+#   grpc_show_servers <server_name>
+#   E.g
+#   grpc_show_server grpc-docker-server
 #
-# Runs the server_type on a GCE instance running docker with server_name
-grpc_launch_server() {
+# Shows the grpc servers on the GCE instance <server_name>
+grpc_show_servers() {
   # declare vars local so that they don't pollute the shell environment
   # where they this func is used.
   local grpc_zone grpc_project dry_run  # set by _grpc_set_project_and_zone
-  # set by grpc_launch_server_args
-  local docker_label docker_name host grpc_port
+  # set by _grpc_show_servers
+  local host
 
   # set the project zone and check that all necessary args are provided
-  _grpc_set_project_and_zone -f grpc_launch_server_args "$@" || return 1
+  _grpc_set_project_and_zone -f _grpc_show_servers_args "$@" || return 1
   gce_has_instance $grpc_project $host || return 1;
 
-  cmd="sudo docker kill $docker_name > /dev/null 2>&1; "
-  cmd+="sudo docker rm $docker_name > /dev/null 2>&1; "
-  cmd+="sudo docker run -d --name $docker_name"
-  cmd+=" -p $grpc_port:$grpc_port $docker_label"
-  local project_opt="--project $grpc_project"
-  local zone_opt="--zone $grpc_zone"
+  local cmd="sudo docker ps | grep grpc_"
   local ssh_cmd="bash -l -c \"$cmd\""
   echo "will run:"
   echo "  $ssh_cmd"
   echo "on $host"
-  [[ $dry_run == 1 ]] && return 0  # don't run the command on a dry run
+  [[ $dry_run == 1 ]] && continue  # don't run the command on a dry run
   gcloud compute $project_opt ssh $zone_opt $host --command "$cmd"
 }
 
+_grpc_launch_servers_args() {
+  [[ -n $1 ]] && {  # host
+    host=$1
+    shift
+  } || {
+    echo "$FUNCNAME: missing arg: host" 1>&2
+    return 1
+  }
+  [[ -n $1 ]] && {
+    servers="$@"
+  } || {
+    servers="cxx java go node ruby"
+    echo "$FUNCNAME: no servers specified, will launch defaults '$servers'"
+  }
+}
+
+# Launches servers on a docker instance.
+#
+# call-seq;
+#   grpc_launch_servers <server_name> [server1 server2 ...]
+#   E.g
+#   grpc_launch_server grpc-docker-server ruby node
+#
+# Restarts all the specified servers on the GCE instance <server_name>
+# If no servers are specified, it launches all known servers
+grpc_launch_servers() {
+  # declare vars local so that they don't pollute the shell environment
+  # where they this func is used.
+  local grpc_zone grpc_project dry_run  # set by _grpc_set_project_and_zone
+  # set by _grpc_launch_servers_args
+  local host servers
+
+  # set the project zone and check that all necessary args are provided
+  _grpc_set_project_and_zone -f _grpc_launch_servers_args "$@" || return 1
+  gce_has_instance $grpc_project $host || return 1;
+
+  # launch each of the servers in turn
+  for server in $servers
+  do
+    local grpc_port
+    case $server in
+      cxx)    grpc_port=8010 ;;
+      go)     grpc_port=8020 ;;
+      java)   grpc_port=8030 ;;
+      node)   grpc_port=8040 ;;
+      python) grpc_port=8050 ;;
+      ruby)   grpc_port=8060 ;;
+      *) echo "bad server_type: $1" 1>&2; return 1 ;;
+    esac
+    local docker_label="grpc/$server"
+    local docker_name="grpc_interop_$server"
+
+    cmd="sudo docker kill $docker_name > /dev/null 2>&1; "
+    cmd+="sudo docker rm $docker_name > /dev/null 2>&1; "
+    cmd+="sudo docker run -d --name $docker_name"
+    cmd+=" -p $grpc_port:$grpc_port $docker_label"
+    local project_opt="--project $grpc_project"
+    local zone_opt="--zone $grpc_zone"
+    local ssh_cmd="bash -l -c \"$cmd\""
+    echo "will run:"
+    echo "  $ssh_cmd"
+    echo "on $host"
+    [[ $dry_run == 1 ]] && return 0  # don't run the command on a dry run
+    gcloud compute $project_opt ssh $zone_opt $host --command "$cmd"
+  done
+}
+
 # Runs a test command on a docker instance.
 #
 # call-seq:
@@ -822,7 +869,6 @@ grpc_interop_gen_ruby_cmd() {
   echo $the_cmd
 }
 
-
 # constructs the full dockerized java interop test cmd.
 #
 # call-seq:
@@ -832,12 +878,43 @@ grpc_cloud_prod_gen_ruby_cmd() {
   local cmd_prefix="sudo docker run grpc/ruby bin/bash -l -c"
   local test_script="/var/local/git/grpc/src/ruby/bin/interop/interop_client.rb"
   local test_script+=" --use_tls"
-  local gfe_flags=" --server_port=443 --server_host=grpc-test.sandbox.google.com --server_host_override=grpc-test.sandbox.google.com"
+  local gfe_flags=$(_grpc_prod_gfe_flags)
   local env_prefix="SSL_CERT_FILE=/cacerts/roots.pem"
   local the_cmd="$cmd_prefix '$env_prefix ruby $test_script $gfe_flags $@'"
   echo $the_cmd
 }
 
+# constructs the full dockerized ruby service_account auth interop test cmd.
+#
+# call-seq:
+#   flags= .... # generic flags to include the command
+#   cmd=$($grpc_gen_test_cmd $flags)
+grpc_cloud_prod_auth_service_account_creds_gen_ruby_cmd() {
+  local cmd_prefix="sudo docker run grpc/ruby bin/bash -l -c";
+  local test_script="/var/local/git/grpc/src/ruby/bin/interop/interop_client.rb"
+  local test_script+=" --use_tls"
+  local gfe_flags=$(_grpc_prod_gfe_flags)
+  local added_gfe_flags=$(_grpc_svc_acc_test_flags)
+  local env_prefix="SSL_CERT_FILE=/cacerts/roots.pem"
+  local the_cmd="$cmd_prefix '$env_prefix ruby $test_script $gfe_flags $added_gfe_flag $@'"
+  echo $the_cmd
+}
+
+# constructs the full dockerized ruby gce auth interop test cmd.
+#
+# call-seq:
+#   flags= .... # generic flags to include the command
+#   cmd=$($grpc_gen_test_cmd $flags)
+grpc_cloud_prod_auth_compute_engine_creds_gen_ruby_cmd() {
+  local cmd_prefix="sudo docker run grpc/ruby bin/bash -l -c";
+  local test_script="/var/local/git/grpc/src/ruby/bin/interop/interop_client.rb"
+  local test_script+=" --use_tls"
+  local gfe_flags=$(_grpc_prod_gfe_flags)
+  local added_gfe_flags=$(_grpc_gce_test_flags)
+  local env_prefix="SSL_CERT_FILE=/cacerts/roots.pem"
+  local the_cmd="$cmd_prefix '$env_prefix ruby $test_script $gfe_flags $added_gfe_flag $@'"
+  echo $the_cmd
+}
 
 # constructs the full dockerized Go interop test cmd.
 #
@@ -888,7 +965,7 @@ grpc_cloud_prod_gen_java_cmd() {
     local cmd_prefix="sudo docker run grpc/java";
     local test_script="/var/local/git/grpc-java/run-test-client.sh";
     local test_script+=" --use_tls=true"
-    local gfe_flags=" --server_port=443 --server_host=grpc-test.sandbox.google.com --server_host_override=grpc-test.sandbox.google.com"
+    local gfe_flags=$(_grpc_prod_gfe_flags)
     local the_cmd="$cmd_prefix $test_script $gfe_flags $@";
     echo $the_cmd
 }
@@ -909,19 +986,11 @@ grpc_interop_gen_php_cmd() {
     echo $the_cmd
 }
 
-# constructs the full dockerized cpp interop test cmd.
-#
+# constructs the full dockerized node interop test cmd.
 #
 # call-seq:
 #   flags= .... # generic flags to include the command
 #   cmd=$($grpc_gen_test_cmd $flags)
-grpc_interop_gen_cxx_cmd() {
-    local cmd_prefix="sudo docker run grpc/cxx";
-    local test_script="/var/local/git/grpc/bins/opt/interop_client --enable_ssl";
-    local the_cmd="$cmd_prefix $test_script $@";
-    echo $the_cmd
-}
-
 grpc_interop_gen_node_cmd() {
   local cmd_prefix="sudo docker run grpc/node";
   local test_script="/usr/bin/nodejs /var/local/git/grpc/src/node/interop/interop_client.js --use_tls=true";
@@ -931,46 +1000,70 @@ grpc_interop_gen_node_cmd() {
 
 # constructs the full dockerized cpp interop test cmd.
 #
+# call-seq:
+#   flags= .... # generic flags to include the command
+#   cmd=$($grpc_gen_test_cmd $flags)
+grpc_interop_gen_cxx_cmd() {
+    local cmd_prefix="sudo docker run grpc/cxx";
+    local test_script="/var/local/git/grpc/bins/opt/interop_client --enable_ssl";
+    local the_cmd="$cmd_prefix $test_script $@";
+    echo $the_cmd
+}
+
+# constructs the full dockerized cpp gce=>prod interop test cmd.
 #
 # call-seq:
 #   flags= .... # generic flags to include the command
 #   cmd=$($grpc_gen_test_cmd $flags)
 grpc_cloud_prod_gen_cxx_cmd() {
     local cmd_prefix="sudo docker run grpc/cxx";
-    local test_script="/var/local/git/grpc/bins/opt/interop_client --enable_ssl";
-    local gfe_flags=" --use_prod_roots --server_port=443 --server_host=grpc-test.sandbox.google.com --server_host_override=grpc-test.sandbox.google.com"
+    local test_script="/var/local/git/grpc/bins/opt/interop_client --enable_ssl --use_prod_roots";
+    local gfe_flags=$(_grpc_prod_gfe_flags)
     local the_cmd="$cmd_prefix $test_script $gfe_flags $@";
     echo $the_cmd
 }
 
-# constructs the full dockerized cpp interop test cmd.
-#
+# constructs the full dockerized cpp service_account auth interop test cmd.
 #
 # call-seq:
 #   flags= .... # generic flags to include the command
 #   cmd=$($grpc_gen_test_cmd $flags)
 grpc_cloud_prod_auth_service_account_creds_gen_cxx_cmd() {
     local cmd_prefix="sudo docker run grpc/cxx";
-    local test_script="/var/local/git/grpc/bins/opt/interop_client --enable_ssl";
-    local gfe_flags=" --use_prod_roots --server_port=443 --server_host=grpc-test.sandbox.google.com --server_host_override=grpc-test.sandbox.google.com"
-    local added_gfe_flags=" --service_account_key_file=/service_account/stubbyCloudTestingTest-7dd63462c60c.json --oauth_scope=https://www.googleapis.com/auth/xapi.zoo"
+    local test_script="/var/local/git/grpc/bins/opt/interop_client --enable_ssl --use_prod_roots";
+    local gfe_flags=$(_grpc_prod_gfe_flags)
+    local added_gfe_flags=$(_grpc_svc_acc_test_flags)
     local the_cmd="$cmd_prefix $test_script $gfe_flags $added_gfe_flags $@";
     echo $the_cmd
 }
 
-# constructs the full dockerized cpp interop test cmd.
-#
+# constructs the full dockerized cpp gce auth interop test cmd.
 #
 # call-seq:
 #   flags= .... # generic flags to include the command
 #   cmd=$($grpc_gen_test_cmd $flags)
 grpc_cloud_prod_auth_compute_engine_creds_gen_cxx_cmd() {
     local cmd_prefix="sudo docker run grpc/cxx";
-    local test_script="/var/local/git/grpc/bins/opt/interop_client --enable_ssl";
-    local gfe_flags=" --use_prod_roots --server_port=443 --server_host=grpc-test.sandbox.google.com --server_host_override=grpc-test.sandbox.google.com"
-    local added_gfe_flags=" --default_service_account=155450119199-r5aaqa2vqoa9g5mv2m6s3m1l293rlmel@developer.gserviceaccount.com --oauth_scope=https://www.googleapis.com/auth/xapi.zoo"
+    local test_script="/var/local/git/grpc/bins/opt/interop_client --enable_ssl --use_prod_roots";
+    local gfe_flags=$(_grpc_prod_gfe_flags)
+    local added_gfe_flags=$(_grpc_gce_test_flags)
     local the_cmd="$cmd_prefix $test_script $gfe_flags $added_gfe_flags $@";
     echo $the_cmd
 }
 
-# TODO(grpc-team): add grpc_interop_gen_xxx_cmd for python|nodejs
+# outputs the flags passed to gfe tests
+_grpc_prod_gfe_flags() {
+  echo " --server_port=443 --server_host=grpc-test.sandbox.google.com --server_host_override=grpc-test.sandbox.google.com"
+}
+
+# outputs the flags passed to the service account auth tests
+_grpc_svc_acc_test_flags() {
+  echo " --service_account_key_file=/service_account/stubbyCloudTestingTest-7dd63462c60c.json --oauth_scope=https://www.googleapis.com/auth/xapi.zoo"
+}
+
+# outputs the flags passed to the gcloud auth tests
+_grpc_gce_test_flags() {
+  echo " --default_service_account=155450119199-r5aaqa2vqoa9g5mv2m6s3m1l293rlmel@developer.gserviceaccount.com --oauth_scope=https://www.googleapis.com/auth/xapi.zoo"
+}
+
+# TODO(grpc-team): add grpc_interop_gen_xxx_cmd for python
diff --git a/tools/gce_setup/new_grpc_docker_builder_on_startup.sh b/tools/gce_setup/new_grpc_docker_builder_on_startup.sh
index 4b20a8be7d..cfd05415a0 100755
--- a/tools/gce_setup/new_grpc_docker_builder_on_startup.sh
+++ b/tools/gce_setup/new_grpc_docker_builder_on_startup.sh
@@ -41,7 +41,82 @@ _source_gs_script() {
   source $script_path
 }
 
+# Args:
+#   $1: numerator
+#   $2: denominator
+#   $3: threshold (optional; defaults to $THRESHOLD)
+#
+# Returns:
+#   1 if (numerator / denominator > threshold)
+#   0 otherwise
+_gce_disk_cmp_ratio() {
+  local DEFAULT_THRESHOLD="1.1"
+  local numer="${1}"
+  local denom="${2}"
+  local threshold="${3:-${DEFAULT_THRESHOLD}}"
+
+  if `which python > /dev/null 2>&1`; then
+    python -c "print(1 if (1. * ${numer} / ${denom} > ${threshold}) else 0)"
+  else
+    echo "Can't find python; calculation not done." 1>&2
+    return 1
+  fi
+}
+
+# Repartitions the disk or resizes the file system, depending on the current
+# state of the partition table.
+#
+# Automates the process described in
+# - https://cloud.google.com/compute/docs/disks/persistent-disks#repartitionrootpd
+_gce_disk_maybe_resize_then_reboot() {
+  # Determine the size in blocks, of the whole disk and the first partition.
+  local dev_sda="$(fdisk -s /dev/sda)"
+  local dev_sda1="$(fdisk -s /dev/sda1)"
+  local dev_sda1_start="$(sudo fdisk -l /dev/sda | grep /dev/sda1 | sed -e 's/ \+/ /g' | cut -d' ' -f 3)"
+
+  # Use fdisk to
+  # - first see if the partion 1 is using as much of the disk as it should
+  # - then to resize the partition if it's not
+  #
+  # fdisk(1) flags:
+  # -c: disable DOS compatibility mode
+  # -u: change display mode to sectors (from cylinders)
+  #
+  # fdisk(1) commands:
+  # d: delete partition (automatically selects the first one)
+  # n: new partition
+  # p: primary
+  # 1: partition number
+  # $dev_sda1_start: specify the value for the start sector, the default may be incorrect
+  # <1 blank lines>: accept the defaults for end sectors
+  # w: write partition table
+  if [ $(_gce_disk_cmp_ratio "${dev_sda}" "${dev_sda1}") -eq 1 ]; then
+    echo "$FUNCNAME: Updating the partition table to use full ${dev_sda} instead ${dev_sda1}"
+    cat <<EOF | fdisk -c -u /dev/sda
+d
+n
+p
+1
+$dev_sda1_start
+
+w
+EOF
+    echo "$FUNCNAME: ... updated the partition table"
+    shutdown -r now
+    return 0
+  fi
+
+  # After repartitioning, use resize2fs to expand sda1.
+  local df_size="$(df -B 1K / | grep ' /$' | sed -e 's/ \+/ /g' | cut -d' ' -f 2)"
+  if [ $(_gce_disk_cmp_ratio "${dev_sda}" "${df_size}") -eq 1 ]; then
+    echo "$FUNCNAME: resizing the partition to make full use of it"
+    resize2fs /dev/sda1
+    echo "$FUNCNAME: ... resize completed"
+  fi
+}
+
 main() {
+    _gce_disk_maybe_resize_then_reboot
     local script_attr='shared_startup_script_url'
     _source_gs_script $script_attr || {
       echo "halting, script 'attributes/$script_attr' could not be sourced"
diff --git a/tools/gce_setup/shared_startup_funcs.sh b/tools/gce_setup/shared_startup_funcs.sh
index 3410a2a2c8..eea940864d 100755
--- a/tools/gce_setup/shared_startup_funcs.sh
+++ b/tools/gce_setup/shared_startup_funcs.sh
@@ -372,7 +372,9 @@ grpc_dockerfile_install() {
 
   [[ -d $dockerfile_dir ]] || { echo "$FUNCNAME: not a valid dir: $dockerfile_dir"; return 1; }
 
-  # For specific base images, sync the ssh key into the .ssh dir in the dockerfile context
+  # For specific base images, sync private files.
+  #
+  # - the ssh key, ssh certs and/or service account info.
   [[ $image_label == "grpc/base" ]] && {
     grpc_docker_sync_github_key $dockerfile_dir/.ssh 'base_ssh_key' || return 1;
   }
@@ -384,6 +386,7 @@ grpc_dockerfile_install() {
   }
   [[ $image_label == "grpc/ruby" ]] && {
     grpc_docker_sync_roots_pem $dockerfile_dir/cacerts || return 1;
+    grpc_docker_sync_service_account $dockerfile_dir/service_account || return 1;
   }
   [[ $image_label == "grpc/cxx" ]] && {
     grpc_docker_sync_service_account $dockerfile_dir/service_account || return 1;