Merge pull request #5213 from ctiller/fix-proto-docker

Proto API for LB request/responses (redux)

9 files changed, 285 insertions, 4 deletions

diff --git a/tools/codegen/core/gen_load_balancing_proto.sh b/tools/codegen/core/gen_load_balancing_proto.sh
new file mode 100755
index 0000000000..fb6a468ee0
--- /dev/null
+++ b/tools/codegen/core/gen_load_balancing_proto.sh
@@ -0,0 +1,138 @@
+#!/bin/bash
+
+# Copyright 2016, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#
+# Example usage:
+#   tools/codegen/core/gen_load_balancing_proto.sh \
+#     src/proto/grpc/lb/v0/load_balancer.proto
+
+read -r -d '' COPYRIGHT <<'EOF'
+/*
+ *
+ * Copyright <YEAR>, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+EOF
+
+CURRENT_YEAR=$(date +%Y)
+COPYRIGHT_FILE=$(mktemp)
+echo "${COPYRIGHT/<YEAR>/$CURRENT_YEAR}" > $COPYRIGHT_FILE
+
+set -ex
+if [ $# -eq 0 ]; then
+  echo "Usage: $0 <load_balancer.proto> [output dir]"
+  exit 1
+fi
+
+readonly GRPC_ROOT=$PWD
+
+OUTPUT_DIR="$GRPC_ROOT/src/core/proto/grpc/lb/v0"
+if [ $# -eq 2 ]; then
+  mkdir -p "$2"
+  if [ $? != 0 ]; then
+    echo "Error creating output directory $2"
+    exit 2
+  fi
+  OUTPUT_DIR="$2"
+fi
+
+readonly EXPECTED_OPTIONS_FILE_PATH="${1%.*}.options"
+
+if [[ ! -f "$1" ]]; then
+  echo "Input proto file '$1' doesn't exist."
+  exit 3
+fi
+if [[ ! -f "${EXPECTED_OPTIONS_FILE_PATH}" ]]; then
+  echo "Expected nanopb options file '${EXPECTED_OPTIONS_FILE_PATH}' missing"
+  exit 4
+fi
+
+readonly VENV_DIR=$(mktemp -d)
+readonly VENV_NAME="nanopb-$(date '+%Y%m%d_%H%M%S_%N')"
+pushd $VENV_DIR
+virtualenv $VENV_NAME
+. $VENV_NAME/bin/activate
+popd
+
+# this should be the same version as the submodule we compile against
+# ideally we'd update this as a template to ensure that
+pip install protobuf==3.0.0b2
+
+pushd "$(dirname $1)" > /dev/null
+
+protoc \
+--plugin=protoc-gen-nanopb="$GRPC_ROOT/third_party/nanopb/generator/protoc-gen-nanopb" \
+--nanopb_out='-T -L#include\ \"third_party/nanopb/pb.h\"'":$OUTPUT_DIR" \
+"$(basename $1)"
+
+readonly PROTO_BASENAME=$(basename $1 .proto)
+sed -i "s:$PROTO_BASENAME.pb.h:src/core/proto/grpc/lb/v0/$PROTO_BASENAME.pb.h:g" \
+    "$OUTPUT_DIR/$PROTO_BASENAME.pb.c"
+
+# prepend copyright
+TMPFILE=$(mktemp)
+cat $COPYRIGHT_FILE "$OUTPUT_DIR/$PROTO_BASENAME.pb.c" > $TMPFILE
+mv -v $TMPFILE "$OUTPUT_DIR/$PROTO_BASENAME.pb.c"
+cat $COPYRIGHT_FILE "$OUTPUT_DIR/$PROTO_BASENAME.pb.h" > $TMPFILE
+mv -v $TMPFILE "$OUTPUT_DIR/$PROTO_BASENAME.pb.h"
+
+deactivate
+rm -rf $VENV_DIR
+
+popd > /dev/null
diff --git a/tools/distrib/check_nanopb_output.sh b/tools/distrib/check_nanopb_output.sh
new file mode 100755
index 0000000000..9cb7581d87
--- /dev/null
+++ b/tools/distrib/check_nanopb_output.sh
@@ -0,0 +1,69 @@
+#!/bin/bash
+# Copyright 2015-2016, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+set -ex
+
+apt-get install -y autoconf automake libtool curl python-virtualenv
+
+readonly NANOPB_TMP_OUTPUT="${LOCAL_GIT_ROOT}/gens/src/proto/grpc/lb/v0"
+
+# install protoc version 3
+pushd third_party/protobuf
+./autogen.sh
+./configure
+make
+make install
+ldconfig
+popd
+
+if [ ! -x "/usr/local/bin/protoc" ]; then
+  echo "Error: protoc not found in path"
+  exit 1
+fi
+readonly PROTOC_PATH='/usr/local/bin'
+# stack up and change to nanopb's proto generator directory
+pushd third_party/nanopb/generator/proto
+PATH="$PROTOC_PATH:$PATH" make
+
+# back to the root directory
+popd
+
+
+# nanopb-compile the proto to a temp location
+PATH="$PROTOC_PATH:$PATH" ./tools/codegen/core/gen_load_balancing_proto.sh \
+  src/proto/grpc/lb/v0/load_balancer.proto \
+  $NANOPB_TMP_OUTPUT
+
+# compare outputs to checked compiled code
+diff -rq $NANOPB_TMP_OUTPUT src/core/proto/grpc/lb/v0
+if [ $? != 0 ]; then
+  echo "Outputs differ: $NANOPB_TMP_OUTPUT vs src/core/proto/grpc/lb/v0"
+  exit 1
+fi
diff --git a/tools/dockerfile/grpc_clang_format/clang_format_all_the_things.sh b/tools/dockerfile/grpc_clang_format/clang_format_all_the_things.sh
index dd8ea1ac30..d56bc01831 100755
--- a/tools/dockerfile/grpc_clang_format/clang_format_all_the_things.sh
+++ b/tools/dockerfile/grpc_clang_format/clang_format_all_the_things.sh
@@ -44,7 +44,7 @@ for dir in $DIRS
 do
   for glob in $GLOB
   do
-    files="$files `find /local-code/$dir -name $glob -and -not -name *.generated.*`"
+    files="$files `find /local-code/$dir -name $glob -and -not -name *.generated.* -and -not -name *.pb.h -and -not -name *.pb.c`"
   done
 done
 
diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal
index 1f9b9ecc18..b0a1561df9 100644
--- a/tools/doxygen/Doxyfile.core.internal
+++ b/tools/doxygen/Doxyfile.core.internal
@@ -787,6 +787,7 @@ src/core/channel/subchannel_call_holder.h \
 src/core/client_config/client_config.h \
 src/core/client_config/connector.h \
 src/core/client_config/initial_connect_string.h \
+src/core/client_config/lb_policies/load_balancer_api.h \
 src/core/client_config/lb_policies/pick_first.h \
 src/core/client_config/lb_policies/round_robin.h \
 src/core/client_config/lb_policy.h \
@@ -847,6 +848,7 @@ src/core/json/json.h \
 src/core/json/json_common.h \
 src/core/json/json_reader.h \
 src/core/json/json_writer.h \
+src/core/proto/grpc/lb/v0/load_balancer.pb.h \
 src/core/statistics/census_interface.h \
 src/core/statistics/census_rpc_stats.h \
 src/core/surface/api_trace.h \
@@ -903,6 +905,10 @@ src/core/tsi/transport_security_interface.h \
 src/core/census/aggregation.h \
 src/core/census/mlog.h \
 src/core/census/rpc_metric_id.h \
+third_party/nanopb/pb.h \
+third_party/nanopb/pb_common.h \
+third_party/nanopb/pb_decode.h \
+third_party/nanopb/pb_encode.h \
 src/core/census/grpc_context.c \
 src/core/census/grpc_filter.c \
 src/core/channel/channel_args.c \
@@ -918,6 +924,7 @@ src/core/client_config/client_config.c \
 src/core/client_config/connector.c \
 src/core/client_config/default_initial_connect_string.c \
 src/core/client_config/initial_connect_string.c \
+src/core/client_config/lb_policies/load_balancer_api.c \
 src/core/client_config/lb_policies/pick_first.c \
 src/core/client_config/lb_policies/round_robin.c \
 src/core/client_config/lb_policy.c \
@@ -982,6 +989,7 @@ src/core/json/json.c \
 src/core/json/json_reader.c \
 src/core/json/json_string.c \
 src/core/json/json_writer.c \
+src/core/proto/grpc/lb/v0/load_balancer.pb.c \
 src/core/surface/alarm.c \
 src/core/surface/api_trace.c \
 src/core/surface/byte_buffer.c \
@@ -1058,6 +1066,9 @@ src/core/census/mlog.c \
 src/core/census/operation.c \
 src/core/census/placeholders.c \
 src/core/census/tracing.c \
+third_party/nanopb/pb_common.c \
+third_party/nanopb/pb_decode.c \
+third_party/nanopb/pb_encode.c \
 include/grpc/support/alloc.h \
 include/grpc/support/atm.h \
 include/grpc/support/atm_gcc_atomic.h \
diff --git a/tools/jenkins/build_docker_and_run_tests.sh b/tools/jenkins/build_docker_and_run_tests.sh
index e2ac7518f0..daa29b2f94 100755
--- a/tools/jenkins/build_docker_and_run_tests.sh
+++ b/tools/jenkins/build_docker_and_run_tests.sh
@@ -60,6 +60,9 @@ docker build -t $DOCKER_IMAGE_NAME $DOCKERFILE_DIR
 # Choose random name for docker container
 CONTAINER_NAME="run_tests_$(uuidgen)"
 
+# Git root as seen by the docker instance
+docker_instance_git_root=/var/local/jenkins/grpc
+
 # Run tests inside docker
 docker run \
   -e "RUN_TESTS_COMMAND=$RUN_TESTS_COMMAND" \
@@ -69,9 +72,10 @@ docker run \
   -e XDG_CACHE_HOME=/tmp/xdg-cache-home \
   -e THIS_IS_REALLY_NEEDED='see https://github.com/docker/docker/issues/14203 for why docker is awful' \
   -e HOST_GIT_ROOT=$git_root \
+  -e LOCAL_GIT_ROOT=$docker_instance_git_root \
   -e "BUILD_ID=$BUILD_ID" \
   -i $TTY_FLAG \
-  -v "$git_root:/var/local/jenkins/grpc" \
+  -v "$git_root:$docker_instance_git_root" \
   -v /tmp/ccache:/tmp/ccache \
   -v /tmp/npm-cache:/tmp/npm-cache \
   -v /tmp/xdg-cache-home:/tmp/xdg-cache-home \
diff --git a/tools/run_tests/sanity/check_submodules.sh b/tools/run_tests/sanity/check_submodules.sh
index f49230e49a..c08b382638 100755
--- a/tools/run_tests/sanity/check_submodules.sh
+++ b/tools/run_tests/sanity/check_submodules.sh
@@ -44,6 +44,7 @@ cat << EOF | awk '{ print $1 }' | sort > $want_submodules
  9f897b25800d2f54f5c442ef01a60721aeca6d87 third_party/boringssl (version_for_cocoapods_1.0-67-g9f897b2)
  05b155ff59114735ec8cd089f669c4c3d8f59029 third_party/gflags (v2.1.0-45-g05b155f)
  c99458533a9b4c743ed51537e25989ea55944908 third_party/googletest (release-1.7.0)
+ 5497a1dfc91a86965383cdd1652e348345400435 third_party/nanopb (nanopb-0.3.3-10-g5497a1d)
  d5fb408ddc281ffcadeb08699e65bb694656d0bd third_party/protobuf (v3.0.0-beta-2)
  50893291621658f355bc5b4d450a8d06a563053d third_party/zlib (v1.2.8)
 EOF
diff --git a/tools/run_tests/sanity/sanity_tests.yaml b/tools/run_tests/sanity/sanity_tests.yaml
index 809e6ce645..1fc0525e02 100644
--- a/tools/run_tests/sanity/sanity_tests.yaml
+++ b/tools/run_tests/sanity/sanity_tests.yaml
@@ -7,3 +7,4 @@
 - script: tools/distrib/check_copyright.py
 - script: tools/distrib/clang_format_code.sh
 - script: tools/distrib/check_trailing_newlines.sh
+- script: tools/distrib/check_nanopb_output.sh
diff --git a/tools/run_tests/sources_and_headers.json b/tools/run_tests/sources_and_headers.json
index e44a7bc79e..88e7343399 100644
--- a/tools/run_tests/sources_and_headers.json
+++ b/tools/run_tests/sources_and_headers.json
@@ -1660,6 +1660,23 @@
   }, 
   {
     "deps": [
+      "grpc", 
+      "grpc++", 
+      "grpc++_test_util", 
+      "grpc_test_util"
+    ], 
+    "headers": [
+      "src/proto/grpc/lb/v0/load_balancer.grpc.pb.h", 
+      "src/proto/grpc/lb/v0/load_balancer.pb.h"
+    ], 
+    "language": "c++", 
+    "name": "grpclb_api_test", 
+    "src": [
+      "test/cpp/grpclb/grpclb_api_test.cc"
+    ]
+  }, 
+  {
+    "deps": [
       "gpr", 
       "gpr_test_util", 
       "grpc", 
@@ -2998,6 +3015,7 @@
       "src/core/client_config/client_config.h", 
       "src/core/client_config/connector.h", 
       "src/core/client_config/initial_connect_string.h", 
+      "src/core/client_config/lb_policies/load_balancer_api.h", 
       "src/core/client_config/lb_policies/pick_first.h", 
       "src/core/client_config/lb_policies/round_robin.h", 
       "src/core/client_config/lb_policy.h", 
@@ -3058,6 +3076,7 @@
       "src/core/json/json_common.h", 
       "src/core/json/json_reader.h", 
       "src/core/json/json_writer.h", 
+      "src/core/proto/grpc/lb/v0/load_balancer.pb.h", 
       "src/core/security/auth_filters.h", 
       "src/core/security/base64.h", 
       "src/core/security/credentials.h", 
@@ -3110,7 +3129,11 @@
       "src/core/tsi/ssl_transport_security.h", 
       "src/core/tsi/ssl_types.h", 
       "src/core/tsi/transport_security.h", 
-      "src/core/tsi/transport_security_interface.h"
+      "src/core/tsi/transport_security_interface.h", 
+      "third_party/nanopb/pb.h", 
+      "third_party/nanopb/pb_common.h", 
+      "third_party/nanopb/pb_decode.h", 
+      "third_party/nanopb/pb_encode.h"
     ], 
     "language": "c", 
     "name": "grpc", 
@@ -3166,6 +3189,8 @@
       "src/core/client_config/default_initial_connect_string.c", 
       "src/core/client_config/initial_connect_string.c", 
       "src/core/client_config/initial_connect_string.h", 
+      "src/core/client_config/lb_policies/load_balancer_api.c", 
+      "src/core/client_config/lb_policies/load_balancer_api.h", 
       "src/core/client_config/lb_policies/pick_first.c", 
       "src/core/client_config/lb_policies/pick_first.h", 
       "src/core/client_config/lb_policies/round_robin.c", 
@@ -3291,6 +3316,8 @@
       "src/core/json/json_string.c", 
       "src/core/json/json_writer.c", 
       "src/core/json/json_writer.h", 
+      "src/core/proto/grpc/lb/v0/load_balancer.pb.c", 
+      "src/core/proto/grpc/lb/v0/load_balancer.pb.h", 
       "src/core/security/auth_filters.h", 
       "src/core/security/base64.c", 
       "src/core/security/base64.h", 
@@ -3537,6 +3564,7 @@
       "src/core/client_config/client_config.h", 
       "src/core/client_config/connector.h", 
       "src/core/client_config/initial_connect_string.h", 
+      "src/core/client_config/lb_policies/load_balancer_api.h", 
       "src/core/client_config/lb_policies/pick_first.h", 
       "src/core/client_config/lb_policies/round_robin.h", 
       "src/core/client_config/lb_policy.h", 
@@ -3597,6 +3625,7 @@
       "src/core/json/json_common.h", 
       "src/core/json/json_reader.h", 
       "src/core/json/json_writer.h", 
+      "src/core/proto/grpc/lb/v0/load_balancer.pb.h", 
       "src/core/statistics/census_interface.h", 
       "src/core/statistics/census_rpc_stats.h", 
       "src/core/surface/api_trace.h", 
@@ -3635,7 +3664,11 @@
       "src/core/transport/metadata_batch.h", 
       "src/core/transport/static_metadata.h", 
       "src/core/transport/transport.h", 
-      "src/core/transport/transport_impl.h"
+      "src/core/transport/transport_impl.h", 
+      "third_party/nanopb/pb.h", 
+      "third_party/nanopb/pb_common.h", 
+      "third_party/nanopb/pb_decode.h", 
+      "third_party/nanopb/pb_encode.h"
     ], 
     "language": "c", 
     "name": "grpc_unsecure", 
@@ -3690,6 +3723,8 @@
       "src/core/client_config/default_initial_connect_string.c", 
       "src/core/client_config/initial_connect_string.c", 
       "src/core/client_config/initial_connect_string.h", 
+      "src/core/client_config/lb_policies/load_balancer_api.c", 
+      "src/core/client_config/lb_policies/load_balancer_api.h", 
       "src/core/client_config/lb_policies/pick_first.c", 
       "src/core/client_config/lb_policies/pick_first.h", 
       "src/core/client_config/lb_policies/round_robin.c", 
@@ -3814,6 +3849,8 @@
       "src/core/json/json_string.c", 
       "src/core/json/json_writer.c", 
       "src/core/json/json_writer.h", 
+      "src/core/proto/grpc/lb/v0/load_balancer.pb.c", 
+      "src/core/proto/grpc/lb/v0/load_balancer.pb.h", 
       "src/core/statistics/census_interface.h", 
       "src/core/statistics/census_rpc_stats.h", 
       "src/core/surface/alarm.c", 
diff --git a/tools/run_tests/tests.json b/tools/run_tests/tests.json
index ea9c129101..480af97368 100644
--- a/tools/run_tests/tests.json
+++ b/tools/run_tests/tests.json
@@ -2051,6 +2051,26 @@
     "exclude_configs": [], 
     "flaky": false, 
     "language": "c++", 
+    "name": "grpclb_api_test", 
+    "platforms": [
+      "linux", 
+      "mac", 
+      "posix", 
+      "windows"
+    ]
+  }, 
+  {
+    "args": [], 
+    "ci_platforms": [
+      "linux", 
+      "mac", 
+      "posix", 
+      "windows"
+    ], 
+    "cpu_cost": 1.0, 
+    "exclude_configs": [], 
+    "flaky": false, 
+    "language": "c++", 
     "name": "hybrid_end2end_test", 
     "platforms": [
       "linux",