Merge branch 'master' of github.com:grpc/grpc into make-ruby-installable

11 files changed, 227 insertions, 35 deletions

diff --git a/include/grpc/grpc_security.h b/include/grpc/grpc_security.h
index fcd09db082..28881c3a11 100644
--- a/include/grpc/grpc_security.h
+++ b/include/grpc/grpc_security.h
@@ -147,6 +147,29 @@ GRPC_API grpc_channel_credentials *grpc_google_default_credentials_create(void);
 #define GRPC_DEFAULT_SSL_ROOTS_FILE_PATH_ENV_VAR \
   "GRPC_DEFAULT_SSL_ROOTS_FILE_PATH"
 
+/* Results for the SSL roots override callback. */
+typedef enum {
+  GRPC_SSL_ROOTS_OVERRIDE_OK,
+  GRPC_SSL_ROOTS_OVERRIDE_FAIL_PERMANENTLY, /* Do not try fallback options. */
+  GRPC_SSL_ROOTS_OVERRIDE_FAIL
+} grpc_ssl_roots_override_result;
+
+/* Callback for getting the SSL roots override from the application.
+   In case of success, *pem_roots_certs must be set to a NULL terminated string
+   containing the list of PEM encoded root certificates. The ownership is passed
+   to the core and freed (laster by the core) with gpr_free.
+   If this function fails and GRPC_DEFAULT_SSL_ROOTS_FILE_PATH environment is
+   set to a valid path, it will override the roots specified this func */
+typedef grpc_ssl_roots_override_result (*grpc_ssl_roots_override_callback)(
+    char **pem_root_certs);
+
+/* Setup a callback to override the default TLS/SSL roots.
+   This function is not thread-safe and must be called at initialization time
+   before any ssl credentials are created to have the desired side effect.
+   If GRPC_DEFAULT_SSL_ROOTS_FILE_PATH environment is set to a valid path, the
+   callback will not be called. */
+void grpc_set_ssl_roots_override_callback(grpc_ssl_roots_override_callback cb);
+
 /* Object that holds a private key / certificate chain pair in PEM format. */
 typedef struct {
   /* private_key is the NULL-terminated string containing the PEM encoding of
@@ -163,8 +186,9 @@ typedef struct {
      of the server root certificates. If this parameter is NULL, the
      implementation will first try to dereference the file pointed by the
      GRPC_DEFAULT_SSL_ROOTS_FILE_PATH environment variable, and if that fails,
-     get the roots from a well-known place on disk (in the grpc install
-     directory).
+     try to get the roots set by grpc_override_ssl_default_roots. Eventually,
+     if all these fail, it will try to get the roots from a well-known place on
+     disk (in the grpc install directory).
    - pem_key_cert_pair is a pointer on the object containing client's private
      key and certificate chain. This parameter can be NULL if the client does
      not have such a key/cert pair. */
diff --git a/src/core/security/security_connector.c b/src/core/security/security_connector.c
index 61336a1057..bdccbabfea 100644
--- a/src/core/security/security_connector.c
+++ b/src/core/security/security_connector.c
@@ -61,6 +61,14 @@ static const char *installed_roots_path =
     INSTALL_PREFIX "/share/grpc/roots.pem";
 #endif
 
+/* -- Overridden default roots. -- */
+
+static grpc_ssl_roots_override_callback ssl_roots_override_cb = NULL;
+
+void grpc_set_ssl_roots_override_callback(grpc_ssl_roots_override_callback cb) {
+  ssl_roots_override_cb = cb;
+}
+
 /* -- Cipher suites. -- */
 
 /* Defines the cipher suites that we accept by default. All these cipher suites
@@ -595,23 +603,44 @@ static grpc_security_connector_vtable ssl_channel_vtable = {
 static grpc_security_connector_vtable ssl_server_vtable = {
     ssl_server_destroy, ssl_server_do_handshake, ssl_server_check_peer};
 
-static gpr_slice default_pem_root_certs;
+static gpr_slice compute_default_pem_root_certs_once(void) {
+  gpr_slice result = gpr_empty_slice();
 
-static void init_default_pem_root_certs(void) {
   /* First try to load the roots from the environment. */
   char *default_root_certs_path =
       gpr_getenv(GRPC_DEFAULT_SSL_ROOTS_FILE_PATH_ENV_VAR);
-  if (default_root_certs_path == NULL) {
-    default_pem_root_certs = gpr_empty_slice();
-  } else {
-    default_pem_root_certs = gpr_load_file(default_root_certs_path, 0, NULL);
+  if (default_root_certs_path != NULL) {
+    result = gpr_load_file(default_root_certs_path, 0, NULL);
     gpr_free(default_root_certs_path);
   }
 
+  /* Try overridden roots if needed. */
+  grpc_ssl_roots_override_result ovrd_res = GRPC_SSL_ROOTS_OVERRIDE_FAIL;
+  if (GPR_SLICE_IS_EMPTY(result) && ssl_roots_override_cb != NULL) {
+    char *pem_root_certs = NULL;
+    ovrd_res = ssl_roots_override_cb(&pem_root_certs);
+    if (ovrd_res == GRPC_SSL_ROOTS_OVERRIDE_OK) {
+      GPR_ASSERT(pem_root_certs != NULL);
+      result = gpr_slice_new(pem_root_certs, strlen(pem_root_certs), gpr_free);
+    }
+  }
+
   /* Fall back to installed certs if needed. */
-  if (GPR_SLICE_IS_EMPTY(default_pem_root_certs)) {
-    default_pem_root_certs = gpr_load_file(installed_roots_path, 0, NULL);
+  if (GPR_SLICE_IS_EMPTY(result) &&
+      ovrd_res != GRPC_SSL_ROOTS_OVERRIDE_FAIL_PERMANENTLY) {
+    result = gpr_load_file(installed_roots_path, 0, NULL);
   }
+  return result;
+}
+
+static gpr_slice default_pem_root_certs;
+
+static void init_default_pem_root_certs(void) {
+  default_pem_root_certs = compute_default_pem_root_certs_once();
+}
+
+gpr_slice grpc_get_default_ssl_roots_for_testing(void) {
+  return compute_default_pem_root_certs_once();
 }
 
 size_t grpc_get_default_ssl_roots(const unsigned char **pem_root_certs) {
diff --git a/src/core/security/security_connector.h b/src/core/security/security_connector.h
index 2b734109b3..39df7821f0 100644
--- a/src/core/security/security_connector.h
+++ b/src/core/security/security_connector.h
@@ -209,6 +209,9 @@ grpc_security_status grpc_ssl_channel_security_connector_create(
 /* Gets the default ssl roots. */
 size_t grpc_get_default_ssl_roots(const unsigned char **pem_root_certs);
 
+/* Exposed for TESTING ONLY!. */
+gpr_slice grpc_get_default_ssl_roots_for_testing(void);
+
 /* Config for ssl servers. */
 typedef struct {
   unsigned char **pem_private_keys;
diff --git a/src/core/transport/chttp2/hpack_encoder.c b/src/core/transport/chttp2/hpack_encoder.c
index 89a80d896c..f30f574d06 100644
--- a/src/core/transport/chttp2/hpack_encoder.c
+++ b/src/core/transport/chttp2/hpack_encoder.c
@@ -283,7 +283,7 @@ static void emit_lithdr_incidx(grpc_chttp2_hpack_compressor *c,
   len_val_len = GRPC_CHTTP2_VARINT_LENGTH((uint32_t)len_val, 1);
   GRPC_CHTTP2_WRITE_VARINT(key_index, 2, 0x40,
                            add_tiny_header_data(st, len_pfx), len_pfx);
-  GRPC_CHTTP2_WRITE_VARINT((uint32_t)len_val, 1, 0x00,
+  GRPC_CHTTP2_WRITE_VARINT((uint32_t)len_val, 1, huffman_prefix,
                            add_tiny_header_data(st, len_val_len), len_val_len);
   add_header_data(st, gpr_slice_ref(value_slice));
 }
@@ -300,7 +300,7 @@ static void emit_lithdr_noidx(grpc_chttp2_hpack_compressor *c,
   len_val_len = GRPC_CHTTP2_VARINT_LENGTH((uint32_t)len_val, 1);
   GRPC_CHTTP2_WRITE_VARINT(key_index, 4, 0x00,
                            add_tiny_header_data(st, len_pfx), len_pfx);
-  GRPC_CHTTP2_WRITE_VARINT((uint32_t)len_val, 1, 0x00,
+  GRPC_CHTTP2_WRITE_VARINT((uint32_t)len_val, 1, huffman_prefix,
                            add_tiny_header_data(st, len_val_len), len_val_len);
   add_header_data(st, gpr_slice_ref(value_slice));
 }
diff --git a/src/core/transport/chttp2/internal.h b/src/core/transport/chttp2/internal.h
index 6823f8f551..c611496e7e 100644
--- a/src/core/transport/chttp2/internal.h
+++ b/src/core/transport/chttp2/internal.h
@@ -35,6 +35,7 @@
 #define GRPC_INTERNAL_CORE_CHTTP2_INTERNAL_H
 
 #include <assert.h>
+#include <stdbool.h>
 
 #include "src/core/iomgr/endpoint.h"
 #include "src/core/transport/chttp2/frame.h"
@@ -67,6 +68,9 @@ typedef enum {
   GRPC_CHTTP2_LIST_CLOSED_WAITING_FOR_PARSING,
   GRPC_CHTTP2_LIST_CLOSED_WAITING_FOR_WRITING,
   GRPC_CHTTP2_LIST_STALLED_BY_TRANSPORT,
+  /* streams waiting for the outgoing window in the writing path, they will be
+   * merged to the stalled list or writable list under transport lock. */
+  GRPC_CHTTP2_LIST_WRITING_STALLED_BY_TRANSPORT,
   /** streams that are waiting to start because there are too many concurrent
       streams on the connection */
   GRPC_CHTTP2_LIST_WAITING_FOR_CONCURRENCY,
@@ -504,11 +508,11 @@ void grpc_chttp2_publish_reads(grpc_exec_ctx *exec_ctx,
                                grpc_chttp2_transport_global *global,
                                grpc_chttp2_transport_parsing *parsing);
 
-/** Get a writable stream
-    returns non-zero if there was a stream available */
 void grpc_chttp2_list_add_writable_stream(
     grpc_chttp2_transport_global *transport_global,
     grpc_chttp2_stream_global *stream_global);
+/** Get a writable stream
+    returns non-zero if there was a stream available */
 int grpc_chttp2_list_pop_writable_stream(
     grpc_chttp2_transport_global *transport_global,
     grpc_chttp2_transport_writing *transport_writing,
@@ -560,9 +564,12 @@ int grpc_chttp2_list_pop_check_read_ops(
     grpc_chttp2_transport_global *transport_global,
     grpc_chttp2_stream_global **stream_global);
 
-void grpc_chttp2_list_add_stalled_by_transport(
+void grpc_chttp2_list_add_writing_stalled_by_transport(
     grpc_chttp2_transport_writing *transport_writing,
     grpc_chttp2_stream_writing *stream_writing);
+void grpc_chttp2_list_flush_writing_stalled_by_transport(
+    grpc_chttp2_transport_writing *transport_writing, bool is_window_available);
+
 int grpc_chttp2_list_pop_stalled_by_transport(
     grpc_chttp2_transport_global *transport_global,
     grpc_chttp2_stream_global **stream_global);
diff --git a/src/core/transport/chttp2/stream_lists.c b/src/core/transport/chttp2/stream_lists.c
index 273a513e2f..2f31a47cb3 100644
--- a/src/core/transport/chttp2/stream_lists.c
+++ b/src/core/transport/chttp2/stream_lists.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -313,12 +313,27 @@ int grpc_chttp2_list_pop_check_read_ops(
   return r;
 }
 
-void grpc_chttp2_list_add_stalled_by_transport(
+void grpc_chttp2_list_add_writing_stalled_by_transport(
     grpc_chttp2_transport_writing *transport_writing,
     grpc_chttp2_stream_writing *stream_writing) {
   stream_list_add(TRANSPORT_FROM_WRITING(transport_writing),
                   STREAM_FROM_WRITING(stream_writing),
-                  GRPC_CHTTP2_LIST_STALLED_BY_TRANSPORT);
+                  GRPC_CHTTP2_LIST_WRITING_STALLED_BY_TRANSPORT);
+}
+
+void grpc_chttp2_list_flush_writing_stalled_by_transport(
+    grpc_chttp2_transport_writing *transport_writing,
+    bool is_window_available) {
+  grpc_chttp2_stream *stream;
+  grpc_chttp2_transport *transport = TRANSPORT_FROM_WRITING(transport_writing);
+  while (stream_list_pop(transport, &stream,
+                         GRPC_CHTTP2_LIST_WRITING_STALLED_BY_TRANSPORT)) {
+    if (is_window_available) {
+      grpc_chttp2_list_add_writable_stream(&transport->global, &stream->global);
+    } else {
+      stream_list_add(transport, stream, GRPC_CHTTP2_LIST_STALLED_BY_TRANSPORT);
+    }
+  }
 }
 
 int grpc_chttp2_list_pop_stalled_by_transport(
diff --git a/src/core/transport/chttp2/writing.c b/src/core/transport/chttp2/writing.c
index f78adb9e26..095883c66d 100644
--- a/src/core/transport/chttp2/writing.c
+++ b/src/core/transport/chttp2/writing.c
@@ -130,8 +130,8 @@ int grpc_chttp2_unlocking_check_writes(
             GRPC_CHTTP2_STREAM_REF(stream_global, "chttp2_writing");
           }
         } else {
-          grpc_chttp2_list_add_stalled_by_transport(transport_writing,
-                                                    stream_writing);
+          grpc_chttp2_list_add_writing_stalled_by_transport(transport_writing,
+                                                            stream_writing);
         }
       }
       if (stream_global->send_trailing_metadata) {
@@ -273,8 +273,8 @@ static void finalize_outbuf(grpc_exec_ctx *exec_ctx,
           stream_writing->sent_message = 1;
         }
       } else if (transport_writing->outgoing_window == 0) {
-        grpc_chttp2_list_add_stalled_by_transport(transport_writing,
-                                                  stream_writing);
+        grpc_chttp2_list_add_writing_stalled_by_transport(transport_writing,
+                                                          stream_writing);
         grpc_chttp2_list_add_written_stream(transport_writing, stream_writing);
       }
     }
@@ -312,8 +312,8 @@ static void finalize_outbuf(grpc_exec_ctx *exec_ctx,
           /* do nothing - already reffed */
         }
       } else {
-        grpc_chttp2_list_add_stalled_by_transport(transport_writing,
-                                                  stream_writing);
+        grpc_chttp2_list_add_writing_stalled_by_transport(transport_writing,
+                                                          stream_writing);
         grpc_chttp2_list_add_written_stream(transport_writing, stream_writing);
       }
     } else {
@@ -329,6 +329,10 @@ void grpc_chttp2_cleanup_writing(
     grpc_chttp2_transport_writing *transport_writing) {
   grpc_chttp2_stream_writing *stream_writing;
   grpc_chttp2_stream_global *stream_global;
+  bool is_window_available = transport_writing->outgoing_window > 0;
+
+  grpc_chttp2_list_flush_writing_stalled_by_transport(transport_writing,
+                                                      is_window_available);
 
   while (grpc_chttp2_list_pop_written_stream(
       transport_global, transport_writing, &stream_global, &stream_writing)) {
diff --git a/src/python/grpcio/commands.py b/src/python/grpcio/commands.py
index 3e3c72ff6e..98ad2e571d 100644
--- a/src/python/grpcio/commands.py
+++ b/src/python/grpcio/commands.py
@@ -320,11 +320,11 @@ class BuildExt(build_ext.build_ext):
         extension.extra_link_args += list(BuildExt.LINK_OPTIONS[compiler])
     try:
       build_ext.build_ext.build_extensions(self)
-    except KeyboardInterrupt:
-      raise
     except Exception as error:
-      support.diagnose_build_ext_error(self, error, traceback.format_exc())
-      raise CommandError("Failed `build_ext` step.")
+      formatted_exception = traceback.format_exc()
+      support.diagnose_build_ext_error(self, error, formatted_exception)
+      raise CommandError(
+          "Failed `build_ext` step:\n{}".format(formatted_exception))
 
 
 class Gather(setuptools.Command):
diff --git a/src/python/grpcio/support.py b/src/python/grpcio/support.py
index 96d9cbf4f3..33244eb388 100644
--- a/src/python/grpcio/support.py
+++ b/src/python/grpcio/support.py
@@ -77,10 +77,27 @@ def _expect_compile(compiler, source_string, error_message):
             .format(error_message))
 
 def diagnose_compile_error(build_ext, error):
-  """Attempt to run a few test files through the compiler to see if we can
-     diagnose the reason for the compile failure."""
+  """Attempt to diagnose an error during compilation."""
   for c_check, message in C_CHECKS.items():
     _expect_compile(build_ext.compiler, c_check, message)
+  python_sources = [
+      source for source in build_ext.get_source_files()
+      if source.startswith('./src/python') and source.endswith('c')
+  ]
+  for source in python_sources:
+    if not os.path.isfile(source):
+      raise commands.CommandError(
+          ("Diagnostics found a missing Python extension source file:\n{}\n\n"
+           "This is usually because the Cython sources haven't been transpiled "
+           "into C yet and you're building from source.\n"
+           "Try setting the environment variable "
+           "`GRPC_PYTHON_BUILD_WITH_CYTHON=1` when invoking `setup.py` or "
+           "when using `pip`, e.g.:\n\n"
+           "pip install -rrequirements.txt\n"
+           "GRPC_PYTHON_BUILD_WITH_CYTHON=1 pip install .")
+            .format(source)
+          )
+
 
 _ERROR_DIAGNOSES = {
     errors.CompileError: diagnose_compile_error
diff --git a/test/core/security/security_connector_test.c b/test/core/security/security_connector_test.c
index 0dcffa40ce..ee5435f01d 100644
--- a/test/core/security/security_connector_test.c
+++ b/test/core/security/security_connector_test.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -36,6 +36,9 @@
 
 #include "src/core/security/security_connector.h"
 #include "src/core/security/security_context.h"
+#include "src/core/support/env.h"
+#include "src/core/support/file.h"
+#include "src/core/support/string.h"
 #include "src/core/tsi/ssl_transport_security.h"
 #include "src/core/tsi/transport_security.h"
 #include "test/core/util/test_config.h"
@@ -44,6 +47,7 @@
 
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
+#include <grpc/support/string_util.h>
 #include <grpc/support/useful.h>
 
 static int check_transport_security_type(const grpc_auth_context *ctx) {
@@ -297,7 +301,66 @@ static void test_cn_and_multiple_sans_and_others_ssl_peer_to_auth_context(
   GRPC_AUTH_CONTEXT_UNREF(ctx, "test");
 }
 
-/* TODO(jboeuf): Unit-test tsi_shallow_peer_from_auth_context. */
+static const char *roots_for_override_api = "roots for override api";
+
+static grpc_ssl_roots_override_result override_roots_success(
+    char **pem_root_certs) {
+  *pem_root_certs = gpr_strdup(roots_for_override_api);
+  return GRPC_SSL_ROOTS_OVERRIDE_OK;
+}
+
+static grpc_ssl_roots_override_result override_roots_permanent_failure(
+    char **pem_root_certs) {
+  return GRPC_SSL_ROOTS_OVERRIDE_FAIL_PERMANENTLY;
+}
+
+static void test_default_ssl_roots(void) {
+  const char *roots_for_env_var = "roots for env var";
+
+  char *roots_env_var_file_path;
+  FILE *roots_env_var_file =
+      gpr_tmpfile("test_roots_for_env_var", &roots_env_var_file_path);
+  fwrite(roots_for_env_var, 1, strlen(roots_for_env_var), roots_env_var_file);
+  fclose(roots_env_var_file);
+
+  /* First let's get the root through the override: set the env to an invalid
+     value. */
+  gpr_setenv(GRPC_DEFAULT_SSL_ROOTS_FILE_PATH_ENV_VAR, "");
+  grpc_set_ssl_roots_override_callback(override_roots_success);
+  gpr_slice roots = grpc_get_default_ssl_roots_for_testing();
+  char *roots_contents = gpr_dump_slice(roots, GPR_DUMP_ASCII);
+  gpr_slice_unref(roots);
+  GPR_ASSERT(strcmp(roots_contents, roots_for_override_api) == 0);
+  gpr_free(roots_contents);
+
+  /* Now let's set the env var: We should get the contents pointed value
+     instead. */
+  gpr_setenv(GRPC_DEFAULT_SSL_ROOTS_FILE_PATH_ENV_VAR, roots_env_var_file_path);
+  roots = grpc_get_default_ssl_roots_for_testing();
+  roots_contents = gpr_dump_slice(roots, GPR_DUMP_ASCII);
+  gpr_slice_unref(roots);
+  GPR_ASSERT(strcmp(roots_contents, roots_for_env_var) == 0);
+  gpr_free(roots_contents);
+
+  /* Now reset the env var. We should fall back to the value overridden using
+     the api. */
+  gpr_setenv(GRPC_DEFAULT_SSL_ROOTS_FILE_PATH_ENV_VAR, "");
+  roots = grpc_get_default_ssl_roots_for_testing();
+  roots_contents = gpr_dump_slice(roots, GPR_DUMP_ASCII);
+  gpr_slice_unref(roots);
+  GPR_ASSERT(strcmp(roots_contents, roots_for_override_api) == 0);
+  gpr_free(roots_contents);
+
+  /* Now setup a permanent failure for the overridden roots and we should get
+     an empty slice. */
+  grpc_set_ssl_roots_override_callback(override_roots_permanent_failure);
+  roots = grpc_get_default_ssl_roots_for_testing();
+  GPR_ASSERT(GPR_SLICE_IS_EMPTY(roots));
+
+  /* Cleanup. */
+  remove(roots_env_var_file_path);
+  gpr_free(roots_env_var_file_path);
+}
 
 int main(int argc, char **argv) {
   grpc_test_init(argc, argv);
@@ -308,6 +371,7 @@ int main(int argc, char **argv) {
   test_cn_and_one_san_ssl_peer_to_auth_context();
   test_cn_and_multiple_sans_ssl_peer_to_auth_context();
   test_cn_and_multiple_sans_and_others_ssl_peer_to_auth_context();
+  test_default_ssl_roots();
 
   grpc_shutdown();
   return 0;
diff --git a/test/cpp/end2end/end2end_test.cc b/test/cpp/end2end/end2end_test.cc
index 5a414ebc86..3ad09aca4c 100644
--- a/test/cpp/end2end/end2end_test.cc
+++ b/test/cpp/end2end/end2end_test.cc
@@ -452,13 +452,18 @@ class End2endTest : public ::testing::TestWithParam<TestScenario> {
   TestServiceImplDupPkg dup_pkg_service_;
 };
 
-static void SendRpc(grpc::testing::EchoTestService::Stub* stub, int num_rpcs) {
+static void SendRpc(grpc::testing::EchoTestService::Stub* stub, int num_rpcs,
+                    bool with_binary_metadata) {
   EchoRequest request;
   EchoResponse response;
   request.set_message("Hello hello hello hello");
 
   for (int i = 0; i < num_rpcs; ++i) {
     ClientContext context;
+    if (with_binary_metadata) {
+      char bytes[8] = {'\0', '\1', '\2', '\3', '\4', '\5', '\6', (char)i};
+      context.AddMetadata("custom-bin", grpc::string(bytes, 8));
+    }
     context.set_compression_algorithm(GRPC_COMPRESS_GZIP);
     Status s = stub->Echo(&context, request, &response);
     EXPECT_EQ(response.message(), request.message());
@@ -466,6 +471,30 @@ static void SendRpc(grpc::testing::EchoTestService::Stub* stub, int num_rpcs) {
   }
 }
 
+TEST_P(End2endTest, MultipleRpcsWithVariedBinaryMetadataValue) {
+  ResetStub();
+  std::vector<std::thread*> threads;
+  for (int i = 0; i < 10; ++i) {
+    threads.push_back(new std::thread(SendRpc, stub_.get(), 10, true));
+  }
+  for (int i = 0; i < 10; ++i) {
+    threads[i]->join();
+    delete threads[i];
+  }
+}
+
+TEST_P(End2endTest, MultipleRpcs) {
+  ResetStub();
+  std::vector<std::thread*> threads;
+  for (int i = 0; i < 10; ++i) {
+    threads.push_back(new std::thread(SendRpc, stub_.get(), 10, false));
+  }
+  for (int i = 0; i < 10; ++i) {
+    threads[i]->join();
+    delete threads[i];
+  }
+}
+
 TEST_P(End2endTest, RequestStreamOneRequest) {
   ResetStub();
   EchoRequest request;
@@ -803,14 +832,14 @@ class ProxyEnd2endTest : public End2endTest {
 
 TEST_P(ProxyEnd2endTest, SimpleRpc) {
   ResetStub();
-  SendRpc(stub_.get(), 1);
+  SendRpc(stub_.get(), 1, false);
 }
 
 TEST_P(ProxyEnd2endTest, MultipleRpcs) {
   ResetStub();
   std::vector<std::thread*> threads;
   for (int i = 0; i < 10; ++i) {
-    threads.push_back(new std::thread(SendRpc, stub_.get(), 10));
+    threads.push_back(new std::thread(SendRpc, stub_.get(), 10, false));
   }
   for (int i = 0; i < 10; ++i) {
     threads[i]->join();