Add a test that measures flow control stalls in a deterministic way

3 files changed, 393 insertions, 10 deletions

diff --git a/test/core/util/trickle_endpoint.c b/test/core/util/trickle_endpoint.c
new file mode 100644
index 0000000000..8d661e04c6
--- /dev/null
+++ b/test/core/util/trickle_endpoint.c
@@ -0,0 +1,178 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include "test/core/util/passthru_endpoint.h"
+
+#include <inttypes.h>
+#include <string.h>
+
+#include <grpc/support/alloc.h>
+#include <grpc/support/log.h>
+#include <grpc/support/string_util.h>
+#include <grpc/support/useful.h>
+
+#include "src/core/lib/iomgr/sockaddr.h"
+
+#include "src/core/lib/slice/slice_internal.h"
+
+typedef struct {
+  grpc_endpoint base;
+  grpc_endpoint *wrapped;
+
+  gpr_mu mu;
+  grpc_slice_buffer write_buffer;
+  grpc_slice_buffer writing_buffer;
+  grpc_error *error;
+  bool writing;
+} trickle_endpoint;
+
+static void te_read(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep,
+                    grpc_slice_buffer *slices, grpc_closure *cb) {
+  trickle_endpoint *te = (trickle_endpoint *)ep;
+  grpc_endpoint_read(exec_ctx, te->wrapped, slices, cb);
+}
+
+static void te_write(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep,
+                     grpc_slice_buffer *slices, grpc_closure *cb) {
+  trickle_endpoint *te = (trickle_endpoint *)ep;
+  for (size_t i = 0; i < slices->count; i++) {
+    grpc_slice_ref_internal(slices->slices[i]);
+  }
+  grpc_slice_buffer_addn(&te->write_buffer, slices->slices, slices->count);
+  gpr_mu_lock(&te->mu);
+  grpc_closure_sched(exec_ctx, cb, GRPC_ERROR_REF(te->error));
+  gpr_mu_unlock(&te->mu);
+}
+
+static grpc_workqueue *te_get_workqueue(grpc_endpoint *ep) {
+  trickle_endpoint *te = (trickle_endpoint *)ep;
+  return grpc_endpoint_get_workqueue(te->wrapped);
+}
+
+static void te_add_to_pollset(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep,
+                              grpc_pollset *pollset) {
+  trickle_endpoint *te = (trickle_endpoint *)ep;
+  grpc_endpoint_add_to_pollset(exec_ctx, te->wrapped, pollset);
+}
+
+static void te_add_to_pollset_set(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep,
+                                  grpc_pollset_set *pollset_set) {
+  trickle_endpoint *te = (trickle_endpoint *)ep;
+  grpc_endpoint_add_to_pollset_set(exec_ctx, te->wrapped, pollset_set);
+}
+
+static void te_shutdown(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep,
+                        grpc_error *why) {
+  trickle_endpoint *te = (trickle_endpoint *)ep;
+  gpr_mu_lock(&te->mu);
+  if (te->error == GRPC_ERROR_NONE) {
+    te->error = GRPC_ERROR_REF(why);
+  }
+  gpr_mu_unlock(&te->mu);
+  grpc_endpoint_shutdown(exec_ctx, te->wrapped, why);
+}
+
+static void te_destroy(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep) {
+  trickle_endpoint *te = (trickle_endpoint *)ep;
+  grpc_endpoint_destroy(exec_ctx, te->wrapped);
+  gpr_mu_destroy(&te->mu);
+  grpc_slice_buffer_destroy_internal(exec_ctx, &te->write_buffer);
+  grpc_slice_buffer_destroy_internal(exec_ctx, &te->writing_buffer);
+  GRPC_ERROR_UNREF(te->error);
+  gpr_free(te);
+}
+
+static grpc_resource_user *te_get_resource_user(grpc_endpoint *ep) {
+  trickle_endpoint *te = (trickle_endpoint *)ep;
+  return grpc_endpoint_get_resource_user(te->wrapped);
+}
+
+static char *te_get_peer(grpc_endpoint *ep) {
+  trickle_endpoint *te = (trickle_endpoint *)ep;
+  return grpc_endpoint_get_peer(te->wrapped);
+}
+
+static int te_get_fd(grpc_endpoint *ep) {
+  trickle_endpoint *te = (trickle_endpoint *)ep;
+  return grpc_endpoint_get_fd(te->wrapped);
+}
+
+static void te_finish_write(grpc_exec_ctx *exec_ctx, void *arg,
+                            grpc_error *error) {
+  trickle_endpoint *te = arg;
+  gpr_mu_lock(&te->mu);
+  te->writing = false;
+  grpc_slice_buffer_reset_and_unref(&te->writing_buffer);
+  gpr_mu_unlock(&te->mu);
+}
+
+static const grpc_endpoint_vtable vtable = {te_read,
+                                            te_write,
+                                            te_get_workqueue,
+                                            te_add_to_pollset,
+                                            te_add_to_pollset_set,
+                                            te_shutdown,
+                                            te_destroy,
+                                            te_get_resource_user,
+                                            te_get_peer,
+                                            te_get_fd};
+
+grpc_endpoint *grpc_trickle_endpoint_create(grpc_endpoint *wrap) {
+  trickle_endpoint *te = gpr_malloc(sizeof(*te));
+  te->base.vtable = &vtable;
+  te->wrapped = wrap;
+  gpr_mu_init(&te->mu);
+  grpc_slice_buffer_init(&te->write_buffer);
+  grpc_slice_buffer_init(&te->writing_buffer);
+  te->error = GRPC_ERROR_NONE;
+  te->writing = false;
+  return &te->base;
+}
+
+size_t grpc_trickle_endpoint_trickle(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep,
+                                     size_t bytes) {
+  trickle_endpoint *te = (trickle_endpoint *)ep;
+  gpr_mu_lock(&te->mu);
+  if (bytes > 0 && !te->writing) {
+    grpc_slice_buffer_move_first(&te->write_buffer,
+                                 GPR_MIN(bytes, te->write_buffer.length),
+                                 &te->writing_buffer);
+    te->writing = true;
+    grpc_endpoint_write(
+        exec_ctx, te->wrapped, &te->writing_buffer,
+        grpc_closure_create(te_finish_write, te, grpc_schedule_on_exec_ctx));
+  }
+  size_t backlog = te->write_buffer.length;
+  gpr_mu_unlock(&te->mu);
+  return backlog;
+}
diff --git a/test/core/util/trickle_endpoint.h b/test/core/util/trickle_endpoint.h
new file mode 100644
index 0000000000..5f16818ebb
--- /dev/null
+++ b/test/core/util/trickle_endpoint.h
@@ -0,0 +1,45 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef TRICKLE_ENDPOINT_H
+#define TRICKLE_ENDPOINT_H
+
+#include "src/core/lib/iomgr/endpoint.h"
+
+grpc_endpoint *grpc_trickle_endpoint_create(grpc_endpoint *wrap);
+
+/* Allow up to \a bytes through the endpoint. Returns the new backlog. */
+size_t grpc_trickle_endpoint_trickle(grpc_exec_ctx *exec_ctx,
+                                     grpc_endpoint *endpoint, size_t bytes);
+
+#endif
diff --git a/test/cpp/microbenchmarks/bm_fullstack.cc b/test/cpp/microbenchmarks/bm_fullstack.cc
index 9d883e68d7..f83c359961 100644
--- a/test/cpp/microbenchmarks/bm_fullstack.cc
+++ b/test/cpp/microbenchmarks/bm_fullstack.cc
@@ -46,6 +46,7 @@
 
 extern "C" {
 #include "src/core/ext/transport/chttp2/transport/chttp2_transport.h"
+#include "src/core/ext/transport/chttp2/transport/internal.h"
 #include "src/core/lib/channel/channel_args.h"
 #include "src/core/lib/iomgr/endpoint.h"
 #include "src/core/lib/iomgr/endpoint_pair.h"
@@ -57,6 +58,7 @@ extern "C" {
 #include "test/core/util/memory_counters.h"
 #include "test/core/util/passthru_endpoint.h"
 #include "test/core/util/port.h"
+#include "test/core/util/trickle_endpoint.h"
 }
 #include "src/core/lib/profiling/timers.h"
 #include "src/cpp/client/create_channel_internal.h"
@@ -197,7 +199,8 @@ class UDS : public FullstackFixture {
 
 class EndpointPairFixture : public BaseFixture {
  public:
-  EndpointPairFixture(Service* service, grpc_endpoint_pair endpoints) {
+  EndpointPairFixture(Service* service, grpc_endpoint_pair endpoints)
+      : endpoint_pair_(endpoints) {
     ServerBuilder b;
     cq_ = b.AddCompletionQueue(true);
     b.RegisterService(service);
@@ -210,7 +213,7 @@ class EndpointPairFixture : public BaseFixture {
     {
       const grpc_channel_args* server_args =
           grpc_server_get_channel_args(server_->c_server());
-      grpc_transport* transport = grpc_create_chttp2_transport(
+      server_transport_ = grpc_create_chttp2_transport(
           &exec_ctx, server_args, endpoints.server, 0 /* is_client */);
 
       grpc_pollset** pollsets;
@@ -221,9 +224,9 @@ class EndpointPairFixture : public BaseFixture {
         grpc_endpoint_add_to_pollset(&exec_ctx, endpoints.server, pollsets[i]);
       }
 
-      grpc_server_setup_transport(&exec_ctx, server_->c_server(), transport,
-                                  NULL, server_args);
-      grpc_chttp2_transport_start_reading(&exec_ctx, transport, NULL);
+      grpc_server_setup_transport(&exec_ctx, server_->c_server(),
+                                  server_transport_, NULL, server_args);
+      grpc_chttp2_transport_start_reading(&exec_ctx, server_transport_, NULL);
     }
 
     /* create channel */
@@ -233,12 +236,13 @@ class EndpointPairFixture : public BaseFixture {
       ApplyCommonChannelArguments(&args);
 
       grpc_channel_args c_args = args.c_channel_args();
-      grpc_transport* transport =
+      client_transport_ =
           grpc_create_chttp2_transport(&exec_ctx, &c_args, endpoints.client, 1);
-      GPR_ASSERT(transport);
-      grpc_channel* channel = grpc_channel_create(
-          &exec_ctx, "target", &c_args, GRPC_CLIENT_DIRECT_CHANNEL, transport);
-      grpc_chttp2_transport_start_reading(&exec_ctx, transport, NULL);
+      GPR_ASSERT(client_transport_);
+      grpc_channel* channel =
+          grpc_channel_create(&exec_ctx, "target", &c_args,
+                              GRPC_CLIENT_DIRECT_CHANNEL, client_transport_);
+      grpc_chttp2_transport_start_reading(&exec_ctx, client_transport_, NULL);
 
       channel_ = CreateChannelInternal("", channel);
     }
@@ -258,6 +262,11 @@ class EndpointPairFixture : public BaseFixture {
   ServerCompletionQueue* cq() { return cq_.get(); }
   std::shared_ptr<Channel> channel() { return channel_; }
 
+ protected:
+  grpc_endpoint_pair endpoint_pair_;
+  grpc_transport* client_transport_;
+  grpc_transport* server_transport_;
+
  private:
   std::unique_ptr<Server> server_;
   std::unique_ptr<ServerCompletionQueue> cq_;
@@ -295,6 +304,74 @@ class InProcessCHTTP2 : public EndpointPairFixture {
   }
 };
 
+class TrickledCHTTP2 : public EndpointPairFixture {
+ public:
+  TrickledCHTTP2(Service* service)
+      : EndpointPairFixture(service, MakeEndpoints()) {}
+
+  void AddToLabel(std::ostream& out, benchmark::State& state) {
+    out << " writes/iter:"
+        << ((double)stats_.num_writes / (double)state.iterations())
+        << " cli-transport-stalls/iter:"
+        << ((double)
+                client_stats_.streams_stalled_due_to_transport_flow_control /
+            (double)state.iterations())
+        << " cli-stream-stalls/iter:"
+        << ((double)client_stats_.streams_stalled_due_to_stream_flow_control /
+            (double)state.iterations())
+        << " svr-transport-stalls/iter:"
+        << ((double)
+                server_stats_.streams_stalled_due_to_transport_flow_control /
+            (double)state.iterations())
+        << " svr-stream-stalls/iter:"
+        << ((double)server_stats_.streams_stalled_due_to_stream_flow_control /
+            (double)state.iterations());
+  }
+
+  void Step(size_t write_size) {
+    grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
+    size_t client_backlog = grpc_trickle_endpoint_trickle(
+        &exec_ctx, endpoint_pair_.client, write_size);
+    size_t server_backlog = grpc_trickle_endpoint_trickle(
+        &exec_ctx, endpoint_pair_.server, write_size);
+    grpc_exec_ctx_finish(&exec_ctx);
+
+    UpdateStats((grpc_chttp2_transport*)client_transport_, &client_stats_,
+                client_backlog);
+    UpdateStats((grpc_chttp2_transport*)server_transport_, &server_stats_,
+                server_backlog);
+  }
+
+ private:
+  grpc_passthru_endpoint_stats stats_;
+  struct Stats {
+    int streams_stalled_due_to_stream_flow_control = 0;
+    int streams_stalled_due_to_transport_flow_control = 0;
+  };
+  Stats client_stats_;
+  Stats server_stats_;
+
+  grpc_endpoint_pair MakeEndpoints() {
+    grpc_endpoint_pair p;
+    grpc_passthru_endpoint_create(&p.client, &p.server, initialize_stuff.rq(),
+                                  &stats_);
+    p.client = grpc_trickle_endpoint_create(p.client);
+    p.server = grpc_trickle_endpoint_create(p.server);
+    return p;
+  }
+
+  void UpdateStats(grpc_chttp2_transport* t, Stats* s, size_t backlog) {
+    if (backlog == 0) {
+      if (t->lists[GRPC_CHTTP2_LIST_STALLED_BY_STREAM].head != NULL) {
+        s->streams_stalled_due_to_stream_flow_control++;
+      }
+      if (t->lists[GRPC_CHTTP2_LIST_STALLED_BY_TRANSPORT].head != NULL) {
+        s->streams_stalled_due_to_transport_flow_control++;
+      }
+    }
+  }
+};
+
 /*******************************************************************************
  * CONTEXT MUTATORS
  */
@@ -777,6 +854,79 @@ static void BM_PumpStreamServerToClient(benchmark::State& state) {
   state.SetBytesProcessed(state.range(0) * state.iterations());
 }
 
+static void TrickleCQNext(TrickledCHTTP2* fixture, void** t, bool* ok,
+                          size_t size) {
+  while (true) {
+    switch (fixture->cq()->AsyncNext(t, ok, gpr_now(GPR_CLOCK_MONOTONIC))) {
+      case CompletionQueue::TIMEOUT:
+        fixture->Step(size);
+        break;
+      case CompletionQueue::SHUTDOWN:
+        GPR_ASSERT(false);
+        break;
+      case CompletionQueue::GOT_EVENT:
+        return;
+    }
+  }
+}
+
+static void BM_PumpStreamServerToClient_Trickle(benchmark::State& state) {
+  EchoTestService::AsyncService service;
+  std::unique_ptr<TrickledCHTTP2> fixture(new TrickledCHTTP2(&service));
+  {
+    EchoResponse send_response;
+    EchoResponse recv_response;
+    if (state.range(0) > 0) {
+      send_response.set_message(std::string(state.range(0), 'a'));
+    }
+    Status recv_status;
+    ServerContext svr_ctx;
+    ServerAsyncReaderWriter<EchoResponse, EchoRequest> response_rw(&svr_ctx);
+    service.RequestBidiStream(&svr_ctx, &response_rw, fixture->cq(),
+                              fixture->cq(), tag(0));
+    std::unique_ptr<EchoTestService::Stub> stub(
+        EchoTestService::NewStub(fixture->channel()));
+    ClientContext cli_ctx;
+    auto request_rw = stub->AsyncBidiStream(&cli_ctx, fixture->cq(), tag(1));
+    int need_tags = (1 << 0) | (1 << 1);
+    void* t;
+    bool ok;
+    while (need_tags) {
+      TrickleCQNext(fixture.get(), &t, &ok, state.range(1));
+      GPR_ASSERT(ok);
+      int i = (int)(intptr_t)t;
+      GPR_ASSERT(need_tags & (1 << i));
+      need_tags &= ~(1 << i);
+    }
+    request_rw->Read(&recv_response, tag(0));
+    while (state.KeepRunning()) {
+      GPR_TIMER_SCOPE("BenchmarkCycle", 0);
+      response_rw.Write(send_response, tag(1));
+      while (true) {
+        TrickleCQNext(fixture.get(), &t, &ok, state.range(1));
+        if (t == tag(0)) {
+          request_rw->Read(&recv_response, tag(0));
+        } else if (t == tag(1)) {
+          break;
+        } else {
+          GPR_ASSERT(false);
+        }
+      }
+    }
+    response_rw.Finish(Status::OK, tag(1));
+    need_tags = (1 << 0) | (1 << 1);
+    while (need_tags) {
+      TrickleCQNext(fixture.get(), &t, &ok, state.range(1));
+      int i = (int)(intptr_t)t;
+      GPR_ASSERT(need_tags & (1 << i));
+      need_tags &= ~(1 << i);
+    }
+  }
+  fixture->Finish(state);
+  fixture.reset();
+  state.SetBytesProcessed(state.range(0) * state.iterations());
+}
+
 /*******************************************************************************
  * CONFIGURATIONS
  */
@@ -866,6 +1016,16 @@ BENCHMARK_TEMPLATE(BM_PumpStreamServerToClient, SockPair)
 BENCHMARK_TEMPLATE(BM_PumpStreamServerToClient, InProcessCHTTP2)
     ->Range(0, 128 * 1024 * 1024);
 
+static void TrickleArgs(benchmark::internal::Benchmark* b) {
+  for (int i = 1; i <= 128 * 1024 * 1024; i *= 8) {
+    for (int j = 1024; j <= 8 * 1024 * 1024; j *= 8) {
+      b->Args({i, j});
+    }
+  }
+}
+
+BENCHMARK(BM_PumpStreamServerToClient_Trickle)->Apply(TrickleArgs);
+
 // Generate Args for StreamingPingPong benchmarks. Currently generates args for
 // only "small streams" (i.e streams with 0, 1 or 2 messages)
 static void StreamingPingPongArgs(benchmark::internal::Benchmark* b) {