Add a test that measures flow control stalls in a deterministic way

12 files changed, 423 insertions, 13 deletions

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 101f0ad2ef..8edc29e9cc 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -1428,6 +1428,7 @@ add_library(grpc_test_util
   test/core/util/port_uv.c
   test/core/util/port_windows.c
   test/core/util/slice_splitter.c
+  test/core/util/trickle_endpoint.c
   src/core/lib/channel/channel_args.c
   src/core/lib/channel/channel_stack.c
   src/core/lib/channel/channel_stack_builder.c
@@ -1635,6 +1636,7 @@ add_library(grpc_test_util_unsecure
   test/core/util/port_uv.c
   test/core/util/port_windows.c
   test/core/util/slice_splitter.c
+  test/core/util/trickle_endpoint.c
 )
 
 if(WIN32 AND MSVC)
diff --git a/Makefile b/Makefile
index 5dde577cca..9bae21dc45 100644
--- a/Makefile
+++ b/Makefile
@@ -3262,6 +3262,7 @@ LIBGRPC_TEST_UTIL_SRC = \
     test/core/util/port_uv.c \
     test/core/util/port_windows.c \
     test/core/util/slice_splitter.c \
+    test/core/util/trickle_endpoint.c \
     src/core/lib/channel/channel_args.c \
     src/core/lib/channel/channel_stack.c \
     src/core/lib/channel/channel_stack_builder.c \
@@ -3462,6 +3463,7 @@ LIBGRPC_TEST_UTIL_UNSECURE_SRC = \
     test/core/util/port_uv.c \
     test/core/util/port_windows.c \
     test/core/util/slice_splitter.c \
+    test/core/util/trickle_endpoint.c \
 
 PUBLIC_HEADERS_C += \
 
diff --git a/build.yaml b/build.yaml
index 2fbaa2e226..2f390b9319 100644
--- a/build.yaml
+++ b/build.yaml
@@ -596,6 +596,7 @@ filegroups:
   - test/core/util/port.h
   - test/core/util/port_server_client.h
   - test/core/util/slice_splitter.h
+  - test/core/util/trickle_endpoint.h
   src:
   - test/core/end2end/cq_verifier.c
   - test/core/end2end/fake_resolver.c
@@ -613,6 +614,7 @@ filegroups:
   - test/core/util/port_uv.c
   - test/core/util/port_windows.c
   - test/core/util/slice_splitter.c
+  - test/core/util/trickle_endpoint.c
   deps:
   - grpc
   - gpr_test_util
diff --git a/src/core/ext/transport/chttp2/transport/frame_settings.h b/src/core/ext/transport/chttp2/transport/frame_settings.h
index a29dc82106..44137798c0 100644
--- a/src/core/ext/transport/chttp2/transport/frame_settings.h
+++ b/src/core/ext/transport/chttp2/transport/frame_settings.h
@@ -87,7 +87,7 @@ extern const grpc_chttp2_setting_parameters
     grpc_chttp2_settings_parameters[GRPC_CHTTP2_NUM_SETTINGS];
 
 /* Create a settings frame by diffing old & new, and updating old to be new */
-grpc_slice grpc_chttp2_settings_create(uint32_t *old, const uint32_t *new,
+grpc_slice grpc_chttp2_settings_create(uint32_t *old, const uint32_t *newval,
                                        uint32_t force_mask, size_t count);
 /* Create an ack settings frame */
 grpc_slice grpc_chttp2_settings_ack_create(void);
diff --git a/test/core/util/trickle_endpoint.c b/test/core/util/trickle_endpoint.c
new file mode 100644
index 0000000000..8d661e04c6
--- /dev/null
+++ b/test/core/util/trickle_endpoint.c
@@ -0,0 +1,178 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include "test/core/util/passthru_endpoint.h"
+
+#include <inttypes.h>
+#include <string.h>
+
+#include <grpc/support/alloc.h>
+#include <grpc/support/log.h>
+#include <grpc/support/string_util.h>
+#include <grpc/support/useful.h>
+
+#include "src/core/lib/iomgr/sockaddr.h"
+
+#include "src/core/lib/slice/slice_internal.h"
+
+typedef struct {
+  grpc_endpoint base;
+  grpc_endpoint *wrapped;
+
+  gpr_mu mu;
+  grpc_slice_buffer write_buffer;
+  grpc_slice_buffer writing_buffer;
+  grpc_error *error;
+  bool writing;
+} trickle_endpoint;
+
+static void te_read(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep,
+                    grpc_slice_buffer *slices, grpc_closure *cb) {
+  trickle_endpoint *te = (trickle_endpoint *)ep;
+  grpc_endpoint_read(exec_ctx, te->wrapped, slices, cb);
+}
+
+static void te_write(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep,
+                     grpc_slice_buffer *slices, grpc_closure *cb) {
+  trickle_endpoint *te = (trickle_endpoint *)ep;
+  for (size_t i = 0; i < slices->count; i++) {
+    grpc_slice_ref_internal(slices->slices[i]);
+  }
+  grpc_slice_buffer_addn(&te->write_buffer, slices->slices, slices->count);
+  gpr_mu_lock(&te->mu);
+  grpc_closure_sched(exec_ctx, cb, GRPC_ERROR_REF(te->error));
+  gpr_mu_unlock(&te->mu);
+}
+
+static grpc_workqueue *te_get_workqueue(grpc_endpoint *ep) {
+  trickle_endpoint *te = (trickle_endpoint *)ep;
+  return grpc_endpoint_get_workqueue(te->wrapped);
+}
+
+static void te_add_to_pollset(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep,
+                              grpc_pollset *pollset) {
+  trickle_endpoint *te = (trickle_endpoint *)ep;
+  grpc_endpoint_add_to_pollset(exec_ctx, te->wrapped, pollset);
+}
+
+static void te_add_to_pollset_set(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep,
+                                  grpc_pollset_set *pollset_set) {
+  trickle_endpoint *te = (trickle_endpoint *)ep;
+  grpc_endpoint_add_to_pollset_set(exec_ctx, te->wrapped, pollset_set);
+}
+
+static void te_shutdown(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep,
+                        grpc_error *why) {
+  trickle_endpoint *te = (trickle_endpoint *)ep;
+  gpr_mu_lock(&te->mu);
+  if (te->error == GRPC_ERROR_NONE) {
+    te->error = GRPC_ERROR_REF(why);
+  }
+  gpr_mu_unlock(&te->mu);
+  grpc_endpoint_shutdown(exec_ctx, te->wrapped, why);
+}
+
+static void te_destroy(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep) {
+  trickle_endpoint *te = (trickle_endpoint *)ep;
+  grpc_endpoint_destroy(exec_ctx, te->wrapped);
+  gpr_mu_destroy(&te->mu);
+  grpc_slice_buffer_destroy_internal(exec_ctx, &te->write_buffer);
+  grpc_slice_buffer_destroy_internal(exec_ctx, &te->writing_buffer);
+  GRPC_ERROR_UNREF(te->error);
+  gpr_free(te);
+}
+
+static grpc_resource_user *te_get_resource_user(grpc_endpoint *ep) {
+  trickle_endpoint *te = (trickle_endpoint *)ep;
+  return grpc_endpoint_get_resource_user(te->wrapped);
+}
+
+static char *te_get_peer(grpc_endpoint *ep) {
+  trickle_endpoint *te = (trickle_endpoint *)ep;
+  return grpc_endpoint_get_peer(te->wrapped);
+}
+
+static int te_get_fd(grpc_endpoint *ep) {
+  trickle_endpoint *te = (trickle_endpoint *)ep;
+  return grpc_endpoint_get_fd(te->wrapped);
+}
+
+static void te_finish_write(grpc_exec_ctx *exec_ctx, void *arg,
+                            grpc_error *error) {
+  trickle_endpoint *te = arg;
+  gpr_mu_lock(&te->mu);
+  te->writing = false;
+  grpc_slice_buffer_reset_and_unref(&te->writing_buffer);
+  gpr_mu_unlock(&te->mu);
+}
+
+static const grpc_endpoint_vtable vtable = {te_read,
+                                            te_write,
+                                            te_get_workqueue,
+                                            te_add_to_pollset,
+                                            te_add_to_pollset_set,
+                                            te_shutdown,
+                                            te_destroy,
+                                            te_get_resource_user,
+                                            te_get_peer,
+                                            te_get_fd};
+
+grpc_endpoint *grpc_trickle_endpoint_create(grpc_endpoint *wrap) {
+  trickle_endpoint *te = gpr_malloc(sizeof(*te));
+  te->base.vtable = &vtable;
+  te->wrapped = wrap;
+  gpr_mu_init(&te->mu);
+  grpc_slice_buffer_init(&te->write_buffer);
+  grpc_slice_buffer_init(&te->writing_buffer);
+  te->error = GRPC_ERROR_NONE;
+  te->writing = false;
+  return &te->base;
+}
+
+size_t grpc_trickle_endpoint_trickle(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep,
+                                     size_t bytes) {
+  trickle_endpoint *te = (trickle_endpoint *)ep;
+  gpr_mu_lock(&te->mu);
+  if (bytes > 0 && !te->writing) {
+    grpc_slice_buffer_move_first(&te->write_buffer,
+                                 GPR_MIN(bytes, te->write_buffer.length),
+                                 &te->writing_buffer);
+    te->writing = true;
+    grpc_endpoint_write(
+        exec_ctx, te->wrapped, &te->writing_buffer,
+        grpc_closure_create(te_finish_write, te, grpc_schedule_on_exec_ctx));
+  }
+  size_t backlog = te->write_buffer.length;
+  gpr_mu_unlock(&te->mu);
+  return backlog;
+}
diff --git a/test/core/util/trickle_endpoint.h b/test/core/util/trickle_endpoint.h
new file mode 100644
index 0000000000..5f16818ebb
--- /dev/null
+++ b/test/core/util/trickle_endpoint.h
@@ -0,0 +1,45 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef TRICKLE_ENDPOINT_H
+#define TRICKLE_ENDPOINT_H
+
+#include "src/core/lib/iomgr/endpoint.h"
+
+grpc_endpoint *grpc_trickle_endpoint_create(grpc_endpoint *wrap);
+
+/* Allow up to \a bytes through the endpoint. Returns the new backlog. */
+size_t grpc_trickle_endpoint_trickle(grpc_exec_ctx *exec_ctx,
+                                     grpc_endpoint *endpoint, size_t bytes);
+
+#endif
diff --git a/test/cpp/microbenchmarks/bm_fullstack.cc b/test/cpp/microbenchmarks/bm_fullstack.cc
index 9d883e68d7..f83c359961 100644
--- a/test/cpp/microbenchmarks/bm_fullstack.cc
+++ b/test/cpp/microbenchmarks/bm_fullstack.cc
@@ -46,6 +46,7 @@
 
 extern "C" {
 #include "src/core/ext/transport/chttp2/transport/chttp2_transport.h"
+#include "src/core/ext/transport/chttp2/transport/internal.h"
 #include "src/core/lib/channel/channel_args.h"
 #include "src/core/lib/iomgr/endpoint.h"
 #include "src/core/lib/iomgr/endpoint_pair.h"
@@ -57,6 +58,7 @@ extern "C" {
 #include "test/core/util/memory_counters.h"
 #include "test/core/util/passthru_endpoint.h"
 #include "test/core/util/port.h"
+#include "test/core/util/trickle_endpoint.h"
 }
 #include "src/core/lib/profiling/timers.h"
 #include "src/cpp/client/create_channel_internal.h"
@@ -197,7 +199,8 @@ class UDS : public FullstackFixture {
 
 class EndpointPairFixture : public BaseFixture {
  public:
-  EndpointPairFixture(Service* service, grpc_endpoint_pair endpoints) {
+  EndpointPairFixture(Service* service, grpc_endpoint_pair endpoints)
+      : endpoint_pair_(endpoints) {
     ServerBuilder b;
     cq_ = b.AddCompletionQueue(true);
     b.RegisterService(service);
@@ -210,7 +213,7 @@ class EndpointPairFixture : public BaseFixture {
     {
       const grpc_channel_args* server_args =
           grpc_server_get_channel_args(server_->c_server());
-      grpc_transport* transport = grpc_create_chttp2_transport(
+      server_transport_ = grpc_create_chttp2_transport(
           &exec_ctx, server_args, endpoints.server, 0 /* is_client */);
 
       grpc_pollset** pollsets;
@@ -221,9 +224,9 @@ class EndpointPairFixture : public BaseFixture {
         grpc_endpoint_add_to_pollset(&exec_ctx, endpoints.server, pollsets[i]);
       }
 
-      grpc_server_setup_transport(&exec_ctx, server_->c_server(), transport,
-                                  NULL, server_args);
-      grpc_chttp2_transport_start_reading(&exec_ctx, transport, NULL);
+      grpc_server_setup_transport(&exec_ctx, server_->c_server(),
+                                  server_transport_, NULL, server_args);
+      grpc_chttp2_transport_start_reading(&exec_ctx, server_transport_, NULL);
     }
 
     /* create channel */
@@ -233,12 +236,13 @@ class EndpointPairFixture : public BaseFixture {
       ApplyCommonChannelArguments(&args);
 
       grpc_channel_args c_args = args.c_channel_args();
-      grpc_transport* transport =
+      client_transport_ =
           grpc_create_chttp2_transport(&exec_ctx, &c_args, endpoints.client, 1);
-      GPR_ASSERT(transport);
-      grpc_channel* channel = grpc_channel_create(
-          &exec_ctx, "target", &c_args, GRPC_CLIENT_DIRECT_CHANNEL, transport);
-      grpc_chttp2_transport_start_reading(&exec_ctx, transport, NULL);
+      GPR_ASSERT(client_transport_);
+      grpc_channel* channel =
+          grpc_channel_create(&exec_ctx, "target", &c_args,
+                              GRPC_CLIENT_DIRECT_CHANNEL, client_transport_);
+      grpc_chttp2_transport_start_reading(&exec_ctx, client_transport_, NULL);
 
       channel_ = CreateChannelInternal("", channel);
     }
@@ -258,6 +262,11 @@ class EndpointPairFixture : public BaseFixture {
   ServerCompletionQueue* cq() { return cq_.get(); }
   std::shared_ptr<Channel> channel() { return channel_; }
 
+ protected:
+  grpc_endpoint_pair endpoint_pair_;
+  grpc_transport* client_transport_;
+  grpc_transport* server_transport_;
+
  private:
   std::unique_ptr<Server> server_;
   std::unique_ptr<ServerCompletionQueue> cq_;
@@ -295,6 +304,74 @@ class InProcessCHTTP2 : public EndpointPairFixture {
   }
 };
 
+class TrickledCHTTP2 : public EndpointPairFixture {
+ public:
+  TrickledCHTTP2(Service* service)
+      : EndpointPairFixture(service, MakeEndpoints()) {}
+
+  void AddToLabel(std::ostream& out, benchmark::State& state) {
+    out << " writes/iter:"
+        << ((double)stats_.num_writes / (double)state.iterations())
+        << " cli-transport-stalls/iter:"
+        << ((double)
+                client_stats_.streams_stalled_due_to_transport_flow_control /
+            (double)state.iterations())
+        << " cli-stream-stalls/iter:"
+        << ((double)client_stats_.streams_stalled_due_to_stream_flow_control /
+            (double)state.iterations())
+        << " svr-transport-stalls/iter:"
+        << ((double)
+                server_stats_.streams_stalled_due_to_transport_flow_control /
+            (double)state.iterations())
+        << " svr-stream-stalls/iter:"
+        << ((double)server_stats_.streams_stalled_due_to_stream_flow_control /
+            (double)state.iterations());
+  }
+
+  void Step(size_t write_size) {
+    grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
+    size_t client_backlog = grpc_trickle_endpoint_trickle(
+        &exec_ctx, endpoint_pair_.client, write_size);
+    size_t server_backlog = grpc_trickle_endpoint_trickle(
+        &exec_ctx, endpoint_pair_.server, write_size);
+    grpc_exec_ctx_finish(&exec_ctx);
+
+    UpdateStats((grpc_chttp2_transport*)client_transport_, &client_stats_,
+                client_backlog);
+    UpdateStats((grpc_chttp2_transport*)server_transport_, &server_stats_,
+                server_backlog);
+  }
+
+ private:
+  grpc_passthru_endpoint_stats stats_;
+  struct Stats {
+    int streams_stalled_due_to_stream_flow_control = 0;
+    int streams_stalled_due_to_transport_flow_control = 0;
+  };
+  Stats client_stats_;
+  Stats server_stats_;
+
+  grpc_endpoint_pair MakeEndpoints() {
+    grpc_endpoint_pair p;
+    grpc_passthru_endpoint_create(&p.client, &p.server, initialize_stuff.rq(),
+                                  &stats_);
+    p.client = grpc_trickle_endpoint_create(p.client);
+    p.server = grpc_trickle_endpoint_create(p.server);
+    return p;
+  }
+
+  void UpdateStats(grpc_chttp2_transport* t, Stats* s, size_t backlog) {
+    if (backlog == 0) {
+      if (t->lists[GRPC_CHTTP2_LIST_STALLED_BY_STREAM].head != NULL) {
+        s->streams_stalled_due_to_stream_flow_control++;
+      }
+      if (t->lists[GRPC_CHTTP2_LIST_STALLED_BY_TRANSPORT].head != NULL) {
+        s->streams_stalled_due_to_transport_flow_control++;
+      }
+    }
+  }
+};
+
 /*******************************************************************************
  * CONTEXT MUTATORS
  */
@@ -777,6 +854,79 @@ static void BM_PumpStreamServerToClient(benchmark::State& state) {
   state.SetBytesProcessed(state.range(0) * state.iterations());
 }
 
+static void TrickleCQNext(TrickledCHTTP2* fixture, void** t, bool* ok,
+                          size_t size) {
+  while (true) {
+    switch (fixture->cq()->AsyncNext(t, ok, gpr_now(GPR_CLOCK_MONOTONIC))) {
+      case CompletionQueue::TIMEOUT:
+        fixture->Step(size);
+        break;
+      case CompletionQueue::SHUTDOWN:
+        GPR_ASSERT(false);
+        break;
+      case CompletionQueue::GOT_EVENT:
+        return;
+    }
+  }
+}
+
+static void BM_PumpStreamServerToClient_Trickle(benchmark::State& state) {
+  EchoTestService::AsyncService service;
+  std::unique_ptr<TrickledCHTTP2> fixture(new TrickledCHTTP2(&service));
+  {
+    EchoResponse send_response;
+    EchoResponse recv_response;
+    if (state.range(0) > 0) {
+      send_response.set_message(std::string(state.range(0), 'a'));
+    }
+    Status recv_status;
+    ServerContext svr_ctx;
+    ServerAsyncReaderWriter<EchoResponse, EchoRequest> response_rw(&svr_ctx);
+    service.RequestBidiStream(&svr_ctx, &response_rw, fixture->cq(),
+                              fixture->cq(), tag(0));
+    std::unique_ptr<EchoTestService::Stub> stub(
+        EchoTestService::NewStub(fixture->channel()));
+    ClientContext cli_ctx;
+    auto request_rw = stub->AsyncBidiStream(&cli_ctx, fixture->cq(), tag(1));
+    int need_tags = (1 << 0) | (1 << 1);
+    void* t;
+    bool ok;
+    while (need_tags) {
+      TrickleCQNext(fixture.get(), &t, &ok, state.range(1));
+      GPR_ASSERT(ok);
+      int i = (int)(intptr_t)t;
+      GPR_ASSERT(need_tags & (1 << i));
+      need_tags &= ~(1 << i);
+    }
+    request_rw->Read(&recv_response, tag(0));
+    while (state.KeepRunning()) {
+      GPR_TIMER_SCOPE("BenchmarkCycle", 0);
+      response_rw.Write(send_response, tag(1));
+      while (true) {
+        TrickleCQNext(fixture.get(), &t, &ok, state.range(1));
+        if (t == tag(0)) {
+          request_rw->Read(&recv_response, tag(0));
+        } else if (t == tag(1)) {
+          break;
+        } else {
+          GPR_ASSERT(false);
+        }
+      }
+    }
+    response_rw.Finish(Status::OK, tag(1));
+    need_tags = (1 << 0) | (1 << 1);
+    while (need_tags) {
+      TrickleCQNext(fixture.get(), &t, &ok, state.range(1));
+      int i = (int)(intptr_t)t;
+      GPR_ASSERT(need_tags & (1 << i));
+      need_tags &= ~(1 << i);
+    }
+  }
+  fixture->Finish(state);
+  fixture.reset();
+  state.SetBytesProcessed(state.range(0) * state.iterations());
+}
+
 /*******************************************************************************
  * CONFIGURATIONS
  */
@@ -866,6 +1016,16 @@ BENCHMARK_TEMPLATE(BM_PumpStreamServerToClient, SockPair)
 BENCHMARK_TEMPLATE(BM_PumpStreamServerToClient, InProcessCHTTP2)
     ->Range(0, 128 * 1024 * 1024);
 
+static void TrickleArgs(benchmark::internal::Benchmark* b) {
+  for (int i = 1; i <= 128 * 1024 * 1024; i *= 8) {
+    for (int j = 1024; j <= 8 * 1024 * 1024; j *= 8) {
+      b->Args({i, j});
+    }
+  }
+}
+
+BENCHMARK(BM_PumpStreamServerToClient_Trickle)->Apply(TrickleArgs);
+
 // Generate Args for StreamingPingPong benchmarks. Currently generates args for
 // only "small streams" (i.e streams with 0, 1 or 2 messages)
 static void StreamingPingPongArgs(benchmark::internal::Benchmark* b) {
diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json
index 0d5aeb233c..cc4a2da628 100644
--- a/tools/run_tests/generated/sources_and_headers.json
+++ b/tools/run_tests/generated/sources_and_headers.json
@@ -7779,7 +7779,8 @@
       "test/core/util/passthru_endpoint.h", 
       "test/core/util/port.h", 
       "test/core/util/port_server_client.h", 
-      "test/core/util/slice_splitter.h"
+      "test/core/util/slice_splitter.h", 
+      "test/core/util/trickle_endpoint.h"
     ], 
     "is_filegroup": true, 
     "language": "c", 
@@ -7814,7 +7815,9 @@
       "test/core/util/port_uv.c", 
       "test/core/util/port_windows.c", 
       "test/core/util/slice_splitter.c", 
-      "test/core/util/slice_splitter.h"
+      "test/core/util/slice_splitter.h", 
+      "test/core/util/trickle_endpoint.c", 
+      "test/core/util/trickle_endpoint.h"
     ], 
     "third_party": false, 
     "type": "filegroup"
diff --git a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj
index 57cbb2c496..504a3fc927 100644
--- a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj
+++ b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj
@@ -193,6 +193,7 @@
     <ClInclude Include="$(SolutionDir)\..\test\core\util\port.h" />
     <ClInclude Include="$(SolutionDir)\..\test\core\util\port_server_client.h" />
     <ClInclude Include="$(SolutionDir)\..\test\core\util\slice_splitter.h" />
+    <ClInclude Include="$(SolutionDir)\..\test\core\util\trickle_endpoint.h" />
     <ClInclude Include="$(SolutionDir)\..\src\core\lib\channel\channel_args.h" />
     <ClInclude Include="$(SolutionDir)\..\src\core\lib\channel\channel_stack.h" />
     <ClInclude Include="$(SolutionDir)\..\src\core\lib\channel\channel_stack_builder.h" />
@@ -343,6 +344,8 @@
     </ClCompile>
     <ClCompile Include="$(SolutionDir)\..\test\core\util\slice_splitter.c">
     </ClCompile>
+    <ClCompile Include="$(SolutionDir)\..\test\core\util\trickle_endpoint.c">
+    </ClCompile>
     <ClCompile Include="$(SolutionDir)\..\src\core\lib\channel\channel_args.c">
     </ClCompile>
     <ClCompile Include="$(SolutionDir)\..\src\core\lib\channel\channel_stack.c">
diff --git a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters
index 620649a06c..8e51a641f1 100644
--- a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters
+++ b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters
@@ -64,6 +64,9 @@
     <ClCompile Include="$(SolutionDir)\..\test\core\util\slice_splitter.c">
       <Filter>test\core\util</Filter>
     </ClCompile>
+    <ClCompile Include="$(SolutionDir)\..\test\core\util\trickle_endpoint.c">
+      <Filter>test\core\util</Filter>
+    </ClCompile>
     <ClCompile Include="$(SolutionDir)\..\src\core\lib\channel\channel_args.c">
       <Filter>src\core\lib\channel</Filter>
     </ClCompile>
@@ -554,6 +557,9 @@
     <ClInclude Include="$(SolutionDir)\..\test\core\util\slice_splitter.h">
       <Filter>test\core\util</Filter>
     </ClInclude>
+    <ClInclude Include="$(SolutionDir)\..\test\core\util\trickle_endpoint.h">
+      <Filter>test\core\util</Filter>
+    </ClInclude>
     <ClInclude Include="$(SolutionDir)\..\src\core\lib\channel\channel_args.h">
       <Filter>src\core\lib\channel</Filter>
     </ClInclude>
diff --git a/vsprojects/vcxproj/grpc_test_util_unsecure/grpc_test_util_unsecure.vcxproj b/vsprojects/vcxproj/grpc_test_util_unsecure/grpc_test_util_unsecure.vcxproj
index daf92305c4..1ea64654e5 100644
--- a/vsprojects/vcxproj/grpc_test_util_unsecure/grpc_test_util_unsecure.vcxproj
+++ b/vsprojects/vcxproj/grpc_test_util_unsecure/grpc_test_util_unsecure.vcxproj
@@ -161,6 +161,7 @@
     <ClInclude Include="$(SolutionDir)\..\test\core\util\port.h" />
     <ClInclude Include="$(SolutionDir)\..\test\core\util\port_server_client.h" />
     <ClInclude Include="$(SolutionDir)\..\test\core\util\slice_splitter.h" />
+    <ClInclude Include="$(SolutionDir)\..\test\core\util\trickle_endpoint.h" />
   </ItemGroup>
   <ItemGroup>
     <ClCompile Include="$(SolutionDir)\..\test\core\end2end\cq_verifier.c">
@@ -195,6 +196,8 @@
     </ClCompile>
     <ClCompile Include="$(SolutionDir)\..\test\core\util\slice_splitter.c">
     </ClCompile>
+    <ClCompile Include="$(SolutionDir)\..\test\core\util\trickle_endpoint.c">
+    </ClCompile>
   </ItemGroup>
   <ItemGroup>
     <ProjectReference Include="$(SolutionDir)\..\vsprojects\vcxproj\.\gpr\gpr.vcxproj">
diff --git a/vsprojects/vcxproj/grpc_test_util_unsecure/grpc_test_util_unsecure.vcxproj.filters b/vsprojects/vcxproj/grpc_test_util_unsecure/grpc_test_util_unsecure.vcxproj.filters
index c9a1b4c10d..e2ad88c96e 100644
--- a/vsprojects/vcxproj/grpc_test_util_unsecure/grpc_test_util_unsecure.vcxproj.filters
+++ b/vsprojects/vcxproj/grpc_test_util_unsecure/grpc_test_util_unsecure.vcxproj.filters
@@ -49,6 +49,9 @@
     <ClCompile Include="$(SolutionDir)\..\test\core\util\slice_splitter.c">
       <Filter>test\core\util</Filter>
     </ClCompile>
+    <ClCompile Include="$(SolutionDir)\..\test\core\util\trickle_endpoint.c">
+      <Filter>test\core\util</Filter>
+    </ClCompile>
   </ItemGroup>
   <ItemGroup>
     <ClInclude Include="$(SolutionDir)\..\test\core\end2end\cq_verifier.h">
@@ -93,6 +96,9 @@
     <ClInclude Include="$(SolutionDir)\..\test\core\util\slice_splitter.h">
       <Filter>test\core\util</Filter>
     </ClInclude>
+    <ClInclude Include="$(SolutionDir)\..\test\core\util\trickle_endpoint.h">
+      <Filter>test\core\util</Filter>
+    </ClInclude>
   </ItemGroup>
 
   <ItemGroup>