Merge github.com:grpc/grpc into count_now

16 files changed, 559 insertions, 73 deletions

diff --git a/test/cpp/qps/BUILD b/test/cpp/qps/BUILD
new file mode 100644
index 0000000000..6492b63ec3
--- /dev/null
+++ b/test/cpp/qps/BUILD
@@ -0,0 +1,194 @@
+# Copyright 2017, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+licenses(["notice"])  # 3-clause BSD
+
+cc_library(
+    name = "parse_json",
+    srcs = ["parse_json.cc"],
+    hdrs = ["parse_json.h"],
+    deps = ["//:grpc++"],
+)
+
+cc_library(
+    name = "qps_worker_impl",
+    srcs = [
+        "client_async.cc",
+        "client_sync.cc",
+        "qps_worker.cc",
+        "server_async.cc",
+        "server_sync.cc",
+    ],
+    hdrs = [
+        "client.h",
+        "qps_worker.h",
+        "server.h",
+    ],
+    deps = [
+        ":histogram",
+        ":interarrival",
+        ":usage_timer",
+        "//:grpc",
+        "//:grpc++",
+        "//external:gtest",
+        "//src/proto/grpc/testing:control_proto",
+        "//src/proto/grpc/testing:payloads_proto",
+        "//src/proto/grpc/testing:services_proto",
+        "//test/core/end2end:ssl_test_data",
+        "//test/core/util:gpr_test_util",
+        "//test/core/util:grpc_test_util",
+        "//test/cpp/util:test_util",
+    ],
+)
+
+cc_library(
+    name = "driver_impl",
+    srcs = [
+        "driver.cc",
+        "report.cc",
+    ],
+    hdrs = [
+        "driver.h",
+        "report.h",
+    ],
+    deps = [
+        ":histogram",
+        ":parse_json",
+        ":qps_worker_impl",
+        "//:grpc++",
+        "//src/proto/grpc/testing:control_proto",
+        "//src/proto/grpc/testing:messages_proto",
+        "//src/proto/grpc/testing:services_proto",
+        "//test/core/util:gpr_test_util",
+        "//test/core/util:grpc_test_util",
+    ],
+)
+
+cc_library(
+    name = "benchmark_config",
+    srcs = [
+        "benchmark_config.cc",
+    ],
+    hdrs = [
+        "benchmark_config.h",
+    ],
+    deps = [
+        ":driver_impl",
+        ":histogram",
+        "//:grpc++",
+        "//external:gflags",
+        "//src/proto/grpc/testing:control_proto",
+    ],
+)
+
+cc_library(
+    name = "histogram",
+    hdrs = [
+        "histogram.h",
+        "stats.h",
+    ],
+    deps = ["//:gpr"],
+)
+
+cc_library(
+    name = "interarrival",
+    hdrs = ["interarrival.h"],
+    deps = ["//:grpc++"],
+)
+
+cc_binary(
+    name = "json_run_localhost",
+    srcs = ["json_run_localhost.cc"],
+    deps = [
+        "//:gpr",
+        "//test/core/util:gpr_test_util",
+        "//test/core/util:grpc_test_util",
+        "//test/cpp/util:test_util",
+    ],
+)
+
+cc_test(
+    name = "qps_interarrival_test",
+    srcs = ["qps_interarrival_test.cc"],
+    deps = [
+        ":histogram",
+        ":interarrival",
+    ],
+)
+
+cc_binary(
+    name = "qps_json_driver",
+    srcs = ["qps_json_driver.cc"],
+    deps = [
+        ":benchmark_config",
+        ":driver_impl",
+        "//:grpc++",
+        "//external:gflags",
+    ],
+)
+
+cc_test(
+    name = "qps_openloop_test",
+    srcs = ["qps_openloop_test.cc"],
+    deps = [
+        ":benchmark_config",
+        ":driver_impl",
+        ":qps_worker_impl",
+    ],
+)
+
+cc_test(
+    name = "secure_sync_unary_ping_pong_test",
+    srcs = ["secure_sync_unary_ping_pong_test.cc"],
+    deps = [
+        ":benchmark_config",
+        ":driver_impl",
+        "//:grpc++",
+    ],
+)
+
+cc_library(
+    name = "usage_timer",
+    srcs = ["usage_timer.cc"],
+    hdrs = ["usage_timer.h"],
+    deps = ["//:gpr"],
+)
+
+cc_binary(
+    name = "qps_worker",
+    srcs = ["worker.cc"],
+    deps = [
+        ":qps_worker_impl",
+        "//:grpc++",
+        "//test/core/util:gpr_test_util",
+        "//test/core/util:grpc_test_util",
+        "//test/cpp/util:test_config",
+        "//test/cpp/util:test_util",
+    ],
+)
diff --git a/test/cpp/qps/benchmark_config.cc b/test/cpp/qps/benchmark_config.cc
new file mode 100644
index 0000000000..d33f3e9ae1
--- /dev/null
+++ b/test/cpp/qps/benchmark_config.cc
@@ -0,0 +1,103 @@
+/*
+ *
+ * Copyright 2015, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include "test/cpp/qps/benchmark_config.h"
+#include <gflags/gflags.h>
+#include <grpc++/create_channel.h>
+#include <grpc++/security/credentials.h>
+#include <grpc/support/log.h>
+
+DEFINE_bool(enable_log_reporter, true,
+            "Enable reporting of benchmark results through GprLog");
+
+DEFINE_string(scenario_result_file, "",
+              "Write JSON benchmark report to the file specified.");
+
+DEFINE_string(hashed_id, "", "Hash of the user id");
+
+DEFINE_string(test_name, "", "Name of the test being executed");
+
+DEFINE_string(sys_info, "", "System information");
+
+DEFINE_string(server_address, "localhost:50052",
+              "Address of the performance database server");
+
+DEFINE_string(tag, "", "Optional tag for the test");
+
+DEFINE_string(rpc_reporter_server_address, "",
+              "Server address for rpc reporter to send results to");
+
+DEFINE_bool(enable_rpc_reporter, false, "Enable use of RPC reporter");
+
+// In some distros, gflags is in the namespace google, and in some others,
+// in gflags. This hack is enabling us to find both.
+namespace google {}
+namespace gflags {}
+using namespace google;
+using namespace gflags;
+
+namespace grpc {
+namespace testing {
+
+void InitBenchmark(int* argc, char*** argv, bool remove_flags) {
+  ParseCommandLineFlags(argc, argv, remove_flags);
+}
+
+static std::shared_ptr<Reporter> InitBenchmarkReporters() {
+  auto* composite_reporter = new CompositeReporter;
+  if (FLAGS_enable_log_reporter) {
+    composite_reporter->add(
+        std::unique_ptr<Reporter>(new GprLogReporter("LogReporter")));
+  }
+  if (FLAGS_scenario_result_file != "") {
+    composite_reporter->add(std::unique_ptr<Reporter>(
+        new JsonReporter("JsonReporter", FLAGS_scenario_result_file)));
+  }
+  if (FLAGS_enable_rpc_reporter) {
+    GPR_ASSERT(!FLAGS_rpc_reporter_server_address.empty());
+    composite_reporter->add(std::unique_ptr<Reporter>(new RpcReporter(
+        "RpcReporter",
+        grpc::CreateChannel(FLAGS_rpc_reporter_server_address,
+                            grpc::InsecureChannelCredentials()))));
+  }
+
+  return std::shared_ptr<Reporter>(composite_reporter);
+}
+
+std::shared_ptr<Reporter> GetReporter() {
+  static std::shared_ptr<Reporter> reporter(InitBenchmarkReporters());
+  return reporter;
+}
+
+}  // namespace testing
+}  // namespace grpc
diff --git a/test/cpp/qps/benchmark_config.h b/test/cpp/qps/benchmark_config.h
new file mode 100644
index 0000000000..6b308a15ff
--- /dev/null
+++ b/test/cpp/qps/benchmark_config.h
@@ -0,0 +1,57 @@
+/*
+ *
+ * Copyright 2015, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef GRPC_TEST_CPP_UTIL_BENCHMARK_CONFIG_H
+#define GRPC_TEST_CPP_UTIL_BENCHMARK_CONFIG_H
+
+#include <memory>
+#include <vector>
+
+#include "test/cpp/qps/report.h"
+
+namespace grpc {
+namespace testing {
+
+void InitBenchmark(int* argc, char*** argv, bool remove_flags);
+
+/** Returns the benchmark Reporter instance.
+ *
+ * The returned instance will take care of generating reports for all the actual
+ * reporters configured via the "enable_*_reporter" command line flags (see
+ * benchmark_config.cc). */
+std::shared_ptr<Reporter> GetReporter();
+
+}  // namespace testing
+}  // namespace grpc
+
+#endif  // GRPC_TEST_CPP_UTIL_BENCHMARK_CONFIG_H
diff --git a/test/cpp/qps/client.h b/test/cpp/qps/client.h
index baa9304cc2..25a19a5a74 100644
--- a/test/cpp/qps/client.h
+++ b/test/cpp/qps/client.h
@@ -46,7 +46,7 @@
 #include <grpc/support/log.h>
 #include <grpc/support/time.h>
 
-#include "src/proto/grpc/testing/payloads.grpc.pb.h"
+#include "src/proto/grpc/testing/payloads.pb.h"
 #include "src/proto/grpc/testing/services.grpc.pb.h"
 
 #include "test/cpp/qps/histogram.h"
diff --git a/test/cpp/qps/client_async.cc b/test/cpp/qps/client_async.cc
index 396d308e2a..29a79e7343 100644
--- a/test/cpp/qps/client_async.cc
+++ b/test/cpp/qps/client_async.cc
@@ -63,13 +63,13 @@ class ClientRpcContext {
   virtual ~ClientRpcContext() {}
   // next state, return false if done. Collect stats when appropriate
   virtual bool RunNextState(bool, HistogramEntry* entry) = 0;
-  virtual ClientRpcContext* StartNewClone() = 0;
+  virtual void StartNewClone(CompletionQueue* cq) = 0;
   static void* tag(ClientRpcContext* c) { return reinterpret_cast<void*>(c); }
   static ClientRpcContext* detag(void* t) {
     return reinterpret_cast<ClientRpcContext*>(t);
   }
 
-  virtual void Start(CompletionQueue* cq) = 0;
+  virtual void Start(CompletionQueue* cq, const ClientConfig& config) = 0;
 };
 
 template <class RequestType, class ResponseType>
@@ -94,22 +94,17 @@ class ClientRpcContextUnaryImpl : public ClientRpcContext {
         next_issue_(next_issue),
         start_req_(start_req) {}
   ~ClientRpcContextUnaryImpl() override {}
-  void Start(CompletionQueue* cq) override {
-    cq_ = cq;
-    if (!next_issue_) {  // ready to issue
-      RunNextState(true, nullptr);
-    } else {  // wait for the issue time
-      alarm_.reset(new Alarm(cq_, next_issue_(), ClientRpcContext::tag(this)));
-    }
+  void Start(CompletionQueue* cq, const ClientConfig& config) override {
+    StartInternal(cq);
   }
   bool RunNextState(bool ok, HistogramEntry* entry) override {
     switch (next_state_) {
       case State::READY:
         start_ = UsageTimer::Now();
         response_reader_ = start_req_(stub_, &context_, req_, cq_);
+        next_state_ = State::RESP_DONE;
         response_reader_->Finish(&response_, &status_,
                                  ClientRpcContext::tag(this));
-        next_state_ = State::RESP_DONE;
         return true;
       case State::RESP_DONE:
         if (status_.ok()) {
@@ -123,9 +118,10 @@ class ClientRpcContextUnaryImpl : public ClientRpcContext {
         return false;
     }
   }
-  ClientRpcContext* StartNewClone() override {
-    return new ClientRpcContextUnaryImpl(stub_, req_, next_issue_, start_req_,
-                                         callback_);
+  void StartNewClone(CompletionQueue* cq) override {
+    auto* clone = new ClientRpcContextUnaryImpl(stub_, req_, next_issue_,
+                                                start_req_, callback_);
+    clone->StartInternal(cq);
   }
 
  private:
@@ -147,6 +143,15 @@ class ClientRpcContextUnaryImpl : public ClientRpcContext {
   double start_;
   std::unique_ptr<grpc::ClientAsyncResponseReader<ResponseType>>
       response_reader_;
+
+  void StartInternal(CompletionQueue* cq) {
+    cq_ = cq;
+    if (!next_issue_) {  // ready to issue
+      RunNextState(true, nullptr);
+    } else {  // wait for the issue time
+      alarm_.reset(new Alarm(cq_, next_issue_(), ClientRpcContext::tag(this)));
+    }
+  }
 };
 
 typedef std::forward_list<ClientRpcContext*> context_list;
@@ -185,7 +190,7 @@ class AsyncClient : public ClientImpl<StubType, RequestType> {
         auto* cq = cli_cqs_[t].get();
         auto ctx =
             setup_ctx(channels_[ch].get_stub(), next_issuers_[t], request_);
-        ctx->Start(cq);
+        ctx->Start(cq, config);
       }
       t = (t + 1) % cli_cqs_.size();
     }
@@ -248,8 +253,7 @@ class AsyncClient : public ClientImpl<StubType, RequestType> {
         } else if (!ctx->RunNextState(ok, entry)) {
           // The RPC and callback are done, so clone the ctx
           // and kickstart the new one
-          auto clone = ctx->StartNewClone();
-          clone->Start(cli_cqs_[thread_idx].get());
+          ctx->StartNewClone(cli_cqs_[thread_idx].get());
           // delete the old version
           delete ctx;
         }
@@ -330,10 +334,8 @@ class ClientRpcContextStreamingImpl : public ClientRpcContext {
         next_issue_(next_issue),
         start_req_(start_req) {}
   ~ClientRpcContextStreamingImpl() override {}
-  void Start(CompletionQueue* cq) override {
-    cq_ = cq;
-    stream_ = start_req_(stub_, &context_, cq, ClientRpcContext::tag(this));
-    next_state_ = State::STREAM_IDLE;
+  void Start(CompletionQueue* cq, const ClientConfig& config) override {
+    StartInternal(cq, config.messages_per_stream());
   }
   bool RunNextState(bool ok, HistogramEntry* entry) override {
     while (true) {
@@ -346,9 +348,9 @@ class ClientRpcContextStreamingImpl : public ClientRpcContext {
           }
           break;  // loop around, don't return
         case State::WAIT:
+          next_state_ = State::READY_TO_WRITE;
           alarm_.reset(
               new Alarm(cq_, next_issue_(), ClientRpcContext::tag(this)));
-          next_state_ = State::READY_TO_WRITE;
           return true;
         case State::READY_TO_WRITE:
           if (!ok) {
@@ -369,17 +371,32 @@ class ClientRpcContextStreamingImpl : public ClientRpcContext {
         case State::READ_DONE:
           entry->set_value((UsageTimer::Now() - start_) * 1e9);
           callback_(status_, &response_);
+          if ((messages_per_stream_ != 0) &&
+              (++messages_issued_ >= messages_per_stream_)) {
+            next_state_ = State::WRITES_DONE_DONE;
+            stream_->WritesDone(ClientRpcContext::tag(this));
+            return true;
+          }
           next_state_ = State::STREAM_IDLE;
           break;  // loop around
+        case State::WRITES_DONE_DONE:
+          next_state_ = State::FINISH_DONE;
+          stream_->Finish(&status_, ClientRpcContext::tag(this));
+          return true;
+        case State::FINISH_DONE:
+          next_state_ = State::INVALID;
+          return false;
+          break;
         default:
           GPR_ASSERT(false);
           return false;
       }
     }
   }
-  ClientRpcContext* StartNewClone() override {
-    return new ClientRpcContextStreamingImpl(stub_, req_, next_issue_,
-                                             start_req_, callback_);
+  void StartNewClone(CompletionQueue* cq) override {
+    auto* clone = new ClientRpcContextStreamingImpl(stub_, req_, next_issue_,
+                                                    start_req_, callback_);
+    clone->StartInternal(cq, messages_per_stream_);
   }
 
  private:
@@ -395,7 +412,9 @@ class ClientRpcContextStreamingImpl : public ClientRpcContext {
     WAIT,
     READY_TO_WRITE,
     WRITE_DONE,
-    READ_DONE
+    READ_DONE,
+    WRITES_DONE_DONE,
+    FINISH_DONE
   };
   State next_state_;
   std::function<void(grpc::Status, ResponseType*)> callback_;
@@ -408,6 +427,18 @@ class ClientRpcContextStreamingImpl : public ClientRpcContext {
   double start_;
   std::unique_ptr<grpc::ClientAsyncReaderWriter<RequestType, ResponseType>>
       stream_;
+
+  // Allow a limit on number of messages in a stream
+  int messages_per_stream_;
+  int messages_issued_;
+
+  void StartInternal(CompletionQueue* cq, int messages_per_stream) {
+    cq_ = cq;
+    next_state_ = State::STREAM_IDLE;
+    stream_ = start_req_(stub_, &context_, cq, ClientRpcContext::tag(this));
+    messages_per_stream_ = messages_per_stream;
+    messages_issued_ = 0;
+  }
 };
 
 class AsyncStreamingClient final
@@ -459,13 +490,8 @@ class ClientRpcContextGenericStreamingImpl : public ClientRpcContext {
         next_issue_(next_issue),
         start_req_(start_req) {}
   ~ClientRpcContextGenericStreamingImpl() override {}
-  void Start(CompletionQueue* cq) override {
-    cq_ = cq;
-    const grpc::string kMethodName(
-        "/grpc.testing.BenchmarkService/StreamingCall");
-    stream_ = start_req_(stub_, &context_, kMethodName, cq,
-                         ClientRpcContext::tag(this));
-    next_state_ = State::STREAM_IDLE;
+  void Start(CompletionQueue* cq, const ClientConfig& config) override {
+    StartInternal(cq, config.messages_per_stream());
   }
   bool RunNextState(bool ok, HistogramEntry* entry) override {
     while (true) {
@@ -478,9 +504,9 @@ class ClientRpcContextGenericStreamingImpl : public ClientRpcContext {
           }
           break;  // loop around, don't return
         case State::WAIT:
+          next_state_ = State::READY_TO_WRITE;
           alarm_.reset(
               new Alarm(cq_, next_issue_(), ClientRpcContext::tag(this)));
-          next_state_ = State::READY_TO_WRITE;
           return true;
         case State::READY_TO_WRITE:
           if (!ok) {
@@ -501,17 +527,32 @@ class ClientRpcContextGenericStreamingImpl : public ClientRpcContext {
         case State::READ_DONE:
           entry->set_value((UsageTimer::Now() - start_) * 1e9);
           callback_(status_, &response_);
+          if ((messages_per_stream_ != 0) &&
+              (++messages_issued_ >= messages_per_stream_)) {
+            next_state_ = State::WRITES_DONE_DONE;
+            stream_->WritesDone(ClientRpcContext::tag(this));
+            return true;
+          }
           next_state_ = State::STREAM_IDLE;
           break;  // loop around
+        case State::WRITES_DONE_DONE:
+          next_state_ = State::FINISH_DONE;
+          stream_->Finish(&status_, ClientRpcContext::tag(this));
+          return true;
+        case State::FINISH_DONE:
+          next_state_ = State::INVALID;
+          return false;
+          break;
         default:
           GPR_ASSERT(false);
           return false;
       }
     }
   }
-  ClientRpcContext* StartNewClone() override {
-    return new ClientRpcContextGenericStreamingImpl(stub_, req_, next_issue_,
-                                                    start_req_, callback_);
+  void StartNewClone(CompletionQueue* cq) override {
+    auto* clone = new ClientRpcContextGenericStreamingImpl(
+        stub_, req_, next_issue_, start_req_, callback_);
+    clone->StartInternal(cq, messages_per_stream_);
   }
 
  private:
@@ -527,7 +568,9 @@ class ClientRpcContextGenericStreamingImpl : public ClientRpcContext {
     WAIT,
     READY_TO_WRITE,
     WRITE_DONE,
-    READ_DONE
+    READ_DONE,
+    WRITES_DONE_DONE,
+    FINISH_DONE
   };
   State next_state_;
   std::function<void(grpc::Status, ByteBuffer*)> callback_;
@@ -539,6 +582,21 @@ class ClientRpcContextGenericStreamingImpl : public ClientRpcContext {
   grpc::Status status_;
   double start_;
   std::unique_ptr<grpc::GenericClientAsyncReaderWriter> stream_;
+
+  // Allow a limit on number of messages in a stream
+  int messages_per_stream_;
+  int messages_issued_;
+
+  void StartInternal(CompletionQueue* cq, int messages_per_stream) {
+    cq_ = cq;
+    const grpc::string kMethodName(
+        "/grpc.testing.BenchmarkService/StreamingCall");
+    next_state_ = State::STREAM_IDLE;
+    stream_ = start_req_(stub_, &context_, kMethodName, cq,
+                         ClientRpcContext::tag(this));
+    messages_per_stream_ = messages_per_stream;
+    messages_issued_ = 0;
+  }
 };
 
 static std::unique_ptr<grpc::GenericStub> GenericStubCreator(
diff --git a/test/cpp/qps/client_sync.cc b/test/cpp/qps/client_sync.cc
index a944c45496..f8ce2cccbe 100644
--- a/test/cpp/qps/client_sync.cc
+++ b/test/cpp/qps/client_sync.cc
@@ -142,24 +142,33 @@ class SynchronousStreamingClient final : public SynchronousClient {
   SynchronousStreamingClient(const ClientConfig& config)
       : SynchronousClient(config),
         context_(num_threads_),
-        stream_(num_threads_) {
+        stream_(num_threads_),
+        messages_per_stream_(config.messages_per_stream()),
+        messages_issued_(num_threads_) {
     for (size_t thread_idx = 0; thread_idx < num_threads_; thread_idx++) {
       auto* stub = channels_[thread_idx % channels_.size()].get_stub();
       stream_[thread_idx] = stub->StreamingCall(&context_[thread_idx]);
+      messages_issued_[thread_idx] = 0;
     }
     StartThreads(num_threads_);
   }
   ~SynchronousStreamingClient() {
+    std::vector<std::thread> cleanup_threads;
     for (size_t i = 0; i < num_threads_; i++) {
-      auto stream = &stream_[i];
-      if (*stream) {
-        (*stream)->WritesDone();
-        Status s = (*stream)->Finish();
-        if (!s.ok()) {
-          gpr_log(GPR_ERROR, "Stream %" PRIuPTR " received an error %s", i,
-                  s.error_message().c_str());
+      cleanup_threads.emplace_back([this, i]() {
+        auto stream = &stream_[i];
+        if (*stream) {
+          (*stream)->WritesDone();
+          Status s = (*stream)->Finish();
+          if (!s.ok()) {
+            gpr_log(GPR_ERROR, "Stream %" PRIuPTR " received an error %s", i,
+                    s.error_message().c_str());
+          }
         }
-      }
+      });
+    }
+    for (size_t i = 0; i < num_threads_; i++) {
+      cleanup_threads[i].join();
     }
   }
 
@@ -173,11 +182,19 @@ class SynchronousStreamingClient final : public SynchronousClient {
         stream_[thread_idx]->Read(&responses_[thread_idx])) {
       entry->set_value((UsageTimer::Now() - start) * 1e9);
       // don't set the status since there isn't one yet
-      return true;
+      if ((messages_per_stream_ != 0) &&
+          (++messages_issued_[thread_idx] < messages_per_stream_)) {
+        return true;
+      } else if (messages_per_stream_ == 0) {
+        return true;
+      } else {
+        // Fall through to the below resetting code after finish
+      }
     }
     stream_[thread_idx]->WritesDone();
     Status s = stream_[thread_idx]->Finish();
-    // don't set the value since the stream is failed and shouldn't be timed
+    // don't set the value since this is either a failure (shouldn't be timed)
+    // or a stream-end (already has been timed)
     entry->set_status(s.error_code());
     if (!s.ok()) {
       gpr_log(GPR_ERROR, "Stream %" PRIuPTR " received an error %s", thread_idx,
@@ -187,6 +204,7 @@ class SynchronousStreamingClient final : public SynchronousClient {
     context_[thread_idx].~ClientContext();
     new (&context_[thread_idx]) ClientContext();
     stream_[thread_idx] = stub->StreamingCall(&context_[thread_idx]);
+    messages_issued_[thread_idx] = 0;
     return true;
   }
 
@@ -197,6 +215,8 @@ class SynchronousStreamingClient final : public SynchronousClient {
   std::vector<
       std::unique_ptr<grpc::ClientReaderWriter<SimpleRequest, SimpleResponse>>>
       stream_;
+  const int messages_per_stream_;
+  std::vector<int> messages_issued_;
 };
 
 std::unique_ptr<Client> CreateSynchronousUnaryClient(
diff --git a/test/cpp/qps/driver.h b/test/cpp/qps/driver.h
index e72d30a4ef..dd32a16c87 100644
--- a/test/cpp/qps/driver.h
+++ b/test/cpp/qps/driver.h
@@ -36,7 +36,7 @@
 
 #include <memory>
 
-#include "src/proto/grpc/testing/control.grpc.pb.h"
+#include "src/proto/grpc/testing/control.pb.h"
 #include "test/cpp/qps/histogram.h"
 
 namespace grpc {
diff --git a/test/cpp/qps/histogram.h b/test/cpp/qps/histogram.h
index acb415f0a1..470a394301 100644
--- a/test/cpp/qps/histogram.h
+++ b/test/cpp/qps/histogram.h
@@ -35,7 +35,7 @@
 #define TEST_QPS_HISTOGRAM_H
 
 #include <grpc/support/histogram.h>
-#include "src/proto/grpc/testing/stats.grpc.pb.h"
+#include "src/proto/grpc/testing/stats.pb.h"
 
 namespace grpc {
 namespace testing {
diff --git a/test/cpp/qps/qps_json_driver.cc b/test/cpp/qps/qps_json_driver.cc
index ddaaa7ca75..a906137474 100644
--- a/test/cpp/qps/qps_json_driver.cc
+++ b/test/cpp/qps/qps_json_driver.cc
@@ -31,6 +31,7 @@
  *
  */
 
+#include <iostream>
 #include <memory>
 #include <set>
 
@@ -39,10 +40,10 @@
 #include <gflags/gflags.h>
 #include <grpc/support/log.h>
 
+#include "test/cpp/qps/benchmark_config.h"
 #include "test/cpp/qps/driver.h"
 #include "test/cpp/qps/parse_json.h"
 #include "test/cpp/qps/report.h"
-#include "test/cpp/util/benchmark_config.h"
 
 DEFINE_string(scenarios_file, "",
               "JSON file containing an array of Scenario objects");
diff --git a/test/cpp/qps/qps_openloop_test.cc b/test/cpp/qps/qps_openloop_test.cc
index 70e2709ac0..28b396739f 100644
--- a/test/cpp/qps/qps_openloop_test.cc
+++ b/test/cpp/qps/qps_openloop_test.cc
@@ -36,9 +36,9 @@
 #include <grpc/support/log.h>
 
 #include "test/core/util/test_config.h"
+#include "test/cpp/qps/benchmark_config.h"
 #include "test/cpp/qps/driver.h"
 #include "test/cpp/qps/report.h"
-#include "test/cpp/util/benchmark_config.h"
 
 namespace grpc {
 namespace testing {
diff --git a/test/cpp/qps/qps_test.cc b/test/cpp/qps/qps_test.cc
index f94ea0cb49..7c4e2cfd3e 100644
--- a/test/cpp/qps/qps_test.cc
+++ b/test/cpp/qps/qps_test.cc
@@ -35,9 +35,9 @@
 
 #include <grpc/support/log.h>
 
+#include "test/cpp/qps/benchmark_config.h"
 #include "test/cpp/qps/driver.h"
 #include "test/cpp/qps/report.h"
-#include "test/cpp/util/benchmark_config.h"
 
 namespace grpc {
 namespace testing {
diff --git a/test/cpp/qps/report.cc b/test/cpp/qps/report.cc
index 7f84816421..a9130bf5d4 100644
--- a/test/cpp/qps/report.cc
+++ b/test/cpp/qps/report.cc
@@ -40,6 +40,9 @@
 #include "test/cpp/qps/parse_json.h"
 #include "test/cpp/qps/stats.h"
 
+#include <grpc++/client_context.h>
+#include "src/proto/grpc/testing/services.grpc.pb.h"
+
 namespace grpc {
 namespace testing {
 
@@ -142,5 +145,37 @@ void JsonReporter::ReportCpuUsage(const ScenarioResult& result) {
   // NOP - all reporting is handled by ReportQPS.
 }
 
+void RpcReporter::ReportQPS(const ScenarioResult& result) {
+  grpc::ClientContext context;
+  grpc::Status status;
+  Void dummy;
+
+  gpr_log(GPR_INFO, "RPC reporter sending scenario result to server");
+  status = stub_->ReportScenario(&context, result, &dummy);
+
+  if (status.ok()) {
+    gpr_log(GPR_INFO, "RpcReporter report RPC success!");
+  } else {
+    gpr_log(GPR_ERROR, "RpcReporter report RPC: code: %d. message: %s",
+            status.error_code(), status.error_message().c_str());
+  }
+}
+
+void RpcReporter::ReportQPSPerCore(const ScenarioResult& result) {
+  // NOP - all reporting is handled by ReportQPS.
+}
+
+void RpcReporter::ReportLatency(const ScenarioResult& result) {
+  // NOP - all reporting is handled by ReportQPS.
+}
+
+void RpcReporter::ReportTimes(const ScenarioResult& result) {
+  // NOP - all reporting is handled by ReportQPS.
+}
+
+void RpcReporter::ReportCpuUsage(const ScenarioResult& result) {
+  // NOP - all reporting is handled by ReportQPS.
+}
+
 }  // namespace testing
 }  // namespace grpc
diff --git a/test/cpp/qps/report.h b/test/cpp/qps/report.h
index faf87ff060..1749be98c6 100644
--- a/test/cpp/qps/report.h
+++ b/test/cpp/qps/report.h
@@ -42,6 +42,9 @@
 
 #include "test/cpp/qps/driver.h"
 
+#include <grpc++/channel.h>
+#include "src/proto/grpc/testing/services.grpc.pb.h"
+
 namespace grpc {
 namespace testing {
 
@@ -124,6 +127,21 @@ class JsonReporter : public Reporter {
   const string report_file_;
 };
 
+class RpcReporter : public Reporter {
+ public:
+  RpcReporter(const string& name, std::shared_ptr<grpc::Channel> channel)
+      : Reporter(name), stub_(ReportQpsScenarioService::NewStub(channel)) {}
+
+ private:
+  void ReportQPS(const ScenarioResult& result) override;
+  void ReportQPSPerCore(const ScenarioResult& result) override;
+  void ReportLatency(const ScenarioResult& result) override;
+  void ReportTimes(const ScenarioResult& result) override;
+  void ReportCpuUsage(const ScenarioResult& result) override;
+
+  std::unique_ptr<ReportQpsScenarioService::Stub> stub_;
+};
+
 }  // namespace testing
 }  // namespace grpc
 
diff --git a/test/cpp/qps/secure_sync_unary_ping_pong_test.cc b/test/cpp/qps/secure_sync_unary_ping_pong_test.cc
index d0c47d102a..509d9f89c3 100644
--- a/test/cpp/qps/secure_sync_unary_ping_pong_test.cc
+++ b/test/cpp/qps/secure_sync_unary_ping_pong_test.cc
@@ -35,9 +35,9 @@
 
 #include <grpc/support/log.h>
 
+#include "test/cpp/qps/benchmark_config.h"
 #include "test/cpp/qps/driver.h"
 #include "test/cpp/qps/report.h"
-#include "test/cpp/util/benchmark_config.h"
 
 namespace grpc {
 namespace testing {
diff --git a/test/cpp/qps/server.h b/test/cpp/qps/server.h
index 821d5935be..8fbf37a095 100644
--- a/test/cpp/qps/server.h
+++ b/test/cpp/qps/server.h
@@ -38,8 +38,8 @@
 #include <grpc/support/cpu.h>
 #include <vector>
 
-#include "src/proto/grpc/testing/control.grpc.pb.h"
-#include "src/proto/grpc/testing/messages.grpc.pb.h"
+#include "src/proto/grpc/testing/control.pb.h"
+#include "src/proto/grpc/testing/messages.pb.h"
 #include "test/core/end2end/data/ssl_test_data.h"
 #include "test/core/util/port.h"
 #include "test/cpp/qps/usage_timer.h"
diff --git a/test/cpp/qps/server_async.cc b/test/cpp/qps/server_async.cc
index b3a06aeaf5..b499b82091 100644
--- a/test/cpp/qps/server_async.cc
+++ b/test/cpp/qps/server_async.cc
@@ -103,24 +103,25 @@ class AsyncQpsServerTest final : public grpc::testing::Server {
 
     server_ = builder.BuildAndStart();
 
-    using namespace std::placeholders;
-
     auto process_rpc_bound =
-        std::bind(process_rpc, config.payload_config(), _1, _2);
+        std::bind(process_rpc, config.payload_config(), std::placeholders::_1,
+                  std::placeholders::_2);
 
     for (int i = 0; i < 15000; i++) {
       for (int j = 0; j < num_threads; j++) {
         if (request_unary_function) {
-          auto request_unary =
-              std::bind(request_unary_function, &async_service_, _1, _2, _3,
-                        srv_cqs_[j].get(), srv_cqs_[j].get(), _4);
+          auto request_unary = std::bind(
+              request_unary_function, &async_service_, std::placeholders::_1,
+              std::placeholders::_2, std::placeholders::_3, srv_cqs_[j].get(),
+              srv_cqs_[j].get(), std::placeholders::_4);
           contexts_.emplace_back(
               new ServerRpcContextUnaryImpl(request_unary, process_rpc_bound));
         }
         if (request_streaming_function) {
-          auto request_streaming =
-              std::bind(request_streaming_function, &async_service_, _1, _2,
-                        srv_cqs_[j].get(), srv_cqs_[j].get(), _3);
+          auto request_streaming = std::bind(
+              request_streaming_function, &async_service_,
+              std::placeholders::_1, std::placeholders::_2, srv_cqs_[j].get(),
+              srv_cqs_[j].get(), std::placeholders::_3);
           contexts_.emplace_back(new ServerRpcContextStreamingImpl(
               request_streaming, process_rpc_bound));
         }
@@ -234,18 +235,17 @@ class AsyncQpsServerTest final : public grpc::testing::Server {
         return false;
       }
 
-      ResponseType response;
-
       // Call the RPC processing function
-      grpc::Status status = invoke_method_(&req_, &response);
+      grpc::Status status = invoke_method_(&req_, &response_);
 
       // Have the response writer work and invoke on_finish when done
       next_state_ = &ServerRpcContextUnaryImpl::finisher;
-      response_writer_.Finish(response, status, AsyncQpsServerTest::tag(this));
+      response_writer_.Finish(response_, status, AsyncQpsServerTest::tag(this));
       return true;
     }
     std::unique_ptr<ServerContextType> srv_ctx_;
     RequestType req_;
+    ResponseType response_;
     bool (ServerRpcContextUnaryImpl::*next_state_)(bool);
     std::function<void(ServerContextType *, RequestType *,
                        grpc::ServerAsyncResponseWriter<ResponseType> *, void *)>
@@ -297,11 +297,10 @@ class AsyncQpsServerTest final : public grpc::testing::Server {
     bool read_done(bool ok) {
       if (ok) {
         // invoke the method
-        ResponseType response;
         // Call the RPC processing function
-        grpc::Status status = invoke_method_(&req_, &response);
+        grpc::Status status = invoke_method_(&req_, &response_);
         // initiate the write
-        stream_.Write(response, AsyncQpsServerTest::tag(this));
+        stream_.Write(response_, AsyncQpsServerTest::tag(this));
         next_state_ = &ServerRpcContextStreamingImpl::write_done;
       } else {  // client has sent writes done
         // finish the stream
@@ -325,6 +324,7 @@ class AsyncQpsServerTest final : public grpc::testing::Server {
 
     std::unique_ptr<ServerContextType> srv_ctx_;
     RequestType req_;
+    ResponseType response_;
     bool (ServerRpcContextStreamingImpl::*next_state_)(bool);
     std::function<void(
         ServerContextType *,