Merge pull request #16142 from apolcyn/optionally_wait_long

Wait a configurable amount of time for benchmark channels to connect

2 files changed, 49 insertions, 3 deletions

diff --git a/test/cpp/qps/client.h b/test/cpp/qps/client.h
index 9d58ea8882..9d7469c9b5 100644
--- a/test/cpp/qps/client.h
+++ b/test/cpp/qps/client.h
@@ -19,6 +19,8 @@
 #ifndef TEST_QPS_CLIENT_H
 #define TEST_QPS_CLIENT_H
 
+#include <stdlib.h>
+
 #include <condition_variable>
 #include <mutex>
 #include <unordered_map>
@@ -34,6 +36,7 @@
 #include "src/proto/grpc/testing/benchmark_service.grpc.pb.h"
 #include "src/proto/grpc/testing/payloads.pb.h"
 
+#include "src/core/lib/gpr/env.h"
 #include "src/cpp/util/core_stats.h"
 #include "test/cpp/qps/histogram.h"
 #include "test/cpp/qps/interarrival.h"
@@ -441,9 +444,24 @@ class ClientImpl : public Client {
     std::unique_ptr<std::thread> WaitForReady() {
       return std::unique_ptr<std::thread>(new std::thread([this]() {
         if (!is_inproc_) {
-          GPR_ASSERT(channel_->WaitForConnected(
-              gpr_time_add(gpr_now(GPR_CLOCK_REALTIME),
-                           gpr_time_from_seconds(10, GPR_TIMESPAN))));
+          int connect_deadline = 10;
+          /* Allow optionally overriding connect_deadline in order
+           * to deal with benchmark environments in which the server
+           * can take a long time to become ready. */
+          char* channel_connect_timeout_str =
+              gpr_getenv("QPS_WORKER_CHANNEL_CONNECT_TIMEOUT");
+          if (channel_connect_timeout_str != nullptr &&
+              strcmp(channel_connect_timeout_str, "") != 0) {
+            connect_deadline = atoi(channel_connect_timeout_str);
+          }
+          gpr_log(GPR_INFO,
+                  "Waiting for up to %d seconds for the channel %p to connect",
+                  connect_deadline, channel_.get());
+          gpr_free(channel_connect_timeout_str);
+          GPR_ASSERT(channel_->WaitForConnected(gpr_time_add(
+              gpr_now(GPR_CLOCK_REALTIME),
+              gpr_time_from_seconds(connect_deadline, GPR_TIMESPAN))));
+          gpr_log(GPR_INFO, "Channel %p connected!", channel_.get());
         }
       }));
     }
diff --git a/tools/run_tests/performance/README.md b/tools/run_tests/performance/README.md
index 2fc1a27c9b..791270ab38 100644
--- a/tools/run_tests/performance/README.md
+++ b/tools/run_tests/performance/README.md
@@ -104,3 +104,31 @@ Example memory profile of grpc-go server, with `go tools pprof`:
 ```
 $ go tool pprof --text --alloc_space http://localhost:<pprof_port>/debug/heap
 ```
+
+### Configuration environment variables:
+
+* QPS_WORKER_CHANNEL_CONNECT_TIMEOUT
+
+  Consuming process: qps_worker
+
+  Type: integer (number of seconds)
+
+  This can be used to configure the amount of time that benchmark
+  clients wait for channels to the benchmark server to become ready.
+  This is useful in certain benchmark environments in which the
+  server can take a long time to become ready. Note: if setting
+  this to a high value, then the scenario config under test should
+  probably also have a large "warmup_seconds".
+
+* QPS_WORKERS
+
+  Consuming process: qps_json_driver
+
+  Type: comma separated list of host:port
+
+  Set this to a comma separated list of QPS worker processes/machines.
+  Each scenario in a scenario config has specifies a certain number
+  of servers, `num_servers`, and the driver will start
+  "benchmark servers"'s on the first `num_server` `host:port` pairs in
+  the comma separated list. The rest will be told to run as clients
+  against the benchmark server.