Merge remote-tracking branch 'upstream/master' into proto_comments

8 files changed, 673 insertions, 22 deletions

diff --git a/tools/run_tests/performance/bq_upload_result.py b/tools/run_tests/performance/bq_upload_result.py
new file mode 100755
index 0000000000..ebd28f7591
--- /dev/null
+++ b/tools/run_tests/performance/bq_upload_result.py
@@ -0,0 +1,135 @@
+#!/usr/bin/env python2.7
+# Copyright 2016, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+# Uploads performance benchmark result file to bigquery.
+
+import argparse
+import calendar
+import json
+import os
+import sys
+import time
+import uuid
+
+
+gcp_utils_dir = os.path.abspath(os.path.join(
+    os.path.dirname(__file__), '../../gcp/utils'))
+sys.path.append(gcp_utils_dir)
+import big_query_utils
+
+
+_PROJECT_ID='grpc-testing'
+
+
+def _upload_scenario_result_to_bigquery(dataset_id, table_id, result_file):
+  bq = big_query_utils.create_big_query()
+  _create_results_table(bq, dataset_id, table_id)
+
+  with open(result_file, 'r') as f:
+    scenario_result = json.loads(f.read())
+
+  if not _insert_result(bq, dataset_id, table_id, scenario_result):
+    print 'Error uploading result to bigquery.'
+    sys.exit(1)
+
+
+def _insert_result(bq, dataset_id, table_id, scenario_result):
+  _flatten_result_inplace(scenario_result)
+  _populate_metadata_inplace(scenario_result)
+  row = big_query_utils.make_row(str(uuid.uuid4()), scenario_result)
+  return big_query_utils.insert_rows(bq,
+                                     _PROJECT_ID,
+                                     dataset_id,
+                                     table_id,
+                                     [row])
+
+
+def _create_results_table(bq, dataset_id, table_id):
+  with open(os.path.dirname(__file__) + '/scenario_result_schema.json', 'r') as f:
+    table_schema = json.loads(f.read())
+  desc = 'Results of performance benchmarks.'
+  return big_query_utils.create_table2(bq, _PROJECT_ID, dataset_id,
+                               table_id, table_schema, desc)
+
+
+def _flatten_result_inplace(scenario_result):
+  """Bigquery is not really great for handling deeply nested data
+  and repeated fields. To maintain values of some fields while keeping
+  the schema relatively simple, we artificially leave some of the fields
+  as JSON strings.
+  """
+  scenario_result['scenario']['clientConfig'] = json.dumps(scenario_result['scenario']['clientConfig'])
+  scenario_result['scenario']['serverConfig'] = json.dumps(scenario_result['scenario']['serverConfig'])
+  scenario_result['latencies'] = json.dumps(scenario_result['latencies'])
+  for stats in scenario_result['clientStats']:
+    stats['latencies'] = json.dumps(stats['latencies'])
+  scenario_result['serverCores'] = json.dumps(scenario_result['serverCores'])
+
+
+def _populate_metadata_inplace(scenario_result):
+  """Populates metadata based on environment variables set by Jenkins."""
+  # NOTE: Grabbing the Jenkins environment variables will only work if the
+  # driver is running locally on the same machine where Jenkins has started
+  # the job. For our setup, this is currently the case, so just assume that.
+  build_number = os.getenv('BUILD_NUMBER')
+  build_url = os.getenv('BUILD_URL')
+  job_name = os.getenv('JOB_NAME')
+  git_commit = os.getenv('GIT_COMMIT')
+  # actual commit is the actual head of PR that is getting tested
+  git_actual_commit = os.getenv('ghprbActualCommit')
+
+  utc_timestamp = str(calendar.timegm(time.gmtime()))
+  metadata = {'created': utc_timestamp}
+
+  if build_number:
+    metadata['buildNumber'] = build_number
+  if build_url:
+    metadata['buildUrl'] = build_url
+  if job_name:
+    metadata['jobName'] = job_name
+  if git_commit:
+    metadata['gitCommit'] = git_commit
+  if git_actual_commit:
+    metadata['gitActualCommit'] = git_actual_commit
+
+  scenario_result['metadata'] = metadata
+
+
+argp = argparse.ArgumentParser(description='Upload result to big query.')
+argp.add_argument('--bq_result_table', required=True, default=None, type=str,
+                  help='Bigquery "dataset.table" to upload results to.')
+argp.add_argument('--file_to_upload', default='scenario_result.json', type=str,
+                  help='Report file to upload.')
+
+args = argp.parse_args()
+
+dataset_id, table_id = args.bq_result_table.split('.', 2)
+_upload_scenario_result_to_bigquery(dataset_id, table_id, args.file_to_upload)
+print 'Successfully uploaded %s to BigQuery.\n' % args.file_to_upload
diff --git a/tools/run_tests/performance/build_performance.sh b/tools/run_tests/performance/build_performance.sh
index 2c962cba37..0c9211b643 100755
--- a/tools/run_tests/performance/build_performance.sh
+++ b/tools/run_tests/performance/build_performance.sh
@@ -28,6 +28,7 @@
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
+source ~/.rvm/scripts/rvm
 set -ex
 
 cd $(dirname $0)/../../..
@@ -45,8 +46,15 @@ make CONFIG=${CONFIG} EMBED_OPENSSL=true EMBED_ZLIB=true qps_worker qps_driver q
 
 for language in $@
 do
-  if [ "$language" != "c++" ]
-  then
+  case "$language" in
+  "c++")
+    ;;  # C++ has already been built.
+  "java")
+    (cd ../grpc-java/ &&
+      ./gradlew -PskipCodegen=true :grpc-benchmarks:installDist)
+    ;;
+  *)
     tools/run_tests/run_tests.py -l $language -c $CONFIG --build_only -j 8
-  fi
+    ;;
+  esac
 done
diff --git a/tools/run_tests/performance/remote_host_prepare.sh b/tools/run_tests/performance/remote_host_prepare.sh
index 18633d1420..17cfa1a599 100755
--- a/tools/run_tests/performance/remote_host_prepare.sh
+++ b/tools/run_tests/performance/remote_host_prepare.sh
@@ -38,7 +38,12 @@ ssh "${USER_AT_HOST}" "rm -rf ~/performance_workspace && mkdir -p ~/performance_
 # TODO(jtattermusch): To be sure there are no running processes that would
 # mess with the results, be rough and reboot the slave here
 # and wait for it to come back online.
-ssh "${USER_AT_HOST}" "killall qps_worker mono node || true"
+# could also kill jenkins.
+ssh "${USER_AT_HOST}" "killall -9 qps_worker mono node ruby || true"
+
+# Kill all java LoadWorker processes. We can't just killall java
+# as one of the processes might be jenkins.
+ssh "${USER_AT_HOST}" 'kill -9 $(jps | grep LoadWorker | cut -f1 -d" ") || true'
 
 # push the current sources to the slave and unpack it.
 scp ../grpc.tar "${USER_AT_HOST}:~/performance_workspace"
diff --git a/tools/run_tests/performance/run_qps_driver.sh b/tools/run_tests/performance/run_qps_driver.sh
new file mode 100755
index 0000000000..c8c6890df9
--- /dev/null
+++ b/tools/run_tests/performance/run_qps_driver.sh
@@ -0,0 +1,40 @@
+#!/bin/bash
+# Copyright 2015, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+set -ex
+
+cd $(dirname $0)/../../..
+
+bins/opt/qps_json_driver "$@"
+
+if [ "$BQ_RESULT_TABLE" != "" ]
+then
+  tools/run_tests/performance/bq_upload_result.py --bq_result_table="$BQ_RESULT_TABLE"
+fi
diff --git a/tools/run_tests/performance/run_worker_java.sh b/tools/run_tests/performance/run_worker_java.sh
new file mode 100755
index 0000000000..d5503a18a4
--- /dev/null
+++ b/tools/run_tests/performance/run_worker_java.sh
@@ -0,0 +1,39 @@
+#!/bin/bash
+# Copyright 2015, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+set -ex
+
+# Enter repo root
+cd $(dirname $0)/../../..
+
+# Enter the grpc-java repo root (expected to be next to grpc repo root)
+cd ../grpc-java
+
+benchmarks/build/install/grpc-benchmarks/bin/benchmark_worker $@
diff --git a/tools/run_tests/performance/run_worker_ruby.sh b/tools/run_tests/performance/run_worker_ruby.sh
new file mode 100755
index 0000000000..43187345bc
--- /dev/null
+++ b/tools/run_tests/performance/run_worker_ruby.sh
@@ -0,0 +1,36 @@
+#!/bin/bash
+# Copyright 2015, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+source ~/.rvm/scripts/rvm
+set -ex
+
+cd $(dirname $0)/../../..
+
+ruby src/ruby/qps/worker.rb $@
diff --git a/tools/run_tests/performance/scenario_config.py b/tools/run_tests/performance/scenario_config.py
index 7a82d257e4..c63e0dbc38 100644
--- a/tools/run_tests/performance/scenario_config.py
+++ b/tools/run_tests/performance/scenario_config.py
@@ -31,8 +31,14 @@
 
 SINGLE_MACHINE_CORES=8
 WARMUP_SECONDS=5
+JAVA_WARMUP_SECONDS=15  # Java needs more warmup time for JIT to kick in.
 BENCHMARK_SECONDS=30
 
+HISTOGRAM_PARAMS = {
+  'resolution': 0.01,
+  'max_possible': 60e9,
+}
+
 EMPTY_GENERIC_PAYLOAD = {
   'bytebuf_params': {
     'req_size': 0,
@@ -83,7 +89,7 @@ class CXXLanguage:
         secargs = None
 
       yield {
-          'name': 'generic_async_streaming_ping_pong_%s'
+          'name': 'cpp_generic_async_streaming_ping_pong_%s'
                   % secstr,
           'num_servers': 1,
           'num_clients': 1,
@@ -98,6 +104,7 @@ class CXXLanguage:
               'closed_loop': {}
             },
             'payload_config': EMPTY_GENERIC_PAYLOAD,
+            'histogram_params': HISTOGRAM_PARAMS,
           },
           'server_config': {
             'server_type': 'ASYNC_GENERIC_SERVER',
@@ -110,7 +117,7 @@ class CXXLanguage:
           'benchmark_seconds': BENCHMARK_SECONDS
       }
       yield {
-          'name': 'generic_async_streaming_qps_unconstrained_%s'
+          'name': 'cpp_generic_async_streaming_qps_unconstrained_%s'
                   % secstr,
           'num_servers': 1,
           'num_clients': 0,
@@ -125,6 +132,7 @@ class CXXLanguage:
               'closed_loop': {}
             },
             'payload_config': EMPTY_GENERIC_PAYLOAD,
+            'histogram_params': HISTOGRAM_PARAMS,
           },
           'server_config': {
             'server_type': 'ASYNC_GENERIC_SERVER',
@@ -137,7 +145,7 @@ class CXXLanguage:
           'benchmark_seconds': BENCHMARK_SECONDS
       }
       yield {
-          'name': 'generic_async_streaming_qps_one_server_core_%s'
+          'name': 'cpp_generic_async_streaming_qps_one_server_core_%s'
                   % secstr,
           'num_servers': 1,
           'num_clients': 0,
@@ -152,6 +160,7 @@ class CXXLanguage:
               'closed_loop': {}
             },
             'payload_config': EMPTY_GENERIC_PAYLOAD,
+            'histogram_params': HISTOGRAM_PARAMS,
           },
           'server_config': {
             'server_type': 'ASYNC_GENERIC_SERVER',
@@ -164,7 +173,7 @@ class CXXLanguage:
           'benchmark_seconds': BENCHMARK_SECONDS
       }
       yield {
-          'name': 'protobuf_async_qps_unconstrained_%s'
+          'name': 'cpp_protobuf_async_streaming_qps_unconstrained_%s'
                   % secstr,
           'num_servers': 1,
           'num_clients': 0,
@@ -178,20 +187,20 @@ class CXXLanguage:
             'load_params': {
               'closed_loop': {}
             },
-            'payload_config': EMPTY_GENERIC_PAYLOAD,
+            'payload_config': EMPTY_PROTO_PAYLOAD,
+            'histogram_params': HISTOGRAM_PARAMS,
           },
           'server_config': {
-            'server_type': 'ASYNC_GENERIC_SERVER',
+            'server_type': 'ASYNC_SERVER',
             'security_params': secargs,
             'core_limit': SINGLE_MACHINE_CORES/2,
             'async_server_threads': 1,
-            'payload_config': EMPTY_GENERIC_PAYLOAD,
           },
           'warmup_seconds': WARMUP_SECONDS,
           'benchmark_seconds': BENCHMARK_SECONDS
       }
       yield {
-          'name': 'single_channel_throughput_%s'
+          'name': 'cpp_single_channel_throughput_%s'
                   % secstr,
           'num_servers': 1,
           'num_clients': 1,
@@ -206,6 +215,7 @@ class CXXLanguage:
               'closed_loop': {}
             },
             'payload_config': BIG_GENERIC_PAYLOAD,
+            'histogram_params': HISTOGRAM_PARAMS,
           },
           'server_config': {
             'server_type': 'ASYNC_GENERIC_SERVER',
@@ -218,7 +228,7 @@ class CXXLanguage:
           'benchmark_seconds': BENCHMARK_SECONDS
       }
       yield {
-          'name': 'protobuf_async_ping_pong_%s'
+          'name': 'cpp_protobuf_async_ping_pong_%s'
                   % secstr,
           'num_servers': 1,
           'num_clients': 1,
@@ -233,13 +243,13 @@ class CXXLanguage:
               'closed_loop': {}
             },
             'payload_config': EMPTY_PROTO_PAYLOAD,
+            'histogram_params': HISTOGRAM_PARAMS,
           },
           'server_config': {
-            'server_type': 'ASYNC_GENERIC_SERVER',
+            'server_type': 'ASYNC_SERVER',
             'security_params': secargs,
             'core_limit': SINGLE_MACHINE_CORES/2,
             'async_server_threads': 1,
-            'payload_config': EMPTY_PROTO_PAYLOAD,
           },
           'warmup_seconds': WARMUP_SECONDS,
           'benchmark_seconds': BENCHMARK_SECONDS
@@ -262,8 +272,9 @@ class CSharpLanguage:
 
   def scenarios(self):
     # TODO(jtattermusch): add more scenarios
+    secargs = None
     yield {
-        'name': 'csharp_async_generic_streaming_ping_pong',
+        'name': 'csharp_generic_async_streaming_ping_pong',
         'num_servers': 1,
         'num_clients': 1,
         'client_config': {
@@ -277,17 +288,97 @@ class CSharpLanguage:
             'closed_loop': {}
           },
           'payload_config': EMPTY_GENERIC_PAYLOAD,
+          'histogram_params': HISTOGRAM_PARAMS,
         },
         'server_config': {
           'server_type': 'ASYNC_GENERIC_SERVER',
           'security_params': secargs,
-          'core_limit': SINGLE_MACHINE_CORES/2,
+          'core_limit': 0,
           'async_server_threads': 1,
           'payload_config': EMPTY_GENERIC_PAYLOAD,
         },
         'warmup_seconds': WARMUP_SECONDS,
         'benchmark_seconds': BENCHMARK_SECONDS
     }
+    yield {
+        'name': 'csharp_protobuf_async_unary_ping_pong',
+        'num_servers': 1,
+        'num_clients': 1,
+        'client_config': {
+          'client_type': 'ASYNC_CLIENT',
+          'security_params': secargs,
+          'outstanding_rpcs_per_channel': 1,
+          'client_channels': 1,
+          'async_client_threads': 1,
+          'rpc_type': 'UNARY',
+          'load_params': {
+            'closed_loop': {}
+          },
+          'payload_config': EMPTY_PROTO_PAYLOAD,
+          'histogram_params': HISTOGRAM_PARAMS,
+        },
+        'server_config': {
+          'server_type': 'ASYNC_SERVER',
+          'security_params': secargs,
+          'core_limit': 0,
+          'async_server_threads': 1,
+        },
+        'warmup_seconds': WARMUP_SECONDS,
+        'benchmark_seconds': BENCHMARK_SECONDS
+    }
+    yield {
+        'name': 'csharp_protobuf_sync_to_async_unary_ping_pong',
+        'num_servers': 1,
+        'num_clients': 1,
+        'client_config': {
+          'client_type': 'SYNC_CLIENT',
+          'security_params': secargs,
+          'outstanding_rpcs_per_channel': 1,
+          'client_channels': 1,
+          'async_client_threads': 1,
+          'rpc_type': 'UNARY',
+          'load_params': {
+            'closed_loop': {}
+          },
+          'payload_config': EMPTY_PROTO_PAYLOAD,
+          'histogram_params': HISTOGRAM_PARAMS,
+        },
+        'server_config': {
+          'server_type': 'ASYNC_SERVER',
+          'security_params': secargs,
+          'core_limit': 0,
+          'async_server_threads': 1,
+        },
+        'warmup_seconds': WARMUP_SECONDS,
+        'benchmark_seconds': BENCHMARK_SECONDS
+    }
+    yield {
+        'name': 'csharp_to_cpp_protobuf_sync_unary_ping_pong',
+        'num_servers': 1,
+        'num_clients': 1,
+        'client_config': {
+          'client_type': 'SYNC_CLIENT',
+          'security_params': secargs,
+          'outstanding_rpcs_per_channel': 1,
+          'client_channels': 1,
+          'async_client_threads': 1,
+          'rpc_type': 'UNARY',
+          'load_params': {
+            'closed_loop': {}
+          },
+          'payload_config': EMPTY_PROTO_PAYLOAD,
+          'histogram_params': HISTOGRAM_PARAMS,
+        },
+        'server_config': {
+          'server_type': 'SYNC_SERVER',
+          'security_params': secargs,
+          'core_limit': 0,
+          'async_server_threads': 1,
+        },
+        'warmup_seconds': WARMUP_SECONDS,
+        'benchmark_seconds': BENCHMARK_SECONDS,
+        'SERVER_LANGUAGE': 'c++'  # recognized by run_performance_tests.py
+    }
 
   def __str__(self):
     return 'csharp'
@@ -307,8 +398,9 @@ class NodeLanguage:
 
   def scenarios(self):
     # TODO(jtattermusch): add more scenarios
+    secargs = None
     yield {
-        'name': 'node_sync_unary_ping_pong_protobuf',
+        'name': 'node_protobuf_unary_ping_pong',
         'num_servers': 1,
         'num_clients': 1,
         'client_config': {
@@ -317,18 +409,18 @@ class NodeLanguage:
           'outstanding_rpcs_per_channel': 1,
           'client_channels': 1,
           'async_client_threads': 1,
-          'rpc_type': 'STREAMING',
+          'rpc_type': 'UNARY',
           'load_params': {
             'closed_loop': {}
           },
           'payload_config': EMPTY_PROTO_PAYLOAD,
+          'histogram_params': HISTOGRAM_PARAMS,
         },
         'server_config': {
-          'server_type': 'ASYNC_GENERIC_SERVER',
+          'server_type': 'ASYNC_SERVER',
           'security_params': secargs,
-          'core_limit': SINGLE_MACHINE_CORES/2,
+          'core_limit': 0,
           'async_server_threads': 1,
-          'payload_config': EMPTY_PROTO_PAYLOAD,
         },
         'warmup_seconds': WARMUP_SECONDS,
         'benchmark_seconds': BENCHMARK_SECONDS
@@ -338,8 +430,102 @@ class NodeLanguage:
     return 'node'
 
 
+class RubyLanguage:
+
+  def __init__(self):
+    pass
+    self.safename = str(self)
+
+  def worker_cmdline(self):
+    return ['tools/run_tests/performance/run_worker_ruby.sh']
+
+  def worker_port_offset(self):
+    return 300
+
+  def scenarios(self):
+    # TODO(jtattermusch): add more scenarios
+    secargs = None
+    yield {
+        'name': 'ruby_protobuf_unary_ping_pong',
+        'num_servers': 1,
+        'num_clients': 1,
+        'client_config': {
+          'client_type': 'SYNC_CLIENT',
+          'security_params': secargs,
+          'outstanding_rpcs_per_channel': 1,
+          'client_channels': 1,
+          'async_client_threads': 1,
+          'rpc_type': 'UNARY',
+          'load_params': {
+            'closed_loop': {}
+          },
+          'payload_config': EMPTY_PROTO_PAYLOAD,
+          'histogram_params': HISTOGRAM_PARAMS,
+        },
+        'server_config': {
+          'server_type': 'SYNC_SERVER',
+          'security_params': secargs,
+          'core_limit': 0,
+          'async_server_threads': 1,
+        },
+        'warmup_seconds': WARMUP_SECONDS,
+        'benchmark_seconds': BENCHMARK_SECONDS
+    }
+
+  def __str__(self):
+    return 'ruby'
+
+
+class JavaLanguage:
+
+  def __init__(self):
+    pass
+    self.safename = str(self)
+
+  def worker_cmdline(self):
+    return ['tools/run_tests/performance/run_worker_java.sh']
+
+  def worker_port_offset(self):
+    return 400
+
+  def scenarios(self):
+    # TODO(jtattermusch): add more scenarios
+    secargs = None
+    yield {
+        'name': 'java_protobuf_unary_ping_pong',
+        'num_servers': 1,
+        'num_clients': 1,
+        'client_config': {
+          'client_type': 'SYNC_CLIENT',
+          'security_params': secargs,
+          'outstanding_rpcs_per_channel': 1,
+          'client_channels': 1,
+          'async_client_threads': 1,
+          'rpc_type': 'UNARY',
+          'load_params': {
+            'closed_loop': {}
+          },
+          'payload_config': EMPTY_PROTO_PAYLOAD,
+          'histogram_params': HISTOGRAM_PARAMS,
+        },
+        'server_config': {
+          'server_type': 'SYNC_SERVER',
+          'security_params': secargs,
+          'core_limit': 0,
+          'async_server_threads': 1,
+        },
+        'warmup_seconds': JAVA_WARMUP_SECONDS,
+        'benchmark_seconds': BENCHMARK_SECONDS
+    }
+
+  def __str__(self):
+    return 'java'
+
+
 LANGUAGES = {
     'c++' : CXXLanguage(),
     'csharp' : CSharpLanguage(),
     'node' : NodeLanguage(),
+    'ruby' : RubyLanguage(),
+    'java' : JavaLanguage(),
 }
diff --git a/tools/run_tests/performance/scenario_result_schema.json b/tools/run_tests/performance/scenario_result_schema.json
new file mode 100644
index 0000000000..0325414757
--- /dev/null
+++ b/tools/run_tests/performance/scenario_result_schema.json
@@ -0,0 +1,202 @@
+[
+  {
+    "name": "metadata",
+    "type": "RECORD",
+    "mode": "NULLABLE",
+    "fields": [
+      {
+        "name": "buildNumber",
+        "type": "INTEGER",
+        "mode": "NULLABLE"
+      },
+      {
+        "name": "buildUrl",
+        "type": "STRING",
+        "mode": "NULLABLE"
+      },
+      {
+        "name": "jobName",
+        "type": "STRING",
+        "mode": "NULLABLE"
+      },
+      {
+        "name": "gitCommit",
+        "type": "STRING",
+        "mode": "NULLABLE"
+      },
+      {
+        "name": "gitActualCommit",
+        "type": "STRING",
+        "mode": "NULLABLE"
+      },
+      {
+        "name": "created",
+        "type": "TIMESTAMP",
+        "mode": "NULLABLE"
+      }
+    ]
+  },
+  {
+    "name": "scenario",
+    "type": "RECORD",
+    "mode": "NULLABLE",
+    "fields": [
+      {
+        "name": "name",
+        "type": "STRING",
+        "mode": "NULLABLE"
+      },
+      {
+        "name": "clientConfig",
+        "type": "STRING",
+        "mode": "NULLABLE"
+      },
+      {
+        "name": "numClients",
+        "type": "INTEGER",
+        "mode": "NULLABLE"
+      },
+      {
+        "name": "serverConfig",
+        "type": "STRING",
+        "mode": "NULLABLE"
+      },
+      {
+        "name": "numServers",
+        "type": "INTEGER",
+        "mode": "NULLABLE"
+      },
+      {
+        "name": "warmupSeconds",
+        "type": "INTEGER",
+        "mode": "NULLABLE"
+      },
+      {
+        "name": "benchmarkSeconds",
+        "type": "INTEGER",
+        "mode": "NULLABLE"
+      }
+    ]
+  },
+  {
+    "name": "latencies",
+    "type": "STRING",
+    "mode": "NULLABLE"
+  },
+  {
+    "name": "clientStats",
+    "type": "RECORD",
+    "mode": "REPEATED",
+    "fields": [
+      {
+        "name": "latencies",
+        "type": "STRING",
+        "mode": "NULLABLE"
+      },
+      {
+        "name": "timeElapsed",
+        "type": "FLOAT",
+        "mode": "NULLABLE"
+      },
+      {
+        "name": "timeUser",
+        "type": "FLOAT",
+        "mode": "NULLABLE"
+      },
+      {
+        "name": "timeSystem",
+        "type": "FLOAT",
+        "mode": "NULLABLE"
+      }
+    ]
+  },
+  {
+    "name": "serverStats",
+    "type": "RECORD",
+    "mode": "REPEATED",
+    "fields": [
+      {
+        "name": "timeElapsed",
+        "type": "FLOAT",
+        "mode": "NULLABLE"
+      },
+      {
+        "name": "timeUser",
+        "type": "FLOAT",
+        "mode": "NULLABLE"
+      },
+      {
+        "name": "timeSystem",
+        "type": "FLOAT",
+        "mode": "NULLABLE"
+      }
+    ]
+  },
+  {
+    "name": "serverCores",
+    "type": "STRING",
+    "mode": "NULLABLE"
+  },
+  {
+    "name": "summary",
+    "type": "RECORD",
+    "mode": "NULLABLE",
+    "fields": [
+      {
+        "name": "qps",
+        "type": "FLOAT",
+        "mode": "NULLABLE"
+      },
+      {
+        "name": "qpsPerServerCore",
+        "type": "FLOAT",
+        "mode": "NULLABLE"
+      },
+      {
+        "name": "serverSystemTime",
+        "type": "FLOAT",
+        "mode": "NULLABLE"
+      },
+      {
+        "name": "serverUserTime",
+        "type": "FLOAT",
+        "mode": "NULLABLE"
+      },
+      {
+        "name": "clientSystemTime",
+        "type": "FLOAT",
+        "mode": "NULLABLE"
+      },
+      {
+        "name": "clientUserTime",
+        "type": "FLOAT",
+        "mode": "NULLABLE"
+      },
+      {
+        "name": "latency50",
+        "type": "FLOAT",
+        "mode": "NULLABLE"
+      },
+      {
+        "name": "latency90",
+        "type": "FLOAT",
+        "mode": "NULLABLE"
+      },
+      {
+        "name": "latency95",
+        "type": "FLOAT",
+        "mode": "NULLABLE"
+      },
+      {
+        "name": "latency99",
+        "type": "FLOAT",
+        "mode": "NULLABLE"
+      },
+      {
+        "name": "latency999",
+        "type": "FLOAT",
+        "mode": "NULLABLE"
+      }
+    ]
+  }
+]