Merge branch 'master' of https://github.com/grpc/grpc into interop_server_split

7 files changed, 35 insertions, 78 deletions

diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal
index 8233da957d..62cf5cc8c7 100644
--- a/tools/doxygen/Doxyfile.core.internal
+++ b/tools/doxygen/Doxyfile.core.internal
@@ -796,6 +796,7 @@ src/core/lib/channel/channel_stack_builder.h \
 src/core/lib/channel/compress_filter.h \
 src/core/lib/channel/connected_channel.h \
 src/core/lib/channel/context.h \
+src/core/lib/channel/handshaker.h \
 src/core/lib/channel/http_client_filter.h \
 src/core/lib/channel/http_server_filter.h \
 src/core/lib/compression/algorithm_metadata.h \
@@ -951,6 +952,7 @@ src/core/lib/channel/channel_stack.c \
 src/core/lib/channel/channel_stack_builder.c \
 src/core/lib/channel/compress_filter.c \
 src/core/lib/channel/connected_channel.c \
+src/core/lib/channel/handshaker.c \
 src/core/lib/channel/http_client_filter.c \
 src/core/lib/channel/http_server_filter.c \
 src/core/lib/compression/compression.c \
diff --git a/tools/gcp/stress_test/stress_test_utils.py b/tools/gcp/stress_test/stress_test_utils.py
index b821fc8fcc..be50af3184 100755
--- a/tools/gcp/stress_test/stress_test_utils.py
+++ b/tools/gcp/stress_test/stress_test_utils.py
@@ -121,7 +121,7 @@ class BigQueryHelper:
       if not page['jobComplete']:
         print('TIMEOUT ERROR: The query %s timed out. Current timeout value is'
               ' %d msec. Returning False (i.e assuming there are no failures)'
-             ) % (query, timeoout_msec)
+             ) % (query, timeout_msec)
         return False
 
       num_failures = int(page['totalRows'])
diff --git a/tools/profiling/latency_profile/profile_analyzer.py b/tools/profiling/latency_profile/profile_analyzer.py
index dad0712d40..48b8e9b950 100755
--- a/tools/profiling/latency_profile/profile_analyzer.py
+++ b/tools/profiling/latency_profile/profile_analyzer.py
@@ -43,6 +43,7 @@ TIME_FROM_SCOPE_START = object()
 TIME_TO_SCOPE_END = object()
 TIME_FROM_STACK_START = object()
 TIME_TO_STACK_END = object()
+TIME_FROM_LAST_IMPORTANT = object()
 
 
 argp = argparse.ArgumentParser(description='Process output of basic_prof builds')
@@ -78,10 +79,14 @@ class ScopeBuilder(object):
     self.call_stack_builder.lines.append(line_item)
 
   def finish(self, line):
-    assert line['tag'] == self.top_line.tag, 'expected %s, got %s; thread=%s; t0=%f t1=%f' % (self.top_line.tag, line['tag'], line['thd'], self.top_line.start_time, line['t'])
+    assert line['tag'] == self.top_line.tag, (
+        'expected %s, got %s; thread=%s; t0=%f t1=%f' %
+        (self.top_line.tag, line['tag'], line['thd'], self.top_line.start_time,
+         line['t']))
     final_time_stamp = line['t']
     assert self.top_line.end_time is None
     self.top_line.end_time = final_time_stamp
+    self.top_line.important = self.top_line.important or line['imp']
     assert SELF_TIME not in self.top_line.times
     self.top_line.times[SELF_TIME] = final_time_stamp - self.top_line.start_time
     for line in self.call_stack_builder.lines[self.first_child_pos:]:
@@ -101,9 +106,14 @@ class CallStackBuilder(object):
     start_time = self.lines[0].start_time
     end_time = self.lines[0].end_time
     self.signature = self.signature.hexdigest()
+    last_important = start_time
     for line in self.lines:
       line.times[TIME_FROM_STACK_START] = line.start_time - start_time
       line.times[TIME_TO_STACK_END] = end_time - line.end_time
+      line.times[TIME_FROM_LAST_IMPORTANT] = line.start_time - last_important
+      if line.important:
+        last_important = line.end_time
+    last_important = end_time
 
   def add(self, line):
     line_type = line['type']
@@ -113,7 +123,9 @@ class CallStackBuilder(object):
       self.stk.append(ScopeBuilder(self, line))
       return False
     elif line_type == '}':
-      assert self.stk, 'expected non-empty stack for closing %s; thread=%s; t=%f' % (line['tag'], line['thd'], line['t'])
+      assert self.stk, (
+          'expected non-empty stack for closing %s; thread=%s; t=%f' %
+          (line['tag'], line['thd'], line['t']))
       self.stk.pop().finish(line)
       if not self.stk:
         self.finish()
@@ -216,9 +228,16 @@ def time_format(idx):
     return ''
   return ent
 
+BANNER = {
+    'simple': 'Count: %(count)d',
+    'html': '<h1>Count: %(count)d</h1>'
+}
+
 FORMAT = [
   ('TAG', lambda line: '..'*line.indent + tidy_tag(line.tag)),
   ('LOC', lambda line: '%s:%d' % (line.filename[line.filename.rfind('/')+1:], line.fileline)),
+  ('IMP', lambda line: '*' if line.important else ''),
+  ('FROM_IMP', time_format(TIME_FROM_LAST_IMPORTANT)),
   ('FROM_STACK_START', time_format(TIME_FROM_STACK_START)),
   ('SELF', time_format(SELF_TIME)),
   ('TO_STACK_END', time_format(TIME_TO_STACK_END)),
@@ -227,11 +246,6 @@ FORMAT = [
   ('TO_SCOPE_END', time_format(TIME_TO_SCOPE_END)),
 ]
 
-BANNER = {
-    'simple': 'Count: %(count)d',
-    'html': '<h1>Count: %(count)d</h1>'
-}
-
 if args.fmt == 'html':
   print '<html>'
   print '<head>'
diff --git a/tools/run_tests/perf_html_report.template b/tools/run_tests/perf_html_report.template
deleted file mode 100644
index c219fa888a..0000000000
--- a/tools/run_tests/perf_html_report.template
+++ /dev/null
@@ -1,21 +0,0 @@
-<!DOCTYPE html>
-<html lang="en">
-<head><title>Performance Test Result</title></head>
-<body>
-  <h2>Performance Test Result</h2> 
-  <table style="width:50%" border="1">
-  <% sorted_test_cases = sorted(resultset.keys()) %>
-  % for test_case in sorted_test_cases:
-    <tr><td bgcolor="#00BFFF" style="width:30%"><b>${test_case}</b></td>
-    <% result = resultset[test_case] %>
-    <td>
-    % for k, v in result.iteritems():
-      ${k}: ${v}<br>
-    % endfor
-    </td>
-    </tr> 
-  % endfor
-  </table>
-
-</body>
-</html>
diff --git a/tools/run_tests/report_utils.py b/tools/run_tests/report_utils.py
index 7188d3dcd7..5648a694cd 100644
--- a/tools/run_tests/report_utils.py
+++ b/tools/run_tests/report_utils.py
@@ -37,8 +37,6 @@ try:
   from mako import exceptions
 except (ImportError):
   pass  # Mako not installed but it is ok. 
-import glob
-import json
 import os
 import string
 import xml.etree.cElementTree as ET
@@ -122,38 +120,3 @@ def render_interop_html_report(
     print(exceptions.text_error_template().render())
     raise
 
-
-def render_perf_html_report(report_dir):
-  """Generate a simple HTML report for the perf tests."""
-  template_file = 'tools/run_tests/perf_html_report.template'
-  try:
-    mytemplate = Template(filename=template_file, format_exceptions=True)
-  except NameError:
-    print('Mako template is not installed. Skipping HTML report generation.')
-    return
-  except IOError as e:
-    print('Failed to find the template %s: %s' % (template_file, e))
-    return
-
-  resultset = {}
-  for result_file in glob.glob(os.path.join(report_dir, '*.json')):
-    with open(result_file, 'r') as f:
-      scenario_result = json.loads(f.read())
-      test_case = scenario_result['scenario']['name']
-      if 'ping_pong' in test_case:
-        latency50 = round(scenario_result['summary']['latency50'], 2)
-        latency99 = round(scenario_result['summary']['latency99'], 2)
-        summary = {'latency50': latency50, 'latency99': latency99}
-      else:
-        summary = {'qps': round(scenario_result['summary']['qps'], 2)}
-      resultset[test_case] = summary
-
-  args = {'resultset': resultset}
-
-  html_file_path = os.path.join(report_dir, 'index.html')
-  try:
-    with open(html_file_path, 'w') as output_file:
-      mytemplate.render_context(Context(output_file, **args))
-  except:
-    print(exceptions.text_error_template().render())
-    raise
diff --git a/tools/run_tests/run_performance_tests.py b/tools/run_tests/run_performance_tests.py
index 5ff9696808..5fdf7a407d 100755
--- a/tools/run_tests/run_performance_tests.py
+++ b/tools/run_tests/run_performance_tests.py
@@ -40,7 +40,6 @@ import multiprocessing
 import os
 import pipes
 import re
-import report_utils
 import subprocess
 import sys
 import tempfile
@@ -55,7 +54,6 @@ os.chdir(_ROOT)
 
 
 _REMOTE_HOST_USERNAME = 'jenkins'
-_REPORT_DIR = 'perf_reports'
 
 
 class QpsWorkerJob:
@@ -105,11 +103,7 @@ def create_scenario_jobspec(scenario_json, workers, remote_host=None,
     cmd += 'BQ_RESULT_TABLE="%s" ' % bq_result_table
   cmd += 'tools/run_tests/performance/run_qps_driver.sh '
   cmd += '--scenarios_json=%s ' % pipes.quote(json.dumps({'scenarios': [scenario_json]}))
-  if not os.path.isdir(_REPORT_DIR):
-    os.makedirs(_REPORT_DIR)
-  report_path = os.path.join(_REPORT_DIR,
-                             '%s-scenario_result.json' % scenario_json['name'])
-  cmd += '--scenario_result_file=%s' % report_path  
+  cmd += '--scenario_result_file=scenario_result.json'
   if remote_host:
     user_at_host = '%s@%s' % (_REMOTE_HOST_USERNAME, remote_host)
     cmd = 'ssh %s "cd ~/performance_workspace/grpc/ && "%s' % (user_at_host, pipes.quote(cmd))
@@ -442,9 +436,6 @@ try:
   jobset.message('START', 'Running scenarios.', do_newline=True)
   num_failures, _ = jobset.run(
       scenarios, newline_on_success=True, maxjobs=1)
-  
-  report_utils.render_perf_html_report(_REPORT_DIR)
-  
   if num_failures == 0:
     jobset.message('SUCCESS',
                    'All scenarios finished successfully.',
diff --git a/tools/run_tests/sources_and_headers.json b/tools/run_tests/sources_and_headers.json
index 6b7ed70da1..4ab784feff 100644
--- a/tools/run_tests/sources_and_headers.json
+++ b/tools/run_tests/sources_and_headers.json
@@ -2133,6 +2133,7 @@
       "gpr_test_util", 
       "grpc", 
       "grpc++", 
+      "grpc++_reflection", 
       "grpc++_test_config", 
       "grpc++_test_util", 
       "grpc_cli_libs", 
@@ -4461,11 +4462,13 @@
   {
     "deps": [
       "grpc++", 
+      "grpc++_reflection", 
       "grpc_plugin_support"
     ], 
     "headers": [
       "test/cpp/util/cli_call.h", 
-      "test/cpp/util/proto_file_parser.h"
+      "test/cpp/util/proto_file_parser.h", 
+      "test/cpp/util/proto_reflection_descriptor_database.h"
     ], 
     "language": "c++", 
     "name": "grpc_cli_libs", 
@@ -4473,7 +4476,9 @@
       "test/cpp/util/cli_call.cc", 
       "test/cpp/util/cli_call.h", 
       "test/cpp/util/proto_file_parser.cc", 
-      "test/cpp/util/proto_file_parser.h"
+      "test/cpp/util/proto_file_parser.h", 
+      "test/cpp/util/proto_reflection_descriptor_database.cc", 
+      "test/cpp/util/proto_reflection_descriptor_database.h"
     ], 
     "third_party": false, 
     "type": "lib"
@@ -5710,6 +5715,7 @@
       "src/core/lib/channel/compress_filter.h", 
       "src/core/lib/channel/connected_channel.h", 
       "src/core/lib/channel/context.h", 
+      "src/core/lib/channel/handshaker.h", 
       "src/core/lib/channel/http_client_filter.h", 
       "src/core/lib/channel/http_server_filter.h", 
       "src/core/lib/compression/algorithm_metadata.h", 
@@ -5803,6 +5809,8 @@
       "src/core/lib/channel/connected_channel.c", 
       "src/core/lib/channel/connected_channel.h", 
       "src/core/lib/channel/context.h", 
+      "src/core/lib/channel/handshaker.c", 
+      "src/core/lib/channel/handshaker.h", 
       "src/core/lib/channel/http_client_filter.c", 
       "src/core/lib/channel/http_client_filter.h", 
       "src/core/lib/channel/http_server_filter.c",