Merge github.com:grpc/grpc into fuzzing-memory

9 files changed, 242 insertions, 14 deletions

diff --git a/tools/buildgen/plugins/expand_filegroups.py b/tools/buildgen/plugins/expand_filegroups.py
index 477e69c869..46739f8f10 100755
--- a/tools/buildgen/plugins/expand_filegroups.py
+++ b/tools/buildgen/plugins/expand_filegroups.py
@@ -85,7 +85,7 @@ def mako_plugin(dictionary):
   skips = 0
 
   while todo:
-    assert skips != len(todo), "infinite loop in filegroup uses clauses"
+    assert skips != len(todo), "infinite loop in filegroup uses clauses: %r" % [t['name'] for t in todo]
     # take the first element of the todo list
     cur = todo[0]
     todo = todo[1:]
diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal
index 8382695f66..285b7dd321 100644
--- a/tools/doxygen/Doxyfile.core.internal
+++ b/tools/doxygen/Doxyfile.core.internal
@@ -1301,6 +1301,8 @@ src/core/lib/surface/validate_metadata.c \
 src/core/lib/surface/validate_metadata.h \
 src/core/lib/surface/version.c \
 src/core/lib/transport/README.md \
+src/core/lib/transport/bdp_estimator.c \
+src/core/lib/transport/bdp_estimator.h \
 src/core/lib/transport/byte_stream.c \
 src/core/lib/transport/byte_stream.h \
 src/core/lib/transport/connectivity_state.c \
diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json
index 900f9a6fe7..52d8ae183b 100644
--- a/tools/run_tests/generated/sources_and_headers.json
+++ b/tools/run_tests/generated/sources_and_headers.json
@@ -104,6 +104,23 @@
   }, 
   {
     "deps": [
+      "gpr", 
+      "gpr_test_util", 
+      "grpc", 
+      "grpc_test_util"
+    ], 
+    "headers": [], 
+    "is_filegroup": false, 
+    "language": "c", 
+    "name": "bdp_estimator_test", 
+    "src": [
+      "test/core/transport/bdp_estimator_test.c"
+    ], 
+    "third_party": false, 
+    "type": "target"
+  }, 
+  {
+    "deps": [
       "grpc", 
       "grpc_test_util"
     ], 
@@ -3447,6 +3464,25 @@
   {
     "deps": [
       "gpr", 
+      "gpr_test_util", 
+      "grpc", 
+      "grpc++", 
+      "grpc++_test_util", 
+      "grpc_test_util"
+    ], 
+    "headers": [], 
+    "is_filegroup": false, 
+    "language": "c++", 
+    "name": "writes_per_rpc_test", 
+    "src": [
+      "test/cpp/performance/writes_per_rpc_test.cc"
+    ], 
+    "third_party": false, 
+    "type": "target"
+  }, 
+  {
+    "deps": [
+      "gpr", 
       "grpc"
     ], 
     "headers": [], 
@@ -7096,6 +7132,7 @@
       "src/core/lib/surface/lame_client.h", 
       "src/core/lib/surface/server.h", 
       "src/core/lib/surface/validate_metadata.h", 
+      "src/core/lib/transport/bdp_estimator.h", 
       "src/core/lib/transport/byte_stream.h", 
       "src/core/lib/transport/connectivity_state.h", 
       "src/core/lib/transport/error_utils.h", 
@@ -7320,6 +7357,8 @@
       "src/core/lib/surface/validate_metadata.c", 
       "src/core/lib/surface/validate_metadata.h", 
       "src/core/lib/surface/version.c", 
+      "src/core/lib/transport/bdp_estimator.c", 
+      "src/core/lib/transport/bdp_estimator.h", 
       "src/core/lib/transport/byte_stream.c", 
       "src/core/lib/transport/byte_stream.h", 
       "src/core/lib/transport/connectivity_state.c", 
@@ -7652,6 +7691,7 @@
       "test/core/end2end/fixtures/http_proxy.h", 
       "test/core/end2end/fixtures/proxy.h", 
       "test/core/iomgr/endpoint_tests.h", 
+      "test/core/util/debugger_macros.h", 
       "test/core/util/grpc_profiler.h", 
       "test/core/util/memory_counters.h", 
       "test/core/util/mock_endpoint.h", 
@@ -7675,6 +7715,8 @@
       "test/core/end2end/fixtures/proxy.h", 
       "test/core/iomgr/endpoint_tests.c", 
       "test/core/iomgr/endpoint_tests.h", 
+      "test/core/util/debugger_macros.c", 
+      "test/core/util/debugger_macros.h", 
       "test/core/util/grpc_profiler.c", 
       "test/core/util/grpc_profiler.h", 
       "test/core/util/memory_counters.c", 
diff --git a/tools/run_tests/generated/tests.json b/tools/run_tests/generated/tests.json
index 2739c42916..b4d3308b00 100644
--- a/tools/run_tests/generated/tests.json
+++ b/tools/run_tests/generated/tests.json
@@ -127,6 +127,28 @@
     "flaky": false, 
     "gtest": false, 
     "language": "c", 
+    "name": "bdp_estimator_test", 
+    "platforms": [
+      "linux", 
+      "mac", 
+      "posix", 
+      "windows"
+    ]
+  }, 
+  {
+    "args": [], 
+    "ci_platforms": [
+      "linux", 
+      "mac", 
+      "posix", 
+      "windows"
+    ], 
+    "cpu_cost": 1.0, 
+    "exclude_configs": [], 
+    "exclude_iomgrs": [], 
+    "flaky": false, 
+    "gtest": false, 
+    "language": "c", 
     "name": "bin_decoder_test", 
     "platforms": [
       "linux", 
@@ -2430,7 +2452,9 @@
     ]
   }, 
   {
-    "args": [], 
+    "args": [
+      "--benchmark_min_time=0"
+    ], 
     "ci_platforms": [
       "linux", 
       "mac", 
@@ -3212,6 +3236,26 @@
     "ci_platforms": [
       "linux", 
       "mac", 
+      "posix"
+    ], 
+    "cpu_cost": 0.5, 
+    "exclude_configs": [], 
+    "exclude_iomgrs": [], 
+    "flaky": false, 
+    "gtest": true, 
+    "language": "c++", 
+    "name": "writes_per_rpc_test", 
+    "platforms": [
+      "linux", 
+      "mac", 
+      "posix"
+    ]
+  }, 
+  {
+    "args": [], 
+    "ci_platforms": [
+      "linux", 
+      "mac", 
       "posix", 
       "windows"
     ], 
diff --git a/tools/run_tests/helper_scripts/pre_build_cmake.bat b/tools/run_tests/helper_scripts/pre_build_cmake.bat
new file mode 100644
index 0000000000..c937b9e09f
--- /dev/null
+++ b/tools/run_tests/helper_scripts/pre_build_cmake.bat
@@ -0,0 +1,48 @@
+@rem Copyright 2017, Google Inc.
+@rem All rights reserved.
+@rem
+@rem Redistribution and use in source and binary forms, with or without
+@rem modification, are permitted provided that the following conditions are
+@rem met:
+@rem
+@rem     * Redistributions of source code must retain the above copyright
+@rem notice, this list of conditions and the following disclaimer.
+@rem     * Redistributions in binary form must reproduce the above
+@rem copyright notice, this list of conditions and the following disclaimer
+@rem in the documentation and/or other materials provided with the
+@rem distribution.
+@rem     * Neither the name of Google Inc. nor the names of its
+@rem contributors may be used to endorse or promote products derived from
+@rem this software without specific prior written permission.
+@rem
+@rem THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+@rem "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+@rem LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+@rem A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+@rem OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+@rem SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+@rem LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+@rem DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+@rem THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+@rem (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+@rem OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+setlocal
+
+cd /d %~dp0\..\..\..
+
+mkdir cmake
+cd cmake
+mkdir build
+cd build
+
+@rem TODO(jtattermusch): Stop hardcoding path to yasm once Jenkins workers can locate yasm correctly
+cmake -G "Visual Studio 14 2015" -DgRPC_BUILD_TESTS=ON -DCMAKE_ASM_NASM_COMPILER="C:/Program Files (x86)/yasm/yasm.exe" ../.. || goto :error
+
+endlocal
+
+goto :EOF
+
+:error
+echo Failed!
+exit /b %errorlevel%
diff --git a/tools/run_tests/helper_scripts/pre_build_cmake.sh b/tools/run_tests/helper_scripts/pre_build_cmake.sh
new file mode 100755
index 0000000000..49083f0ede
--- /dev/null
+++ b/tools/run_tests/helper_scripts/pre_build_cmake.sh
@@ -0,0 +1,39 @@
+#!/bin/bash
+# Copyright 2015, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+set -ex
+
+cd $(dirname $0)/../../..
+
+mkdir -p cmake/build
+cd cmake/build
+
+# MSBUILD_CONFIG's values are suitable for cmake as well
+cmake -DgRPC_BUILD_TESTS=ON -DCMAKE_BUILD_TYPE=${MSBUILD_CONFIG} ../..
diff --git a/tools/run_tests/run_performance_tests.py b/tools/run_tests/run_performance_tests.py
index e47c9a40d1..7c04d228ba 100755
--- a/tools/run_tests/run_performance_tests.py
+++ b/tools/run_tests/run_performance_tests.py
@@ -96,16 +96,18 @@ def create_qpsworker_job(language, shortname=None, port=10000, remote_host=None,
     # specify -o output file so perf.data gets collected when worker stopped
     cmdline = perf_cmd + ['-o', '%s-perf.data' % perf_file_base_name] + cmdline
 
+  worker_timeout = 3 * 60
   if remote_host:
     user_at_host = '%s@%s' % (_REMOTE_HOST_USERNAME, remote_host)
     ssh_cmd = ['ssh']
+    cmdline = ['timeout', '%s' % (worker_timeout + 30)] + cmdline
     ssh_cmd.extend([str(user_at_host), 'cd ~/performance_workspace/grpc/ && %s' % ' '.join(cmdline)])
     cmdline = ssh_cmd
 
   jobspec = jobset.JobSpec(
       cmdline=cmdline,
       shortname=shortname,
-      timeout_seconds=5*60,  # workers get restarted after each scenario
+      timeout_seconds=worker_timeout,  # workers get restarted after each scenario
       verbose_success=True)
   return QpsWorkerJob(jobspec, language, host_and_port, perf_file_base_name)
 
diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py
index 9d76725810..a7278ed407 100755
--- a/tools/run_tests/run_tests.py
+++ b/tools/run_tests/run_tests.py
@@ -197,10 +197,17 @@ class CLanguage(object):
   def configure(self, config, args):
     self.config = config
     self.args = args
-    if self.platform == 'windows':
+    if self.args.compiler == 'cmake':
+      _check_arch(self.args.arch, ['default'])
+      self._use_cmake = True
+      self._docker_distro = 'jessie'
+      self._make_options = []
+    elif self.platform == 'windows':
+      self._use_cmake = False
       self._make_options = [_windows_toolset_option(self.args.compiler),
                             _windows_arch_option(self.args.arch)]
     else:
+      self._use_cmake = False
       self._docker_distro, self._make_options = self._compiler_options(self.args.use_docker,
                                                                        self.args.compiler)
     if args.iomgr_platform == "uv":
@@ -220,6 +227,9 @@ class CLanguage(object):
     out = []
     binaries = get_c_tests(self.args.travis, self.test_lang)
     for target in binaries:
+      if self._use_cmake and target.get('boringssl', False):
+        # cmake doesn't build boringssl tests
+        continue
       polling_strategies = (_POLLING_STRATEGIES.get(self.platform, ['all'])
                             if target.get('uses_polling', True)
                             else ['all'])
@@ -253,12 +263,18 @@ class CLanguage(object):
         if self.args.iomgr_platform in target.get('exclude_iomgrs', []):
           continue
         if self.platform == 'windows':
-          binary = 'vsprojects/%s%s/%s.exe' % (
-              'x64/' if self.args.arch == 'x64' else '',
-              _MSBUILD_CONFIG[self.config.build_config],
-              target['name'])
+          if self._use_cmake:
+            binary = 'cmake/build/%s/%s.exe' % (_MSBUILD_CONFIG[self.config.build_config], target['name'])
+          else:
+            binary = 'vsprojects/%s%s/%s.exe' % (
+                'x64/' if self.args.arch == 'x64' else '',
+                _MSBUILD_CONFIG[self.config.build_config],
+                target['name'])
         else:
-          binary = 'bins/%s/%s' % (self.config.build_config, target['name'])
+          if self._use_cmake:
+            binary = 'cmake/build/%s' % target['name']
+          else:
+            binary = 'bins/%s/%s' % (self.config.build_config, target['name'])
         cpu_cost = target['cpu_cost']
         if cpu_cost == 'capacity':
           cpu_cost = multiprocessing.cpu_count()
@@ -313,10 +329,16 @@ class CLanguage(object):
     return self._make_options;
 
   def pre_build_steps(self):
-    if self.platform == 'windows':
-      return [['tools\\run_tests\\helper_scripts\\pre_build_c.bat']]
+    if self._use_cmake:
+      if self.platform == 'windows':
+        return [['tools\\run_tests\\helper_scripts\\pre_build_cmake.bat']]
+      else:
+        return [['tools/run_tests/helper_scripts/pre_build_cmake.sh']]
     else:
-      return []
+      if self.platform == 'windows':
+        return [['tools\\run_tests\\helper_scripts\\pre_build_c.bat']]
+      else:
+        return []
 
   def build_steps(self):
     return []
@@ -328,7 +350,10 @@ class CLanguage(object):
       return [['tools/run_tests/helper_scripts/post_tests_c.sh']]
 
   def makefile_name(self):
-    return 'Makefile'
+    if self._use_cmake:
+      return 'cmake/build/Makefile'
+    else:
+      return 'Makefile'
 
   def _clang_make_options(self, version_suffix=''):
     return ['CC=clang%s' % version_suffix,
@@ -1112,7 +1137,8 @@ argp.add_argument('--compiler',
                            'python2.7', 'python3.4', 'python3.5', 'python3.6', 'pypy', 'pypy3',
                            'node0.12', 'node4', 'node5', 'node6', 'node7',
                            'electron1.3',
-                           'coreclr'],
+                           'coreclr',
+                           'cmake'],
                   default='default',
                   help='Selects compiler to use. Allowed values depend on the platform and language.')
 argp.add_argument('--iomgr_platform',
@@ -1248,6 +1274,12 @@ _check_arch_option(args.arch)
 
 def make_jobspec(cfg, targets, makefile='Makefile'):
   if platform_string() == 'windows':
+    if makefile.startswith('cmake/build/'):
+      return [jobset.JobSpec(['cmake', '--build', '.',
+                              '--target', '%s' % target,
+                              '--config', _MSBUILD_CONFIG[cfg]],
+                             cwd='cmake/build',
+                             timeout_seconds=None) for target in targets]
     extra_args = []
     # better do parallel compilation
     # empirically /m:2 gives the best performance/price and should prevent
@@ -1264,6 +1296,13 @@ def make_jobspec(cfg, targets, makefile='Makefile'):
                       shell=True, timeout_seconds=None)
       for target in targets]
   else:
+    if targets and makefile.startswith('cmake/build/'):
+      # With cmake, we've passed all the build configuration in the pre-build step already
+      return [jobset.JobSpec([os.getenv('MAKE', 'make'),
+                              '-j', '%d' % args.jobs] +
+                             targets,
+                             cwd='cmake/build',
+                             timeout_seconds=None)]
     if targets:
       return [jobset.JobSpec([os.getenv('MAKE', 'make'),
                               '-f', makefile,
diff --git a/tools/run_tests/run_tests_matrix.py b/tools/run_tests/run_tests_matrix.py
index 8f70a6d2ea..a428fb4853 100755
--- a/tools/run_tests/run_tests_matrix.py
+++ b/tools/run_tests/run_tests_matrix.py
@@ -214,6 +214,18 @@ def _create_portability_test_jobs(extra_args=[], inner_jobs=_DEFAULT_INNER_JOBS)
                                   extra_args=extra_args,
                                   inner_jobs=inner_jobs)
 
+  # cmake build for C and C++
+  # TODO(jtattermusch): some of the tests are failing, so we force --build_only
+  # to make sure it's buildable at least.
+  test_jobs += _generate_jobs(languages=['c', 'c++'],
+                              configs=['dbg'],
+                              platforms=['linux', 'windows'],
+                              arch='default',
+                              compiler='cmake',
+                              labels=['portability'],
+                              extra_args=extra_args + ['--build_only'],
+                              inner_jobs=inner_jobs)
+
   test_jobs += _generate_jobs(languages=['python'],
                               configs=['dbg'],
                               platforms=['linux'],