Merge branch 'master' into externC

14 files changed, 120 insertions, 22 deletions

diff --git a/tools/distrib/build_ruby_environment_macos.sh b/tools/distrib/build_ruby_environment_macos.sh
index c2240ce976..fe0c5a4d70 100644
--- a/tools/distrib/build_ruby_environment_macos.sh
+++ b/tools/distrib/build_ruby_environment_macos.sh
@@ -21,9 +21,10 @@ CROSS_RUBY=`mktemp tmpfile.XXXXXXXX`
 
 curl https://raw.githubusercontent.com/rake-compiler/rake-compiler/v1.0.3/tasks/bin/cross-ruby.rake > $CROSS_RUBY
 
+# See https://github.com/grpc/grpc/issues/12161 for verconf.h patch details
 patch $CROSS_RUBY << EOF
---- cross-ruby.rake	2016-02-05 16:26:53.000000000 -0800
-+++ cross-ruby.rake.patched	2016-02-05 16:27:33.000000000 -0800
+--- cross-ruby.rake 2017-09-27 16:46:00.311020325 +0200
++++ patched 2017-09-27 16:49:46.127016895 +0200
 @@ -133,7 +133,8 @@
      "--host=#{MINGW_HOST}",
      "--target=#{MINGW_TARGET}",
@@ -32,8 +33,16 @@ patch $CROSS_RUBY << EOF
 +    '--enable-static',
 +    '--disable-shared',
      '--disable-install-doc',
-     '--without-tk',
-     '--without-tcl'
+     '--with-ext='
+   ]
+@@ -151,6 +152,7 @@
+ # make
+ file "#{USER_HOME}/builds/#{MINGW_HOST}/#{RUBY_CC_VERSION}/ruby.exe" => ["#{USER_HOME}/builds/#{MINGW_HOST}/#{RUBY_CC_VERSION}/Makefile"] do |t|
+   chdir File.dirname(t.prerequisites.first) do
++    sh "test -s verconf.h || rm -f verconf.h"  # if verconf.h has size 0, make sure it gets re-built by make
+     sh MAKE
+   end
+ end
 EOF
 
 MAKE="make -j8"
diff --git a/tools/distrib/python/grpcio_tools/grpc_version.py b/tools/distrib/python/grpcio_tools/grpc_version.py
index a4178a58c1..db92b10c60 100644
--- a/tools/distrib/python/grpcio_tools/grpc_version.py
+++ b/tools/distrib/python/grpcio_tools/grpc_version.py
@@ -14,4 +14,4 @@
 
 # AUTO-GENERATED FROM `$REPO_ROOT/templates/tools/distrib/python/grpcio_tools/grpc_version.py.template`!!!
 
-VERSION='1.7.0.dev0'
+VERSION='1.8.0.dev0'
diff --git a/tools/dockerfile/test/multilang_jessie_x64/Dockerfile b/tools/dockerfile/test/multilang_jessie_x64/Dockerfile
index 1a4b681576..8e29fcb88c 100644
--- a/tools/dockerfile/test/multilang_jessie_x64/Dockerfile
+++ b/tools/dockerfile/test/multilang_jessie_x64/Dockerfile
@@ -71,6 +71,24 @@ RUN apt-get update && apt-get -y dist-upgrade && apt-get install -y \
 
 RUN nuget update -self
 
+# Install dotnet SDK based on https://www.microsoft.com/net/core#debian
+RUN apt-get update && apt-get install -y curl libunwind8 gettext
+# dotnet-dev-1.0.0-preview2-003131
+RUN curl -sSL -o dotnet100.tar.gz https://go.microsoft.com/fwlink/?LinkID=827530
+RUN mkdir -p /opt/dotnet && tar zxf dotnet100.tar.gz -C /opt/dotnet
+# dotnet-dev-1.0.1
+RUN curl -sSL -o dotnet101.tar.gz https://go.microsoft.com/fwlink/?LinkID=843453
+RUN mkdir -p /opt/dotnet && tar zxf dotnet101.tar.gz -C /opt/dotnet
+RUN ln -s /opt/dotnet/dotnet /usr/local/bin
+
+# Trigger the population of the local package cache
+ENV NUGET_XMLDOC_MODE skip
+RUN mkdir warmup \
+    && cd warmup \
+    && dotnet new \
+    && cd .. \
+    && rm -rf warmup
+
 #=================
 # C++ dependencies
 RUN apt-get update && apt-get -y install libgflags-dev libgtest-dev libc++-dev clang && apt-get clean
diff --git a/tools/doxygen/Doxyfile.c++ b/tools/doxygen/Doxyfile.c++
index d81b7b4d11..eb27eed075 100644
--- a/tools/doxygen/Doxyfile.c++
+++ b/tools/doxygen/Doxyfile.c++
@@ -40,7 +40,7 @@ PROJECT_NAME           = "GRPC C++"
 # could be handy for archiving the generated documentation or if some version
 # control system is used.
 
-PROJECT_NUMBER         = 1.7.0-dev
+PROJECT_NUMBER         = 1.8.0-dev
 
 # Using the PROJECT_BRIEF tag one can provide an optional one line description
 # for a project that appears at the top of each page and should give viewer a
diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal
index eacb40c212..584dd0af57 100644
--- a/tools/doxygen/Doxyfile.c++.internal
+++ b/tools/doxygen/Doxyfile.c++.internal
@@ -40,7 +40,7 @@ PROJECT_NAME           = "GRPC C++"
 # could be handy for archiving the generated documentation or if some version
 # control system is used.
 
-PROJECT_NUMBER         = 1.7.0-dev
+PROJECT_NUMBER         = 1.8.0-dev
 
 # Using the PROJECT_BRIEF tag one can provide an optional one line description
 # for a project that appears at the top of each page and should give viewer a
diff --git a/tools/internal_ci/helper_scripts/prepare_build_macos_rc b/tools/internal_ci/helper_scripts/prepare_build_macos_rc
index bec529f85e..5a5898f94c 100644
--- a/tools/internal_ci/helper_scripts/prepare_build_macos_rc
+++ b/tools/internal_ci/helper_scripts/prepare_build_macos_rc
@@ -51,7 +51,6 @@ export LANG=en_US.UTF-8
 pod repo update  # needed by python
 
 # python
-brew install coreutils  # we need grealpath
 pip install virtualenv --user python
 pip install -U six tox setuptools twisted pyyaml --user python
 export PYTHONPATH=/Library/Python/3.4/site-packages
@@ -59,4 +58,7 @@ export PYTHONPATH=/Library/Python/3.4/site-packages
 # set xcode version for Obj-C tests
 sudo xcode-select -switch /Applications/Xcode_8.2.1.app/Contents/Developer
 
+# TODO(jtattermusch): better debugging of clock skew, remove once not needed
+date
+
 git submodule update --init
diff --git a/tools/internal_ci/linux/grpc_coverage.cfg b/tools/internal_ci/linux/grpc_coverage.cfg
new file mode 100644
index 0000000000..56b7745b21
--- /dev/null
+++ b/tools/internal_ci/linux/grpc_coverage.cfg
@@ -0,0 +1,24 @@
+# Copyright 2017 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Config file for the internal CI (in protobuf text format)
+
+# Location of the continuous shell script in repository.
+build_file: "grpc/tools/internal_ci/linux/grpc_coverage.sh"
+timeout_mins: 420
+action {
+  define_artifacts {
+    regex: "github/grpc/reports/**"
+  }
+}
diff --git a/tools/internal_ci/linux/grpc_coverage.sh b/tools/internal_ci/linux/grpc_coverage.sh
new file mode 100755
index 0000000000..31623d2618
--- /dev/null
+++ b/tools/internal_ci/linux/grpc_coverage.sh
@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+# Copyright 2017 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -ex
+
+# Enter the gRPC repo root
+cd $(dirname $0)/../../..
+
+source tools/internal_ci/helper_scripts/prepare_build_linux_rc
+
+python tools/run_tests/run_tests.py \
+  --use_docker				        \
+  -t                                \
+  -l all                            \
+  -c gcov                           \
+  -x report.xml                     \
+  -j 16
+  
\ No newline at end of file
diff --git a/tools/internal_ci/macos/grpc_build_artifacts.sh b/tools/internal_ci/macos/grpc_build_artifacts.sh
index 09784e3bb4..eb4568c32b 100755
--- a/tools/internal_ci/macos/grpc_build_artifacts.sh
+++ b/tools/internal_ci/macos/grpc_build_artifacts.sh
@@ -27,8 +27,7 @@ python3.5 -m pip install cython setuptools wheel
 python3.6 -m pip install cython setuptools wheel
 
 # needed to build ruby artifacts
-wget https://raw.githubusercontent.com/grpc/grpc/master/tools/distrib/build_ruby_environment_macos.sh
-bash build_ruby_environment_macos.sh
+time bash tools/distrib/build_ruby_environment_macos.sh
 
 gem install rubygems-update
 update_rubygems
diff --git a/tools/internal_ci/macos/grpc_run_tests_matrix.sh b/tools/internal_ci/macos/grpc_run_tests_matrix.sh
index 8e7fd54a62..6e0c2bb487 100755
--- a/tools/internal_ci/macos/grpc_run_tests_matrix.sh
+++ b/tools/internal_ci/macos/grpc_run_tests_matrix.sh
@@ -28,6 +28,9 @@ ps aux | grep port_server\\.py | awk '{print $2}' | xargs kill -9
 # Reveal leftover processes that might be left behind by the build
 ps aux | grep -i kbuilder
 
+# TODO(jtattermusch): better debugging of clock skew, remove once not needed
+date
+
 echo 'Exiting gRPC main test script.'
 
 if [ "$FAILED" != "" ]
diff --git a/tools/jenkins/run_full_performance.sh b/tools/jenkins/run_full_performance.sh
index a9661c7e26..9598fd7734 100755
--- a/tools/jenkins/run_full_performance.sh
+++ b/tools/jenkins/run_full_performance.sh
@@ -21,7 +21,7 @@ cd $(dirname $0)/../..
 
 # run 8core client vs 8core server
 tools/run_tests/run_performance_tests.py \
-    -l c++ csharp node ruby java python go node_express php \
+    -l c++ csharp node ruby java python go node_express php_protobuf_php php_protobuf_c \
     --netperf \
     --category scalable \
     --bq_result_table performance_test.performance_experiment \
diff --git a/tools/run_tests/performance/run_worker_php.sh b/tools/run_tests/performance/run_worker_php.sh
index 5d0c4fa4fd..e524d5286d 100755
--- a/tools/run_tests/performance/run_worker_php.sh
+++ b/tools/run_tests/performance/run_worker_php.sh
@@ -16,13 +16,18 @@
 source ~/.rvm/scripts/rvm
 set -ex
 
-repo=$(dirname $0)/../../..
-
+cd $(dirname $0)/../../..
+repo=$(pwd)
 # First set up all dependences needed for PHP QPS test
 cd $repo
 cd src/php/tests/qps
 composer install
+# Install protobuf C-extension for php
+cd vendor/google/protobuf/php/ext/google/protobuf
+phpize
+./configure
+make
 # The proxy worker for PHP is implemented in Ruby
-cd ../../../..
+cd $repo
 ruby src/ruby/qps/proxy-worker.rb $@
 
diff --git a/tools/run_tests/performance/scenario_config.py b/tools/run_tests/performance/scenario_config.py
index 5efc9f5648..5019358ab3 100644
--- a/tools/run_tests/performance/scenario_config.py
+++ b/tools/run_tests/performance/scenario_config.py
@@ -802,25 +802,32 @@ class RubyLanguage:
 
 class PhpLanguage:
 
-  def __init__(self):
+  def __init__(self, use_protobuf_c_extension=False):
     pass
+    self.use_protobuf_c_extension=use_protobuf_c_extension
     self.safename = str(self)
 
   def worker_cmdline(self):
+    if self.use_protobuf_c_extension:
+        return ['tools/run_tests/performance/run_worker_php.sh -c']
     return ['tools/run_tests/performance/run_worker_php.sh']
 
   def worker_port_offset(self):
     return 800
 
   def scenarios(self):
+    php_extension_mode='php_protobuf_php_extension'
+    if self.use_protobuf_c_extension:
+        php_extension_mode='php_protobuf_c_extension'
+    
     yield _ping_pong_scenario(
-        'php_to_cpp_protobuf_sync_unary_ping_pong', rpc_type='UNARY',
-        client_type='SYNC_CLIENT', server_type='SYNC_SERVER',
+        '%s_to_cpp_protobuf_sync_unary_ping_pong' % php_extension_mode, 
+        rpc_type='UNARY', client_type='SYNC_CLIENT', server_type='SYNC_SERVER',
         server_language='c++', async_server_threads=1)
 
     yield _ping_pong_scenario(
-        'php_to_cpp_protobuf_sync_streaming_ping_pong', rpc_type='STREAMING',
-        client_type='SYNC_CLIENT', server_type='SYNC_SERVER',
+        '%s_to_cpp_protobuf_sync_streaming_ping_pong' % php_extension_mode, 
+        rpc_type='STREAMING', client_type='SYNC_CLIENT', server_type='SYNC_SERVER',
         server_language='c++', async_server_threads=1)
 
   def __str__(self):
@@ -1024,7 +1031,8 @@ LANGUAGES = {
     'node' : NodeLanguage(),
     'node_express': NodeExpressLanguage(),
     'ruby' : RubyLanguage(),
-    'php' : PhpLanguage(),
+    'php_protobuf_php' : PhpLanguage(),
+    'php_protobuf_c' : PhpLanguage(use_protobuf_c_extension=True),
     'java' : JavaLanguage(),
     'python' : PythonLanguage(),
     'go' : GoLanguage(),
diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py
index 6e2115c279..803f2decbc 100755
--- a/tools/run_tests/run_tests.py
+++ b/tools/run_tests/run_tests.py
@@ -701,7 +701,7 @@ class PythonLanguage(object):
     return [config.build for config in self.pythons]
 
   def post_tests_steps(self):
-    if self.config != 'gcov':
+    if self.config.build_config != 'gcov':
       return []
     else:
       return [['tools/run_tests/helper_scripts/post_tests_python.sh']]
@@ -1641,7 +1641,7 @@ def _build_and_run(
                                            suite_name=args.report_suite_name)
 
   number_failures, _ = jobset.run(
-      post_tests_steps, maxjobs=1, stop_on_failure=True,
+      post_tests_steps, maxjobs=1, stop_on_failure=False,
       newline_on_success=newline_on_success, travis=args.travis)
 
   out = []