7 files changed, 149 insertions, 38 deletions
diff --git a/tools/gcp/utils/big_query_utils.py b/tools/gcp/utils/big_query_utils.py
index 76c86645b7..77a5f5691e 100755
--- a/tools/gcp/utils/big_query_utils.py
+++ b/tools/gcp/utils/big_query_utils.py
@@ -116,6 +116,33 @@ def create_table2(big_query, project_id, dataset_id, table_id, fields_schema,
   return is_success
 
 
+def patch_table(big_query, project_id, dataset_id, table_id, fields_schema):
+  is_success = True
+
+  body = {
+      'schema': {
+          'fields': fields_schema
+      },
+      'tableReference': {
+          'datasetId': dataset_id,
+          'projectId': project_id,
+          'tableId': table_id
+      }
+  }
+
+  try:
+    table_req = big_query.tables().patch(projectId=project_id,
+                                         datasetId=dataset_id,
+                                         tableId=table_id,
+                                         body=body)
+    res = table_req.execute(num_retries=NUM_RETRIES)
+    print 'Successfully patched %s "%s"' % (res['kind'], res['id'])
+  except HttpError as http_error:
+    print 'Error in creating table: %s. Err: %s' % (table_id, http_error)
+    is_success = False
+  return is_success
+
+
 def insert_rows(big_query, project_id, dataset_id, table_id, rows_list):
   is_success = True
   body = {'rows': rows_list}
diff --git a/tools/internal_ci/helper_scripts/prepare_build_macos_rc b/tools/internal_ci/helper_scripts/prepare_build_macos_rc
index 3851e565a4..22e80d2afa 100644
--- a/tools/internal_ci/helper_scripts/prepare_build_macos_rc
+++ b/tools/internal_ci/helper_scripts/prepare_build_macos_rc
@@ -15,53 +15,37 @@
 
 # Source this rc script to prepare the environment for macos builds
 
-# TODO(jtattermusch): remove all deps once installed on MacOS workers
+ulimit -n 1000
 
-# brew and C++ deps
-yes | ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)"
-brew install autoconf automake libtool ccache cmake gflags gpg wget
+# show current limits
+ulimit -a
 
-# TODO(jtattermusch): hkp://keys.gnupg.net fails with "No route to host"
-gpg --keyserver hkp://193.164.133.100 --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3
-curl -sSL https://get.rvm.io | sudo bash -s stable
 
-# add ourselves to rvm group to prevent later "access denied" errors.
-sudo dseditgroup -o edit -a `whoami` -t user rvm
-
-set +ex
-source /etc/profile.d/rvm.sh
-rvm install ruby-2.3
-gem install bundler
+# required to build protobuf
+brew install gflags
 
+set +ex  # rvm script is very verbose and exits with errorcode
+source $HOME/.rvm/scripts/rvm
+set -e  # rvm commands are very verbose  
+rvm install ruby-2.4
 rvm osx-ssl-certs status all
 rvm osx-ssl-certs update all
 set -ex
 
+gem install bundler
+
 # cocoapods
-gem install cocoapods --version 1.0.0
+export LANG=en_US.UTF-8
+gem install cocoapods
+pod repo update  # needed by python
 
 # python
-wget -q https://bootstrap.pypa.io/get-pip.py
-sudo python get-pip.py
+brew install coreutils  # we need grealpath
 sudo pip install virtualenv
+sudo pip install -U six tox setuptools
 
-# TODO(jtattermusch): install python3
-
-# mono
-wget -q https://download.mono-project.com/archive/5.0.1/macos-10-universal/MonoFramework-MDK-5.0.1.1.macos10.xamarin.universal.pkg
-sudo installer -pkg MonoFramework-MDK-5.0.1.1.macos10.xamarin.universal.pkg -target /
-ln -s /Library/Frameworks/Mono.framework/Versions/Current/bin/mono /usr/local/bin/mono
- 
-# dotnet SDK
-brew install openssl
-wget -q https://go.microsoft.com/fwlink/?linkid=843444 -O dotnet-dev-osx-x64.1.0.1.pkg
-sudo installer -pkg dotnet-dev-osx-x64.1.0.1.pkg -target /
-ln -s /usr/local/share/dotnet/dotnet /usr/local/bin/dotnet
-dotnet --version  # bootstrap dotnet SDK
-
-# nvm
-wget -qO- https://raw.githubusercontent.com/creationix/nvm/v0.30.2/install.sh | bash
-
-# TODO(jtattermusch): install node if needed
+# python 3.4
+wget -q https://www.python.org/ftp/python/3.4.4/python-3.4.4-macosx10.6.pkg
+sudo installer -pkg python-3.4.4-macosx10.6.pkg -target /
 
 git submodule update --init
diff --git a/tools/internal_ci/macos/grpc_build_artifacts.sh b/tools/internal_ci/macos/grpc_build_artifacts.sh
index aad99b068d..603c15f210 100755
--- a/tools/internal_ci/macos/grpc_build_artifacts.sh
+++ b/tools/internal_ci/macos/grpc_build_artifacts.sh
@@ -18,6 +18,31 @@ set -ex
 # change to grpc repo root
 cd $(dirname $0)/../../..
 
-git submodule update --init
+source tools/internal_ci/helper_scripts/prepare_build_macos_rc
+
+# python 3.5
+wget -q https://www.python.org/ftp/python/3.5.2/python-3.5.2-macosx10.6.pkg
+sudo installer -pkg python-3.5.2-macosx10.6.pkg -target /
+
+# install cython for all python versions
+python2.7 -m pip install cython setuptools wheel
+python3.4 -m pip install cython setuptools wheel
+python3.5 -m pip install cython setuptools wheel
+python3.6 -m pip install cython setuptools wheel
+
+# node-gyp (needed for node artifacts)
+npm install -g node-gyp
+
+# php dependencies: pecl
+curl -O http://pear.php.net/go-pear.phar
+sudo php -d detect_unicode=0 go-pear.phar
+
+# needed to build ruby artifacts
+gem install rake-compiler
+wget https://raw.githubusercontent.com/grpc/grpc/master/tools/distrib/build_ruby_environment_macos.sh
+bash build_ruby_environment_macos.sh
+
+gem install rubygems-update
+update_rubygems
 
 tools/run_tests/task_runner.py -f artifact macos
diff --git a/tools/internal_ci/macos/grpc_master.sh b/tools/internal_ci/macos/grpc_master.sh
index 786859be3f..c64666b2de 100755
--- a/tools/internal_ci/macos/grpc_master.sh
+++ b/tools/internal_ci/macos/grpc_master.sh
@@ -20,7 +20,7 @@ cd $(dirname $0)/../../..
 
 source tools/internal_ci/helper_scripts/prepare_build_macos_rc
 
-tools/run_tests/run_tests_matrix.py -f basictests macos --internal_ci || FAILED="true"
+tools/run_tests/run_tests_matrix.py -f basictests macos --internal_ci -j 2 --inner_jobs 4 || FAILED="true"
 
 # kill port_server.py to prevent the build from hanging
 ps aux | grep port_server\\.py | awk '{print $2}' | xargs kill -9
diff --git a/tools/run_tests/performance/patch_scenario_results_schema.py b/tools/run_tests/performance/patch_scenario_results_schema.py
new file mode 100755
index 0000000000..81ba5381b3
--- /dev/null
+++ b/tools/run_tests/performance/patch_scenario_results_schema.py
@@ -0,0 +1,55 @@
+#!/usr/bin/env python
+# Copyright 2016 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Use to patch schema of existing scenario results tables (after adding fields).
+
+from __future__ import print_function
+
+import argparse
+import calendar
+import json
+import os
+import sys
+import time
+import uuid
+
+
+gcp_utils_dir = os.path.abspath(os.path.join(
+    os.path.dirname(__file__), '../../gcp/utils'))
+sys.path.append(gcp_utils_dir)
+import big_query_utils
+
+
+_PROJECT_ID='grpc-testing'
+
+def _patch_results_table(dataset_id, table_id):
+  bq = big_query_utils.create_big_query()
+  with open(os.path.dirname(__file__) + '/scenario_result_schema.json', 'r') as f:
+    table_schema = json.loads(f.read())
+  desc = 'Results of performance benchmarks.'
+  return big_query_utils.patch_table(bq, _PROJECT_ID, dataset_id,
+                                     table_id, table_schema)
+
+
+argp = argparse.ArgumentParser(description='Patch schema of scenario results table.')
+argp.add_argument('--bq_result_table', required=True, default=None, type=str,
+                  help='Bigquery "dataset.table" to patch.')
+
+args = argp.parse_args()
+
+dataset_id, table_id = args.bq_result_table.split('.', 2)
+
+_patch_results_table(dataset_id, table_id)
+print('Successfully patched schema of %s.\n' % args.bq_result_table)
diff --git a/tools/run_tests/performance/scenario_result_schema.json b/tools/run_tests/performance/scenario_result_schema.json
index 8ec41c377c..245861f8c2 100644
--- a/tools/run_tests/performance/scenario_result_schema.json
+++ b/tools/run_tests/performance/scenario_result_schema.json
@@ -107,6 +107,11 @@
         "name": "timeSystem",
         "type": "FLOAT",
         "mode": "NULLABLE"
+      },
+      {
+        "name": "cqPollCount",
+        "type": "INTEGER",
+        "mode": "NULLABLE"
       }
     ]
   },
@@ -129,6 +134,11 @@
         "name": "timeSystem",
         "type": "FLOAT",
         "mode": "NULLABLE"
+      },
+      {
+        "name": "cqPollCount",
+        "type": "INTEGER",
+        "mode": "NULLABLE"
       }
     ]
   },
@@ -196,6 +206,16 @@
         "name": "latency999",
         "type": "FLOAT",
         "mode": "NULLABLE"
+      },
+      {
+        "name": "clientPollsPerRequest",
+        "type": "FLOAT",
+        "mode": "NULLABLE"
+      },
+      {
+        "name": "serverPollsPerRequest",
+        "type": "FLOAT",
+        "mode": "NULLABLE"
       }
     ]
   },
diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py
index 6541ebc52a..611868ce5a 100755
--- a/tools/run_tests/run_tests.py
+++ b/tools/run_tests/run_tests.py
@@ -81,7 +81,7 @@ def get_flaky_tests(limit=None):
     FROM
       [grpc-testing:jenkins_test_results.aggregate_results]
     WHERE
-      timestamp >= DATE_ADD(DATE(CURRENT_TIMESTAMP()), -1, "WEEK")
+      timestamp >= DATE_ADD(CURRENT_DATE(), -1, "WEEK")
       AND NOT REGEXP_MATCH(job_name, '.*portability.*')
     GROUP BY
       test_name