Merge remote-tracking branch 'upstream/master' into stream_compression_config

8 files changed, 149 insertions, 137 deletions

diff --git a/tools/run_tests/dockerize/build_interop_stress_image.sh b/tools/run_tests/dockerize/build_interop_stress_image.sh
deleted file mode 100755
index acb566f5d5..0000000000
--- a/tools/run_tests/dockerize/build_interop_stress_image.sh
+++ /dev/null
@@ -1,93 +0,0 @@
-#!/bin/bash
-# Copyright 2015 gRPC authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# This script is invoked by run_interop_tests.py to build the docker image
-# for interop testing. You should never need to call this script on your own.
-
-set -x
-
-# Params:
-#  INTEROP_IMAGE - Name of tag of the final interop image
-#  INTEROP_IMAGE_REPOSITORY_TAG - Optional. If set, the created image will be tagged using
-#    the command: 'docker tag $INTEROP_IMAGE $INTEROP_IMAGE_REPOSITORY_TAG'
-#  BASE_NAME - Base name used to locate the base Dockerfile and build script
-#  BUILD_TYPE - The 'CONFIG' variable passed to the 'make' command (example:
-#  asan, tsan. Default value: opt).
-#  TTY_FLAG - optional -t flag to make docker allocate tty
-#  BUILD_INTEROP_DOCKER_EXTRA_ARGS - optional args to be passed to the
-#    docker run command
-
-cd `dirname $0`/../../..
-GRPC_ROOT=`pwd`
-MOUNT_ARGS="-v $GRPC_ROOT:/var/local/jenkins/grpc:ro"
-
-GRPC_JAVA_ROOT=`cd ../grpc-java && pwd`
-if [ "$GRPC_JAVA_ROOT" != "" ]
-then
-  MOUNT_ARGS+=" -v $GRPC_JAVA_ROOT:/var/local/jenkins/grpc-java:ro"
-else
-  echo "WARNING: grpc-java not found, it won't be mounted to the docker container."
-fi
-
-GRPC_GO_ROOT=`cd ../grpc-go && pwd`
-if [ "$GRPC_GO_ROOT" != "" ]
-then
-  MOUNT_ARGS+=" -v $GRPC_GO_ROOT:/var/local/jenkins/grpc-go:ro"
-else
-  echo "WARNING: grpc-go not found, it won't be mounted to the docker container."
-fi
-
-mkdir -p /tmp/ccache
-
-# Mount service account dir if available.
-# If service_directory does not contain the service account JSON file,
-# some of the tests will fail.
-if [ -e $HOME/service_account ]
-then
-  MOUNT_ARGS+=" -v $HOME/service_account:/var/local/jenkins/service_account:ro"
-fi
-
-# Use image name based on Dockerfile checksum
-BASE_IMAGE=${BASE_NAME}_base:`sha1sum tools/dockerfile/stress_test/$BASE_NAME/Dockerfile | cut -f1 -d\ `
-
-# Make sure base docker image has been built. Should be instantaneous if so.
-docker build -t $BASE_IMAGE --force-rm=true tools/dockerfile/stress_test/$BASE_NAME || exit $?
-
-# Create a local branch so the child Docker script won't complain
-git branch -f jenkins-docker
-
-CONTAINER_NAME="build_${BASE_NAME}_$(uuidgen)"
-
-# Prepare image for interop tests, commit it on success.
-(docker run \
-  -e CCACHE_DIR=/tmp/ccache \
-  -e THIS_IS_REALLY_NEEDED='see https://github.com/docker/docker/issues/14203 for why docker is awful' \
-  -e BUILD_TYPE=${BUILD_TYPE:=opt} \
-  -i $TTY_FLAG \
-  $MOUNT_ARGS \
-  $BUILD_INTEROP_DOCKER_EXTRA_ARGS \
-  -v /tmp/ccache:/tmp/ccache \
-  --name=$CONTAINER_NAME \
-  $BASE_IMAGE \
-  bash -l /var/local/jenkins/grpc/tools/dockerfile/stress_test/$BASE_NAME/build_interop_stress.sh \
-  && docker commit $CONTAINER_NAME $INTEROP_IMAGE \
-  && ( if [ -n "$INTEROP_IMAGE_REPOSITORY_TAG" ]; then docker tag $INTEROP_IMAGE $INTEROP_IMAGE_REPOSITORY_TAG ; fi ) \
-  && echo "Successfully built image $INTEROP_IMAGE")
-EXITCODE=$?
-
-# remove intermediate container, possibly killing it first
-docker rm -f $CONTAINER_NAME
-
-exit $EXITCODE
diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json
index 76cf23dcfb..727506a816 100644
--- a/tools/run_tests/generated/sources_and_headers.json
+++ b/tools/run_tests/generated/sources_and_headers.json
@@ -7839,6 +7839,9 @@
       "src/core/lib/iomgr/ev_windows.c", 
       "src/core/lib/iomgr/exec_ctx.c", 
       "src/core/lib/iomgr/executor.c", 
+      "src/core/lib/iomgr/gethostname_fallback.c", 
+      "src/core/lib/iomgr/gethostname_host_name_max.c", 
+      "src/core/lib/iomgr/gethostname_sysconf.c", 
       "src/core/lib/iomgr/iocp_windows.c", 
       "src/core/lib/iomgr/iomgr.c", 
       "src/core/lib/iomgr/iomgr_posix.c", 
@@ -7985,6 +7988,7 @@
       "src/core/lib/iomgr/ev_posix.h", 
       "src/core/lib/iomgr/exec_ctx.h", 
       "src/core/lib/iomgr/executor.h", 
+      "src/core/lib/iomgr/gethostname.h", 
       "src/core/lib/iomgr/iocp_windows.h", 
       "src/core/lib/iomgr/iomgr.h", 
       "src/core/lib/iomgr/iomgr_internal.h", 
@@ -8041,6 +8045,7 @@
       "src/core/lib/slice/slice_hash_table.h", 
       "src/core/lib/slice/slice_internal.h", 
       "src/core/lib/slice/slice_string_helpers.h", 
+      "src/core/lib/surface/alarm_internal.h", 
       "src/core/lib/surface/api_trace.h", 
       "src/core/lib/surface/call.h", 
       "src/core/lib/surface/call_test_only.h", 
@@ -8113,6 +8118,7 @@
       "src/core/lib/iomgr/ev_posix.h", 
       "src/core/lib/iomgr/exec_ctx.h", 
       "src/core/lib/iomgr/executor.h", 
+      "src/core/lib/iomgr/gethostname.h", 
       "src/core/lib/iomgr/iocp_windows.h", 
       "src/core/lib/iomgr/iomgr.h", 
       "src/core/lib/iomgr/iomgr_internal.h", 
@@ -8169,6 +8175,7 @@
       "src/core/lib/slice/slice_hash_table.h", 
       "src/core/lib/slice/slice_internal.h", 
       "src/core/lib/slice/slice_string_helpers.h", 
+      "src/core/lib/surface/alarm_internal.h", 
       "src/core/lib/surface/api_trace.h", 
       "src/core/lib/surface/call.h", 
       "src/core/lib/surface/call_test_only.h", 
@@ -9094,16 +9101,15 @@
     "deps": [
       "gpr", 
       "grpc_base", 
-      "grpc_trace"
+      "grpc_trace", 
+      "tsi_interface"
     ], 
     "headers": [
       "src/core/tsi/fake_transport_security.h", 
       "src/core/tsi/gts_transport_security.h", 
       "src/core/tsi/ssl_transport_security.h", 
       "src/core/tsi/ssl_types.h", 
-      "src/core/tsi/transport_security.h", 
-      "src/core/tsi/transport_security_adapter.h", 
-      "src/core/tsi/transport_security_interface.h"
+      "src/core/tsi/transport_security_grpc.h"
     ], 
     "is_filegroup": true, 
     "language": "c", 
@@ -9116,6 +9122,26 @@
       "src/core/tsi/ssl_transport_security.c", 
       "src/core/tsi/ssl_transport_security.h", 
       "src/core/tsi/ssl_types.h", 
+      "src/core/tsi/transport_security_grpc.c", 
+      "src/core/tsi/transport_security_grpc.h"
+    ], 
+    "third_party": false, 
+    "type": "filegroup"
+  }, 
+  {
+    "deps": [
+      "gpr", 
+      "grpc_trace"
+    ], 
+    "headers": [
+      "src/core/tsi/transport_security.h", 
+      "src/core/tsi/transport_security_adapter.h", 
+      "src/core/tsi/transport_security_interface.h"
+    ], 
+    "is_filegroup": true, 
+    "language": "c", 
+    "name": "tsi_interface", 
+    "src": [
       "src/core/tsi/transport_security.c", 
       "src/core/tsi/transport_security.h", 
       "src/core/tsi/transport_security_adapter.c", 
diff --git a/tools/run_tests/run_build_statistics.py b/tools/run_tests/run_build_statistics.py
index d63dc3e86f..1e957b6677 100755
--- a/tools/run_tests/run_build_statistics.py
+++ b/tools/run_tests/run_build_statistics.py
@@ -88,24 +88,26 @@ _KNOWN_ERRORS = [
     'FAILED: bins/tsan/qps_openloop_test GRPC_POLL_STRATEGY=poll',
     ('tests.bins/asan/h2_proxy_test streaming_error_response '
      'GRPC_POLL_STRATEGY=legacy'),
+    'hudson.plugins.git.GitException',
+    'Couldn\'t find any revision to build',
+    'org.jenkinsci.plugin.Diskcheck.preCheckout',
+    'Something went wrong while deleting Files',
 ]
-_NO_REPORT_FILES_FOUND_ERROR = 'No test report files were found. Configuration error?'
+_NO_REPORT_FILES_FOUND_ERROR = 'No test report files were found.'
 _UNKNOWN_ERROR = 'Unknown error'
 _DATASET_ID = 'build_statistics'
 
 
 def _scrape_for_known_errors(html):
   error_list = []
-  known_error_count = 0
   for known_error in _KNOWN_ERRORS:
     errors = re.findall(known_error, html)
     this_error_count = len(errors)
     if this_error_count > 0: 
-      known_error_count += this_error_count
       error_list.append({'description': known_error,
                          'count': this_error_count})
       print('====> %d failures due to %s' % (this_error_count, known_error))
-  return error_list, known_error_count
+  return error_list
 
 
 def _no_report_files_found(html):
@@ -152,30 +154,29 @@ def _process_build(json_url, console_url):
     failure_count = test_result['failCount']
     build_result['pass_count'] = test_result['passCount']
     build_result['failure_count'] = failure_count
+    # This means Jenkins failure occurred.
     build_result['no_report_files_found'] = _no_report_files_found(html)
-    if failure_count > 0:
-      error_list, known_error_count = _scrape_for_known_errors(html)
-      unknown_error_count = failure_count - known_error_count
-      # This can happen if the same error occurs multiple times in one test.
-      if failure_count < known_error_count:
-        print('====> Some errors are duplicates.')
-        unknown_error_count = 0
-      error_list.append({'description': _UNKNOWN_ERROR, 
-                         'count': unknown_error_count})
+    # Only check errors if Jenkins failure occurred.
+    if build_result['no_report_files_found']:
+      error_list = _scrape_for_known_errors(html)
   except Exception as e:
     print('====> Got exception for %s: %s.' % (json_url, str(e)))   
     print('====> Parsing errors from %s.' % console_url)
     html = urllib.urlopen(console_url).read()
     build_result['pass_count'] = 0  
     build_result['failure_count'] = 1
-    error_list, _ = _scrape_for_known_errors(html)
-    if error_list:
-      error_list.append({'description': _UNKNOWN_ERROR, 'count': 0})
-    else:
-      error_list.append({'description': _UNKNOWN_ERROR, 'count': 1})
- 
+    # In this case, the string doesn't exist in the result html but the fact 
+    # that we fail to parse the result html indicates Jenkins failure and hence 
+    # no report files were generated.
+    build_result['no_report_files_found'] = True
+    error_list = _scrape_for_known_errors(html)
+
   if error_list:
     build_result['error'] = error_list
+  elif build_result['no_report_files_found']:
+    build_result['error'] = [{'description': _UNKNOWN_ERROR, 'count': 1}]
+  else:
+    build_result['error'] = [{'description': '', 'count': 0}]
 
   return build_result 
 
@@ -214,6 +215,15 @@ for build_name in _BUILDS.keys() if 'all' in args.builds else args.builds:
     build = None
     try:
       build = job.get_build_metadata(build_number)
+      print('====> Build status: %s.' % build.get_status())
+      if build.get_status() == 'ABORTED':
+        continue
+      # If any build is still running, stop processing this job. Next time, we
+      # start from where it was left so that all builds are processed 
+      # sequentially.
+      if build.is_running():
+        print('====> Build %d is still running.' % build_number)
+        break
     except KeyError:
       print('====> Build %s is missing. Skip.' % build_number)
       continue
@@ -226,10 +236,10 @@ for build_name in _BUILDS.keys() if 'all' in args.builds else args.builds:
       json_url = '%s/testReport/api/json' % url_base
       console_url = '%s/consoleFull' % url_base
       build_result['duration'] = build.get_duration().total_seconds()
-      build_result.update(_process_build(json_url, console_url))
+      build_stat = _process_build(json_url, console_url)
+      build_result.update(build_stat)
     rows = [big_query_utils.make_row(build_number, build_result)]
     if not big_query_utils.insert_rows(bq, _PROJECT_ID, _DATASET_ID, build_name, 
                                        rows):
       print('====> Error uploading result to bigquery.')
       sys.exit(1)
-
diff --git a/tools/run_tests/run_interop_tests.py b/tools/run_tests/run_interop_tests.py
index dbbf2adb22..1537641aee 100755
--- a/tools/run_tests/run_interop_tests.py
+++ b/tools/run_tests/run_interop_tests.py
@@ -731,7 +731,7 @@ def cloud_to_cloud_jobspec(language, test_case, server_name, server_host,
     if manual_cmd_log is not None:
       if manual_cmd_log == []:
         manual_cmd_log.append('echo "Testing ${docker_image:=%s}"' % docker_image)
-      manual_cmd_log.append(manual_cmdline(cmdline, docker_iamge))
+      manual_cmd_log.append(manual_cmdline(cmdline, docker_image))
     cwd = None
 
   test_job = jobset.JobSpec(
@@ -793,7 +793,7 @@ def server_jobspec(language, docker_image, insecure=False, manual_cmd_log=None):
   if manual_cmd_log is not None:
       if manual_cmd_log == []:
         manual_cmd_log.append('echo "Testing ${docker_image:=%s}"' % docker_image)
-      manual_cmd_log.append(manual_cmdline(docker_cmdline, docker_iamge))
+      manual_cmd_log.append(manual_cmdline(docker_cmdline, docker_image))
   server_job = jobset.JobSpec(
           cmdline=docker_cmdline,
           environ=environ,
diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py
index 106c7e0699..a695a62827 100755
--- a/tools/run_tests/run_tests.py
+++ b/tools/run_tests/run_tests.py
@@ -74,19 +74,21 @@ def get_flaky_tests(limit=None):
 
   bq = big_query_utils.create_big_query()
   query = """
-    SELECT
-      test_name,
-      SUM(result != 'PASSED'
-        AND result != 'SKIPPED') AS count_failed,
-    FROM
-      [grpc-testing:jenkins_test_results.aggregate_results]
-    WHERE
-      timestamp >= DATE_ADD(CURRENT_DATE(), -1, "WEEK")
-      AND NOT REGEXP_MATCH(job_name, '.*portability.*')
-    GROUP BY
-      test_name
-    HAVING
-      count_failed > 0"""
+SELECT
+  filtered_test_name,
+  FROM (
+  SELECT
+    REGEXP_REPLACE(test_name, r'/\d+', '') AS filtered_test_name,
+    result
+  FROM
+    [grpc-testing:jenkins_test_results.aggregate_results]
+  WHERE
+    timestamp >= DATE_ADD(CURRENT_DATE(), -1, "WEEK")
+    AND NOT REGEXP_MATCH(job_name, '.*portability.*') )
+GROUP BY
+  filtered_test_name
+HAVING
+  SUM(result != 'PASSED' AND result != 'SKIPPED') > 0"""
   if limit:
     query += " limit {}".format(limit)
   query_job = big_query_utils.sync_query_job(bq, 'grpc-testing', query)
@@ -886,11 +888,50 @@ class ObjCLanguage(object):
         self.config.job_spec(['src/objective-c/tests/run_tests.sh'],
                               timeout_seconds=60*60,
                               shortname='objc-tests',
+                              cpu_cost=1e6,
                               environ=_FORCE_ENVIRON_FOR_WRAPPERS),
-        self.config.job_spec(['src/objective-c/tests/build_example_test.sh'],
-                              timeout_seconds=30*60,
-                              shortname='objc-examples-build',
+        self.config.job_spec(['src/objective-c/tests/run_plugin_tests.sh'],
+                              timeout_seconds=60*60,
+                              shortname='objc-plugin-tests',
+                              cpu_cost=1e6,
                               environ=_FORCE_ENVIRON_FOR_WRAPPERS),
+        self.config.job_spec(['src/objective-c/tests/build_one_example.sh'],
+                              timeout_seconds=10*60,
+                              shortname='objc-build-example-helloworld',
+                              cpu_cost=1e6,
+                              environ={'SCHEME': 'HelloWorld',
+                                       'EXAMPLE_PATH': 'examples/objective-c/helloworld'}),
+        self.config.job_spec(['src/objective-c/tests/build_one_example.sh'],
+                              timeout_seconds=10*60,
+                              shortname='objc-build-example-routeguide',
+                              cpu_cost=1e6,
+                              environ={'SCHEME': 'RouteGuideClient',
+                                       'EXAMPLE_PATH': 'examples/objective-c/route_guide'}),
+        self.config.job_spec(['src/objective-c/tests/build_one_example.sh'],
+                              timeout_seconds=10*60,
+                              shortname='objc-build-example-authsample',
+                              cpu_cost=1e6,
+                              environ={'SCHEME': 'AuthSample',
+                                       'EXAMPLE_PATH': 'examples/objective-c/auth_sample'}),
+        self.config.job_spec(['src/objective-c/tests/build_one_example.sh'],
+                              timeout_seconds=10*60,
+                              shortname='objc-build-example-sample',
+                              cpu_cost=1e6,
+                              environ={'SCHEME': 'Sample',
+                                       'EXAMPLE_PATH': 'src/objective-c/examples/Sample'}),
+        self.config.job_spec(['src/objective-c/tests/build_one_example.sh'],
+                              timeout_seconds=10*60,
+                              shortname='objc-build-example-sample-frameworks',
+                              cpu_cost=1e6,
+                              environ={'SCHEME': 'Sample',
+                                       'EXAMPLE_PATH': 'src/objective-c/examples/Sample',
+                                       'FRAMEWORKS': 'YES'}),
+        self.config.job_spec(['src/objective-c/tests/build_one_example.sh'],
+                              timeout_seconds=10*60,
+                              shortname='objc-build-example-switftsample',
+                              cpu_cost=1e6,
+                              environ={'SCHEME': 'SwiftSample',
+                                       'EXAMPLE_PATH': 'src/objective-c/examples/SwiftSample'}),
     ]
 
   def pre_build_steps(self):
diff --git a/tools/run_tests/sanity/check_submodules.sh b/tools/run_tests/sanity/check_submodules.sh
index d856e22176..569aaded03 100755
--- a/tools/run_tests/sanity/check_submodules.sh
+++ b/tools/run_tests/sanity/check_submodules.sh
@@ -31,7 +31,7 @@ cat << EOF | awk '{ print $1 }' | sort > $want_submodules
  886e7d75368e3f4fab3f4d0d3584e4abfc557755 third_party/boringssl-with-bazel (version_for_cocoapods_7.0-857-g886e7d7)
  30dbc81fb5ffdc98ea9b14b1918bfe4e8779b26e third_party/gflags (v2.2.0)
  ec44c6c1675c25b9827aacd08c02433cccde7780 third_party/googletest (release-1.8.0)
- a6189acd18b00611c1dc7042299ad75486f08a1a third_party/protobuf (v3.3.0)
+ 97d50e306e576dadf2184e643b7c934da45730c8 third_party/protobuf (v3.4.0rc2)
  cacf7f1d4e3d44d871b605da3b647f07d718623f third_party/zlib (v1.2.11)
  7691f773af79bf75a62d1863fd0f13ebf9dc51b1 third_party/cares/cares (1.12.0)
 EOF
diff --git a/tools/run_tests/sanity/core_untyped_structs.sh b/tools/run_tests/sanity/core_untyped_structs.sh
new file mode 100755
index 0000000000..792dd68fdd
--- /dev/null
+++ b/tools/run_tests/sanity/core_untyped_structs.sh
@@ -0,0 +1,27 @@
+#!/bin/sh
+# Copyright 2017 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+cd `dirname $0`/../../..
+
+#
+# Make sure that all core struct/unions have a name or are typedef'ed
+#
+
+egrep -Irn '(struct|union) *{' include/grpc |
+    egrep -v typedef |
+    diff - /dev/null
+
diff --git a/tools/run_tests/sanity/sanity_tests.yaml b/tools/run_tests/sanity/sanity_tests.yaml
index a86ebee7b4..7e582bc40b 100644
--- a/tools/run_tests/sanity/sanity_tests.yaml
+++ b/tools/run_tests/sanity/sanity_tests.yaml
@@ -6,6 +6,7 @@
 - script: tools/run_tests/sanity/check_test_filtering.py
 - script: tools/run_tests/sanity/check_tracer_sanity.py
 - script: tools/run_tests/sanity/core_banned_functions.py
+- script: tools/run_tests/sanity/core_untyped_structs.sh
 - script: tools/buildgen/generate_projects.sh -j 3
   cpu_cost: 3
 - script: tools/distrib/check_copyright.py