Merge branch 'master' of https://github.com/grpc/grpc into no-more-extern-c

9 files changed, 126 insertions, 11 deletions

diff --git a/tools/distrib/python/check_grpcio_tools.py b/tools/distrib/python/check_grpcio_tools.py
index c5afa713e6..b56ccaea7a 100755
--- a/tools/distrib/python/check_grpcio_tools.py
+++ b/tools/distrib/python/check_grpcio_tools.py
@@ -14,17 +14,21 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import cStringIO
-
-import make_grpcio_tools as make
+import make_grpcio_tools as _make
 
 OUT_OF_DATE_MESSAGE = """file {} is out of date
 
 Have you called tools/distrib/python/make_grpcio_tools.py since upgrading protobuf?"""
 
-check_protoc_lib_deps_content = make.get_deps()
+submodule_commit_hash = _make.protobuf_submodule_commit_hash()
+
+with open(_make.GRPC_PYTHON_PROTOC_LIB_DEPS, 'r') as _protoc_lib_deps_file:
+  content = _protoc_lib_deps_file.read().splitlines()
+
+testString = (_make.COMMIT_HASH_PREFIX +
+              submodule_commit_hash +
+              _make.COMMIT_HASH_SUFFIX)
 
-with open(make.GRPC_PYTHON_PROTOC_LIB_DEPS, 'r') as protoc_lib_deps_file:
-  if protoc_lib_deps_file.read() != check_protoc_lib_deps_content:
-    print(OUT_OF_DATE_MESSAGE.format(make.GRPC_PYTHON_PROTOC_LIB_DEPS))
-    raise SystemExit(1)
+if testString not in content:
+  print(OUT_OF_DATE_MESSAGE.format(_make.GRPC_PYTHON_PROTOC_LIB_DEPS))
+  raise SystemExit(1)
diff --git a/tools/distrib/python/grpcio_tools/protoc_lib_deps.py b/tools/distrib/python/grpcio_tools/protoc_lib_deps.py
index 18470d59d5..2a42a518fc 100644
--- a/tools/distrib/python/grpcio_tools/protoc_lib_deps.py
+++ b/tools/distrib/python/grpcio_tools/protoc_lib_deps.py
@@ -1,5 +1,5 @@
 
-# Copyright 2016 gRPC authors.
+# Copyright 2017 gRPC authors.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -19,3 +19,5 @@ PROTO_FILES=['google/protobuf/wrappers.proto', 'google/protobuf/type.proto', 'go
 
 CC_INCLUDE='third_party/protobuf/src'
 PROTO_INCLUDE='third_party/protobuf/src'
+
+PROTOBUF_SUBMODULE_VERSION="80a37e0782d2d702d52234b62dd4b9ec74fd2c95"
diff --git a/tools/distrib/python/make_grpcio_tools.py b/tools/distrib/python/make_grpcio_tools.py
index 5bc07ff360..c865f0bcc0 100755
--- a/tools/distrib/python/make_grpcio_tools.py
+++ b/tools/distrib/python/make_grpcio_tools.py
@@ -28,7 +28,7 @@ import traceback
 import uuid
 
 DEPS_FILE_CONTENT="""
-# Copyright 2016 gRPC authors.
+# Copyright 2017 gRPC authors.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -48,8 +48,13 @@ PROTO_FILES={proto_files}
 
 CC_INCLUDE={cc_include}
 PROTO_INCLUDE={proto_include}
+
+{commit_hash}
 """
 
+COMMIT_HASH_PREFIX = 'PROTOBUF_SUBMODULE_VERSION="'
+COMMIT_HASH_SUFFIX = '"'
+
 # Bazel query result prefix for expected source files in protobuf.
 PROTOBUF_CC_PREFIX = '//:src/'
 PROTOBUF_PROTO_PREFIX = '//:src/'
@@ -63,6 +68,7 @@ GRPC_PYTHON_ROOT = os.path.join(GRPC_ROOT, 'tools', 'distrib',
 
 GRPC_PYTHON_PROTOBUF_RELATIVE_ROOT = os.path.join('third_party', 'protobuf', 'src')
 GRPC_PROTOBUF = os.path.join(GRPC_ROOT, GRPC_PYTHON_PROTOBUF_RELATIVE_ROOT)
+GRPC_PROTOBUF_SUBMODULE_ROOT = os.path.join(GRPC_ROOT, 'third_party', 'protobuf')
 GRPC_PROTOC_PLUGINS = os.path.join(GRPC_ROOT, 'src', 'compiler')
 GRPC_PYTHON_PROTOBUF = os.path.join(GRPC_PYTHON_ROOT, 'third_party', 'protobuf',
                                     'src')
@@ -78,6 +84,14 @@ BAZEL_DEPS = os.path.join(GRPC_ROOT, 'tools', 'distrib', 'python', 'bazel_deps.s
 BAZEL_DEPS_PROTOC_LIB_QUERY = '//:protoc_lib'
 BAZEL_DEPS_COMMON_PROTOS_QUERY = '//:well_known_protos'
 
+def protobuf_submodule_commit_hash():
+  """Gets the commit hash for the HEAD of the protobuf submodule currently
+     checked out."""
+  cwd = os.getcwd()
+  os.chdir(GRPC_PROTOBUF_SUBMODULE_ROOT)
+  output = subprocess.check_output(['git', 'rev-parse', 'HEAD'])
+  os.chdir(cwd)
+  return output.splitlines()[0].strip()
 
 def bazel_query(query):
   output = subprocess.check_output([BAZEL_DEPS, query])
@@ -94,11 +108,13 @@ def get_deps():
   proto_files = [
       name[len(PROTOBUF_PROTO_PREFIX):] for name in proto_files_output
       if name.endswith('.proto') and name.startswith(PROTOBUF_PROTO_PREFIX)]
+  commit_hash = protobuf_submodule_commit_hash()
   deps_file_content = DEPS_FILE_CONTENT.format(
       cc_files=cc_files,
       proto_files=proto_files,
       cc_include=repr(GRPC_PYTHON_PROTOBUF_RELATIVE_ROOT),
-      proto_include=repr(GRPC_PYTHON_PROTOBUF_RELATIVE_ROOT))
+      proto_include=repr(GRPC_PYTHON_PROTOBUF_RELATIVE_ROOT),
+      commit_hash=COMMIT_HASH_PREFIX + commit_hash + COMMIT_HASH_SUFFIX)
   return deps_file_content
 
 def long_path(path):
diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal
index c9c64399f7..3c564e203a 100644
--- a/tools/doxygen/Doxyfile.c++.internal
+++ b/tools/doxygen/Doxyfile.c++.internal
@@ -1026,6 +1026,7 @@ src/core/lib/slice/percent_encoding.h \
 src/core/lib/slice/slice_hash_table.h \
 src/core/lib/slice/slice_internal.h \
 src/core/lib/slice/slice_string_helpers.h \
+src/core/lib/support/abstract.h \
 src/core/lib/support/arena.h \
 src/core/lib/support/atomic.h \
 src/core/lib/support/atomic_with_atm.h \
diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal
index b8434bb819..5674124f44 100644
--- a/tools/doxygen/Doxyfile.core.internal
+++ b/tools/doxygen/Doxyfile.core.internal
@@ -1267,6 +1267,7 @@ src/core/lib/slice/slice_intern.cc \
 src/core/lib/slice/slice_internal.h \
 src/core/lib/slice/slice_string_helpers.cc \
 src/core/lib/slice/slice_string_helpers.h \
+src/core/lib/support/abstract.h \
 src/core/lib/support/alloc.cc \
 src/core/lib/support/arena.cc \
 src/core/lib/support/arena.h \
diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json
index a372c8282b..5a3429c81f 100644
--- a/tools/run_tests/generated/sources_and_headers.json
+++ b/tools/run_tests/generated/sources_and_headers.json
@@ -756,6 +756,21 @@
     "headers": [], 
     "is_filegroup": false, 
     "language": "c", 
+    "name": "gpr_manual_constructor_test", 
+    "src": [
+      "test/core/support/manual_constructor_test.cc"
+    ], 
+    "third_party": false, 
+    "type": "target"
+  }, 
+  {
+    "deps": [
+      "gpr", 
+      "gpr_test_util"
+    ], 
+    "headers": [], 
+    "is_filegroup": false, 
+    "language": "c", 
     "name": "gpr_mpscq_test", 
     "src": [
       "test/core/support/mpscq_test.cc"
@@ -7814,6 +7829,7 @@
       "include/grpc/support/tls_pthread.h", 
       "include/grpc/support/useful.h", 
       "src/core/lib/profiling/timers.h", 
+      "src/core/lib/support/abstract.h", 
       "src/core/lib/support/arena.h", 
       "src/core/lib/support/atomic.h", 
       "src/core/lib/support/atomic_with_atm.h", 
@@ -7862,6 +7878,7 @@
       "include/grpc/support/tls_pthread.h", 
       "include/grpc/support/useful.h", 
       "src/core/lib/profiling/timers.h", 
+      "src/core/lib/support/abstract.h", 
       "src/core/lib/support/arena.h", 
       "src/core/lib/support/atomic.h", 
       "src/core/lib/support/atomic_with_atm.h", 
diff --git a/tools/run_tests/generated/tests.json b/tools/run_tests/generated/tests.json
index 4a5817cad1..dfd7bfafe6 100644
--- a/tools/run_tests/generated/tests.json
+++ b/tools/run_tests/generated/tests.json
@@ -920,6 +920,30 @@
       "posix", 
       "windows"
     ], 
+    "cpu_cost": 3, 
+    "exclude_configs": [], 
+    "exclude_iomgrs": [], 
+    "flaky": false, 
+    "gtest": false, 
+    "language": "c", 
+    "name": "gpr_manual_constructor_test", 
+    "platforms": [
+      "linux", 
+      "mac", 
+      "posix", 
+      "windows"
+    ], 
+    "uses_polling": false
+  }, 
+  {
+    "args": [], 
+    "benchmark": false, 
+    "ci_platforms": [
+      "linux", 
+      "mac", 
+      "posix", 
+      "windows"
+    ], 
     "cpu_cost": 30, 
     "exclude_configs": [], 
     "exclude_iomgrs": [], 
diff --git a/tools/run_tests/sanity/check_bazel_workspace.py b/tools/run_tests/sanity/check_bazel_workspace.py
new file mode 100755
index 0000000000..776c78b03f
--- /dev/null
+++ b/tools/run_tests/sanity/check_bazel_workspace.py
@@ -0,0 +1,49 @@
+#!/usr/bin/env python
+
+# Copyright 2016 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import print_function
+
+import ast
+import os
+import re
+import subprocess
+import sys
+
+os.chdir(os.path.join(os.path.dirname(sys.argv[0]), '../../..'))
+
+git_hash_pattern = re.compile('[0-9a-f]{40}')
+
+# Parse git hashes from submodules
+git_submodules = subprocess.check_output('git submodule', shell=True).strip().split('\n')
+git_submodule_hashes = {re.search(git_hash_pattern, s).group() for s in git_submodules}
+
+# Parse git hashes from Bazel WORKSPACE {new_}http_archive rules
+with open('WORKSPACE', 'r') as f:
+  workspace_rules = [expr.value for expr in ast.parse(f.read()).body]
+
+http_archive_rules = [rule for rule in workspace_rules if rule.func.id.endswith('http_archive')]
+archive_urls = [kw.value.s for rule in http_archive_rules for kw in rule.keywords if kw.arg == 'url']
+workspace_git_hashes = {re.search(git_hash_pattern, url).group() for url in archive_urls}
+
+# Validate the equivalence of the git submodules and Bazel git dependencies. The
+# condition we impose is that there is a git submodule for every dependency in
+# the workspace, but not necessarily conversely. E.g. Bloaty is a dependency
+# not used by any of the targets built by Bazel.
+if len(workspace_git_hashes - git_submodule_hashes) > 0:
+    print("Found discrepancies between git submodules and Bazel WORKSPACE dependencies")
+    sys.exit(1)
+
+sys.exit(0)
diff --git a/tools/run_tests/sanity/sanity_tests.yaml b/tools/run_tests/sanity/sanity_tests.yaml
index 3ca5c1b6c0..81eec4d149 100644
--- a/tools/run_tests/sanity/sanity_tests.yaml
+++ b/tools/run_tests/sanity/sanity_tests.yaml
@@ -1,4 +1,5 @@
 # a set of tests that are run in parallel for sanity tests
+- script: tools/run_tests/sanity/check_bazel_workspace.py
 - script: tools/run_tests/sanity/check_cache_mk.sh
 - script: tools/run_tests/sanity/check_owners.sh
 - script: tools/run_tests/sanity/check_sources_and_headers.py