Do not require bazel for check_grpcio_tools.py

The `make_grpcio_tools.py` rube-goldberg machinery relies on
`bazel` to extract the list of files required to compile the
`grpcio-tools` package that are provided by the `protobuf`
submodule.  In order to ensure that list is up to date,
`check_grpcio_tools.py` sanity check does the same `bazel`
query, and checks the full contents against the already
existing list in the repository.  This has the downside of
requiring `bazel` to run that particular check at sanity test
time, and flakiness has been seen there.

This commit changes the code generation process to include
the git hash of the submodule at the time `make_grpcio_tools.py`
is invoked and stores it in the generated file and instead of
doing the whole process at test time, the `check_grpcio_tools.py`
sanity test simply checks the submodule version at test time
and verifies it against the version included in the file by
`make_grpcio_tools.py` thus removing the `bazel` dependency
at test time and increasing test robustness and speed.

3 files changed, 33 insertions, 11 deletions

diff --git a/tools/distrib/python/check_grpcio_tools.py b/tools/distrib/python/check_grpcio_tools.py
index c5afa713e6..b56ccaea7a 100755
--- a/tools/distrib/python/check_grpcio_tools.py
+++ b/tools/distrib/python/check_grpcio_tools.py
@@ -14,17 +14,21 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import cStringIO
-
-import make_grpcio_tools as make
+import make_grpcio_tools as _make
 
 OUT_OF_DATE_MESSAGE = """file {} is out of date
 
 Have you called tools/distrib/python/make_grpcio_tools.py since upgrading protobuf?"""
 
-check_protoc_lib_deps_content = make.get_deps()
+submodule_commit_hash = _make.protobuf_submodule_commit_hash()
+
+with open(_make.GRPC_PYTHON_PROTOC_LIB_DEPS, 'r') as _protoc_lib_deps_file:
+  content = _protoc_lib_deps_file.read().splitlines()
+
+testString = (_make.COMMIT_HASH_PREFIX +
+              submodule_commit_hash +
+              _make.COMMIT_HASH_SUFFIX)
 
-with open(make.GRPC_PYTHON_PROTOC_LIB_DEPS, 'r') as protoc_lib_deps_file:
-  if protoc_lib_deps_file.read() != check_protoc_lib_deps_content:
-    print(OUT_OF_DATE_MESSAGE.format(make.GRPC_PYTHON_PROTOC_LIB_DEPS))
-    raise SystemExit(1)
+if testString not in content:
+  print(OUT_OF_DATE_MESSAGE.format(_make.GRPC_PYTHON_PROTOC_LIB_DEPS))
+  raise SystemExit(1)
diff --git a/tools/distrib/python/grpcio_tools/protoc_lib_deps.py b/tools/distrib/python/grpcio_tools/protoc_lib_deps.py
index 18470d59d5..2a42a518fc 100644
--- a/tools/distrib/python/grpcio_tools/protoc_lib_deps.py
+++ b/tools/distrib/python/grpcio_tools/protoc_lib_deps.py
@@ -1,5 +1,5 @@
 
-# Copyright 2016 gRPC authors.
+# Copyright 2017 gRPC authors.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -19,3 +19,5 @@ PROTO_FILES=['google/protobuf/wrappers.proto', 'google/protobuf/type.proto', 'go
 
 CC_INCLUDE='third_party/protobuf/src'
 PROTO_INCLUDE='third_party/protobuf/src'
+
+PROTOBUF_SUBMODULE_VERSION="80a37e0782d2d702d52234b62dd4b9ec74fd2c95"
diff --git a/tools/distrib/python/make_grpcio_tools.py b/tools/distrib/python/make_grpcio_tools.py
index 5bc07ff360..c865f0bcc0 100755
--- a/tools/distrib/python/make_grpcio_tools.py
+++ b/tools/distrib/python/make_grpcio_tools.py
@@ -28,7 +28,7 @@ import traceback
 import uuid
 
 DEPS_FILE_CONTENT="""
-# Copyright 2016 gRPC authors.
+# Copyright 2017 gRPC authors.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -48,8 +48,13 @@ PROTO_FILES={proto_files}
 
 CC_INCLUDE={cc_include}
 PROTO_INCLUDE={proto_include}
+
+{commit_hash}
 """
 
+COMMIT_HASH_PREFIX = 'PROTOBUF_SUBMODULE_VERSION="'
+COMMIT_HASH_SUFFIX = '"'
+
 # Bazel query result prefix for expected source files in protobuf.
 PROTOBUF_CC_PREFIX = '//:src/'
 PROTOBUF_PROTO_PREFIX = '//:src/'
@@ -63,6 +68,7 @@ GRPC_PYTHON_ROOT = os.path.join(GRPC_ROOT, 'tools', 'distrib',
 
 GRPC_PYTHON_PROTOBUF_RELATIVE_ROOT = os.path.join('third_party', 'protobuf', 'src')
 GRPC_PROTOBUF = os.path.join(GRPC_ROOT, GRPC_PYTHON_PROTOBUF_RELATIVE_ROOT)
+GRPC_PROTOBUF_SUBMODULE_ROOT = os.path.join(GRPC_ROOT, 'third_party', 'protobuf')
 GRPC_PROTOC_PLUGINS = os.path.join(GRPC_ROOT, 'src', 'compiler')
 GRPC_PYTHON_PROTOBUF = os.path.join(GRPC_PYTHON_ROOT, 'third_party', 'protobuf',
                                     'src')
@@ -78,6 +84,14 @@ BAZEL_DEPS = os.path.join(GRPC_ROOT, 'tools', 'distrib', 'python', 'bazel_deps.s
 BAZEL_DEPS_PROTOC_LIB_QUERY = '//:protoc_lib'
 BAZEL_DEPS_COMMON_PROTOS_QUERY = '//:well_known_protos'
 
+def protobuf_submodule_commit_hash():
+  """Gets the commit hash for the HEAD of the protobuf submodule currently
+     checked out."""
+  cwd = os.getcwd()
+  os.chdir(GRPC_PROTOBUF_SUBMODULE_ROOT)
+  output = subprocess.check_output(['git', 'rev-parse', 'HEAD'])
+  os.chdir(cwd)
+  return output.splitlines()[0].strip()
 
 def bazel_query(query):
   output = subprocess.check_output([BAZEL_DEPS, query])
@@ -94,11 +108,13 @@ def get_deps():
   proto_files = [
       name[len(PROTOBUF_PROTO_PREFIX):] for name in proto_files_output
       if name.endswith('.proto') and name.startswith(PROTOBUF_PROTO_PREFIX)]
+  commit_hash = protobuf_submodule_commit_hash()
   deps_file_content = DEPS_FILE_CONTENT.format(
       cc_files=cc_files,
       proto_files=proto_files,
       cc_include=repr(GRPC_PYTHON_PROTOBUF_RELATIVE_ROOT),
-      proto_include=repr(GRPC_PYTHON_PROTOBUF_RELATIVE_ROOT))
+      proto_include=repr(GRPC_PYTHON_PROTOBUF_RELATIVE_ROOT),
+      commit_hash=COMMIT_HASH_PREFIX + commit_hash + COMMIT_HASH_SUFFIX)
   return deps_file_content
 
 def long_path(path):