Merge github.com:grpc/grpc into error

6 files changed, 76 insertions, 16 deletions

diff --git a/tools/distrib/python/check_grpcio_tools.py b/tools/distrib/python/check_grpcio_tools.py
index baf2ff4eff..80c63278f4 100755
--- a/tools/distrib/python/check_grpcio_tools.py
+++ b/tools/distrib/python/check_grpcio_tools.py
@@ -37,7 +37,7 @@ OUT_OF_DATE_MESSAGE = """file {} is out of date
 
 Have you called tools/distrib/python/make_grpcio_tools.py since upgrading protobuf?"""
 
-check_protoc_lib_deps_content = make.get_deps(make.BAZEL_DEPS_PROTOC_LIB_QUERY)
+check_protoc_lib_deps_content = make.get_deps()
 
 with open(make.GRPC_PYTHON_PROTOC_LIB_DEPS, 'r') as protoc_lib_deps_file:
   if protoc_lib_deps_file.read() != check_protoc_lib_deps_content:
diff --git a/tools/distrib/python/grpcio_tools/README.rst b/tools/distrib/python/grpcio_tools/README.rst
index 10d2fe8c30..e9f137493b 100644
--- a/tools/distrib/python/grpcio_tools/README.rst
+++ b/tools/distrib/python/grpcio_tools/README.rst
@@ -126,3 +126,13 @@ Help, I ...
   GCC 6.0), this is probably a bug where GCC chokes on constant expressions
   when the :code:`-fwrapv` flag is specified. You should consider setting your
   environment with :code:`CFLAGS=-fno-wrapv` or using clang (:code:`CC=clang`).
+
+Usage
+-----
+
+Given protobuf include directories :code:`$INCLUDE`, an output directory
+:code:`$OUTPUT`, and proto files :code:`$PROTO_FILES`, invoke as:
+
+::
+
+  $ python -m grpc.tools.protoc -I$INCLUDE --python_out=$OUTPUT --grpc_python_out=$OUTPUT $PROTO_FILES
diff --git a/tools/distrib/python/grpcio_tools/grpc/tools/protoc.py b/tools/distrib/python/grpcio_tools/grpc/tools/protoc.py
index b4dd0ecae2..1c69e78920 100644
--- a/tools/distrib/python/grpcio_tools/grpc/tools/protoc.py
+++ b/tools/distrib/python/grpcio_tools/grpc/tools/protoc.py
@@ -29,10 +29,13 @@
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
+import pkg_resources
 import sys
 
 from grpc.tools import protoc_compiler
 
 
 if __name__ == '__main__':
-  protoc_compiler.run_main(sys.argv)
+  proto_include = pkg_resources.resource_filename('grpc.tools', '_proto')
+  protoc_compiler.run_main(
+      sys.argv + ['-I{}'.format(proto_include)])
diff --git a/tools/distrib/python/grpcio_tools/protoc_lib_deps.py b/tools/distrib/python/grpcio_tools/protoc_lib_deps.py
index 135ac5cbb3..cd4effa7ae 100644
--- a/tools/distrib/python/grpcio_tools/protoc_lib_deps.py
+++ b/tools/distrib/python/grpcio_tools/protoc_lib_deps.py
@@ -30,3 +30,7 @@
 
 # AUTO-GENERATED BY make_grpcio_tools.py!
 CC_FILES=['google/protobuf/compiler/zip_writer.cc', 'google/protobuf/compiler/subprocess.cc', 'google/protobuf/compiler/ruby/ruby_generator.cc', 'google/protobuf/compiler/python/python_generator.cc', 'google/protobuf/compiler/plugin.pb.cc', 'google/protobuf/compiler/plugin.cc', 'google/protobuf/compiler/objectivec/objectivec_primitive_field.cc', 'google/protobuf/compiler/objectivec/objectivec_oneof.cc', 'google/protobuf/compiler/objectivec/objectivec_message_field.cc', 'google/protobuf/compiler/objectivec/objectivec_message.cc', 'google/protobuf/compiler/objectivec/objectivec_map_field.cc', 'google/protobuf/compiler/objectivec/objectivec_helpers.cc', 'google/protobuf/compiler/objectivec/objectivec_generator.cc', 'google/protobuf/compiler/objectivec/objectivec_file.cc', 'google/protobuf/compiler/objectivec/objectivec_field.cc', 'google/protobuf/compiler/objectivec/objectivec_extension.cc', 'google/protobuf/compiler/objectivec/objectivec_enum_field.cc', 'google/protobuf/compiler/objectivec/objectivec_enum.cc', 'google/protobuf/compiler/js/js_generator.cc', 'google/protobuf/compiler/javanano/javanano_primitive_field.cc', 'google/protobuf/compiler/javanano/javanano_message_field.cc', 'google/protobuf/compiler/javanano/javanano_message.cc', 'google/protobuf/compiler/javanano/javanano_map_field.cc', 'google/protobuf/compiler/javanano/javanano_helpers.cc', 'google/protobuf/compiler/javanano/javanano_generator.cc', 'google/protobuf/compiler/javanano/javanano_file.cc', 'google/protobuf/compiler/javanano/javanano_field.cc', 'google/protobuf/compiler/javanano/javanano_extension.cc', 'google/protobuf/compiler/javanano/javanano_enum_field.cc', 'google/protobuf/compiler/javanano/javanano_enum.cc', 'google/protobuf/compiler/java/java_string_field_lite.cc', 'google/protobuf/compiler/java/java_string_field.cc', 'google/protobuf/compiler/java/java_shared_code_generator.cc', 'google/protobuf/compiler/java/java_service.cc', 'google/protobuf/compiler/java/java_primitive_field_lite.cc', 'google/protobuf/compiler/java/java_primitive_field.cc', 'google/protobuf/compiler/java/java_name_resolver.cc', 'google/protobuf/compiler/java/java_message_lite.cc', 'google/protobuf/compiler/java/java_message_field_lite.cc', 'google/protobuf/compiler/java/java_message_field.cc', 'google/protobuf/compiler/java/java_message_builder_lite.cc', 'google/protobuf/compiler/java/java_message_builder.cc', 'google/protobuf/compiler/java/java_message.cc', 'google/protobuf/compiler/java/java_map_field_lite.cc', 'google/protobuf/compiler/java/java_map_field.cc', 'google/protobuf/compiler/java/java_lazy_message_field_lite.cc', 'google/protobuf/compiler/java/java_lazy_message_field.cc', 'google/protobuf/compiler/java/java_helpers.cc', 'google/protobuf/compiler/java/java_generator_factory.cc', 'google/protobuf/compiler/java/java_generator.cc', 'google/protobuf/compiler/java/java_file.cc', 'google/protobuf/compiler/java/java_field.cc', 'google/protobuf/compiler/java/java_extension_lite.cc', 'google/protobuf/compiler/java/java_extension.cc', 'google/protobuf/compiler/java/java_enum_lite.cc', 'google/protobuf/compiler/java/java_enum_field_lite.cc', 'google/protobuf/compiler/java/java_enum_field.cc', 'google/protobuf/compiler/java/java_enum.cc', 'google/protobuf/compiler/java/java_doc_comment.cc', 'google/protobuf/compiler/java/java_context.cc', 'google/protobuf/compiler/csharp/csharp_wrapper_field.cc', 'google/protobuf/compiler/csharp/csharp_source_generator_base.cc', 'google/protobuf/compiler/csharp/csharp_repeated_primitive_field.cc', 'google/protobuf/compiler/csharp/csharp_repeated_message_field.cc', 'google/protobuf/compiler/csharp/csharp_repeated_enum_field.cc', 'google/protobuf/compiler/csharp/csharp_reflection_class.cc', 'google/protobuf/compiler/csharp/csharp_primitive_field.cc', 'google/protobuf/compiler/csharp/csharp_message_field.cc', 'google/protobuf/compiler/csharp/csharp_message.cc', 'google/protobuf/compiler/csharp/csharp_map_field.cc', 'google/protobuf/compiler/csharp/csharp_helpers.cc', 'google/protobuf/compiler/csharp/csharp_generator.cc', 'google/protobuf/compiler/csharp/csharp_field_base.cc', 'google/protobuf/compiler/csharp/csharp_enum_field.cc', 'google/protobuf/compiler/csharp/csharp_enum.cc', 'google/protobuf/compiler/csharp/csharp_doc_comment.cc', 'google/protobuf/compiler/cpp/cpp_string_field.cc', 'google/protobuf/compiler/cpp/cpp_service.cc', 'google/protobuf/compiler/cpp/cpp_primitive_field.cc', 'google/protobuf/compiler/cpp/cpp_message_field.cc', 'google/protobuf/compiler/cpp/cpp_message.cc', 'google/protobuf/compiler/cpp/cpp_map_field.cc', 'google/protobuf/compiler/cpp/cpp_helpers.cc', 'google/protobuf/compiler/cpp/cpp_generator.cc', 'google/protobuf/compiler/cpp/cpp_file.cc', 'google/protobuf/compiler/cpp/cpp_field.cc', 'google/protobuf/compiler/cpp/cpp_extension.cc', 'google/protobuf/compiler/cpp/cpp_enum_field.cc', 'google/protobuf/compiler/cpp/cpp_enum.cc', 'google/protobuf/compiler/command_line_interface.cc', 'google/protobuf/compiler/code_generator.cc', 'google/protobuf/wrappers.pb.cc', 'google/protobuf/wire_format.cc', 'google/protobuf/util/type_resolver_util.cc', 'google/protobuf/util/time_util.cc', 'google/protobuf/util/message_differencer.cc', 'google/protobuf/util/json_util.cc', 'google/protobuf/util/internal/utility.cc', 'google/protobuf/util/internal/type_info_test_helper.cc', 'google/protobuf/util/internal/type_info.cc', 'google/protobuf/util/internal/protostream_objectwriter.cc', 'google/protobuf/util/internal/protostream_objectsource.cc', 'google/protobuf/util/internal/proto_writer.cc', 'google/protobuf/util/internal/object_writer.cc', 'google/protobuf/util/internal/json_stream_parser.cc', 'google/protobuf/util/internal/json_objectwriter.cc', 'google/protobuf/util/internal/json_escaping.cc', 'google/protobuf/util/internal/field_mask_utility.cc', 'google/protobuf/util/internal/error_listener.cc', 'google/protobuf/util/internal/default_value_objectwriter.cc', 'google/protobuf/util/internal/datapiece.cc', 'google/protobuf/util/field_mask_util.cc', 'google/protobuf/util/field_comparator.cc', 'google/protobuf/unknown_field_set.cc', 'google/protobuf/type.pb.cc', 'google/protobuf/timestamp.pb.cc', 'google/protobuf/text_format.cc', 'google/protobuf/stubs/substitute.cc', 'google/protobuf/stubs/mathlimits.cc', 'google/protobuf/struct.pb.cc', 'google/protobuf/source_context.pb.cc', 'google/protobuf/service.cc', 'google/protobuf/reflection_ops.cc', 'google/protobuf/message.cc', 'google/protobuf/map_field.cc', 'google/protobuf/io/zero_copy_stream_impl.cc', 'google/protobuf/io/tokenizer.cc', 'google/protobuf/io/strtod.cc', 'google/protobuf/io/printer.cc', 'google/protobuf/io/gzip_stream.cc', 'google/protobuf/generated_message_reflection.cc', 'google/protobuf/field_mask.pb.cc', 'google/protobuf/extension_set_heavy.cc', 'google/protobuf/empty.pb.cc', 'google/protobuf/dynamic_message.cc', 'google/protobuf/duration.pb.cc', 'google/protobuf/descriptor_database.cc', 'google/protobuf/descriptor.pb.cc', 'google/protobuf/descriptor.cc', 'google/protobuf/compiler/parser.cc', 'google/protobuf/compiler/importer.cc', 'google/protobuf/api.pb.cc', 'google/protobuf/any.pb.cc', 'google/protobuf/any.cc', 'google/protobuf/wire_format_lite.cc', 'google/protobuf/stubs/time.cc', 'google/protobuf/stubs/strutil.cc', 'google/protobuf/stubs/structurally_valid.cc', 'google/protobuf/stubs/stringprintf.cc', 'google/protobuf/stubs/stringpiece.cc', 'google/protobuf/stubs/statusor.cc', 'google/protobuf/stubs/status.cc', 'google/protobuf/stubs/once.cc', 'google/protobuf/stubs/int128.cc', 'google/protobuf/stubs/common.cc', 'google/protobuf/stubs/bytestream.cc', 'google/protobuf/stubs/atomicops_internals_x86_msvc.cc', 'google/protobuf/stubs/atomicops_internals_x86_gcc.cc', 'google/protobuf/repeated_field.cc', 'google/protobuf/message_lite.cc', 'google/protobuf/io/zero_copy_stream_impl_lite.cc', 'google/protobuf/io/zero_copy_stream.cc', 'google/protobuf/io/coded_stream.cc', 'google/protobuf/generated_message_util.cc', 'google/protobuf/extension_set.cc', 'google/protobuf/arenastring.cc', 'google/protobuf/arena.cc']
+PROTO_FILES=['google/protobuf/wrappers.proto', 'google/protobuf/type.proto', 'google/protobuf/timestamp.proto', 'google/protobuf/struct.proto', 'google/protobuf/source_context.proto', 'google/protobuf/field_mask.proto', 'google/protobuf/empty.proto', 'google/protobuf/duration.proto', 'google/protobuf/descriptor.proto', 'google/protobuf/compiler/plugin.proto', 'google/protobuf/api.proto', 'google/protobuf/any.proto']
+
+CC_INCLUDE='third_party/protobuf/src'
+PROTO_INCLUDE='third_party/protobuf/src'
diff --git a/tools/distrib/python/grpcio_tools/setup.py b/tools/distrib/python/grpcio_tools/setup.py
index 576f7ae32a..fbe69f43d5 100644
--- a/tools/distrib/python/grpcio_tools/setup.py
+++ b/tools/distrib/python/grpcio_tools/setup.py
@@ -28,9 +28,11 @@
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 from distutils import extension
+import errno
 import os
 import os.path
 import shlex
+import shutil
 import sys
 
 import setuptools
@@ -47,18 +49,41 @@ sys.path.insert(0, os.path.abspath('.'))
 # ourselves in w.r.t. the multitude of operating systems this ought to build on.
 # By default we assume a GCC-like compiler.
 EXTRA_COMPILE_ARGS = shlex.split(os.environ.get('GRPC_PYTHON_CFLAGS',
-                                                '-frtti -std=c++11'))
+                                                '-fno-wrapv -frtti -std=c++11'))
 EXTRA_LINK_ARGS = shlex.split(os.environ.get('GRPC_PYTHON_LDFLAGS',
                                              '-lpthread'))
 
+GRPC_PYTHON_TOOLS_PACKAGE = 'grpc.tools'
+GRPC_PYTHON_PROTO_RESOURCES_NAME = '_proto'
+
 import protoc_lib_deps
 import grpc_version
 
+def package_data():
+  tools_path = GRPC_PYTHON_TOOLS_PACKAGE.replace('.', os.path.sep)
+  proto_resources_path = os.path.join(tools_path,
+                                      GRPC_PYTHON_PROTO_RESOURCES_NAME)
+  proto_files = []
+  for proto_file in protoc_lib_deps.PROTO_FILES:
+    source = os.path.join(protoc_lib_deps.PROTO_INCLUDE, proto_file)
+    target = os.path.join(proto_resources_path, proto_file)
+    relative_target = os.path.join(GRPC_PYTHON_PROTO_RESOURCES_NAME, proto_file)
+    try:
+      os.makedirs(os.path.dirname(target))
+    except OSError as error:
+      if error.errno == errno.EEXIST:
+        pass
+      else:
+        raise
+    shutil.copy(source, target)
+    proto_files.append(relative_target)
+  return {GRPC_PYTHON_TOOLS_PACKAGE: proto_files}
+
 def protoc_ext_module():
   plugin_sources = [
       'grpc/tools/main.cc',
       'grpc_root/src/compiler/python_generator.cc'] + [
-      os.path.join('third_party/protobuf/src', cc_file)
+      os.path.join(protoc_lib_deps.CC_INCLUDE, cc_file)
       for cc_file in protoc_lib_deps.CC_FILES]
   plugin_ext = extension.Extension(
       name='grpc.tools.protoc_compiler',
@@ -67,7 +92,7 @@ def protoc_ext_module():
           '.',
           'grpc_root',
           'grpc_root/include',
-          'third_party/protobuf/src',
+          protoc_lib_deps.CC_INCLUDE,
       ],
       language='c++',
       define_macros=[('HAVE_PTHREAD', 1)],
@@ -88,9 +113,10 @@ setuptools.setup(
       protoc_ext_module(),
   ]),
   packages=setuptools.find_packages('.'),
-  # TODO(atash): Figure out why auditwheel doesn't like namespace packages.
-  #namespace_packages=['grpc'],
+  namespace_packages=['grpc'],
   install_requires=[
     'protobuf>=3.0.0a3',
+    'grpcio>=0.14.0',
   ],
+  package_data=package_data(),
 )
diff --git a/tools/distrib/python/make_grpcio_tools.py b/tools/distrib/python/make_grpcio_tools.py
index 50fbdbb14c..fd9b38b084 100755
--- a/tools/distrib/python/make_grpcio_tools.py
+++ b/tools/distrib/python/make_grpcio_tools.py
@@ -67,11 +67,16 @@ DEPS_FILE_CONTENT="""
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 # AUTO-GENERATED BY make_grpcio_tools.py!
-CC_FILES={}
+CC_FILES={cc_files}
+PROTO_FILES={proto_files}
+
+CC_INCLUDE={cc_include}
+PROTO_INCLUDE={proto_include}
 """
 
 # Bazel query result prefix for expected source files in protobuf.
 PROTOBUF_CC_PREFIX = '//:src/'
+PROTOBUF_PROTO_PREFIX = '//:src/'
 
 GRPC_ROOT = os.path.abspath(
     os.path.join(os.path.dirname(os.path.abspath(__file__)),
@@ -79,7 +84,8 @@ GRPC_ROOT = os.path.abspath(
 
 GRPC_PYTHON_ROOT = os.path.join(GRPC_ROOT, 'tools/distrib/python/grpcio_tools')
 
-GRPC_PROTOBUF = os.path.join(GRPC_ROOT, 'third_party/protobuf/src')
+GRPC_PYTHON_PROTOBUF_RELATIVE_ROOT = 'third_party/protobuf/src'
+GRPC_PROTOBUF = os.path.join(GRPC_ROOT, GRPC_PYTHON_PROTOBUF_RELATIVE_ROOT)
 GRPC_PROTOC_PLUGINS = os.path.join(GRPC_ROOT, 'src/compiler')
 GRPC_PYTHON_PROTOBUF = os.path.join(GRPC_PYTHON_ROOT,
                                     'third_party/protobuf/src')
@@ -93,18 +99,29 @@ GRPC_PYTHON_INCLUDE = os.path.join(GRPC_PYTHON_ROOT, 'grpc_root/include')
 
 BAZEL_DEPS = os.path.join(GRPC_ROOT, 'tools/distrib/python/bazel_deps.sh')
 BAZEL_DEPS_PROTOC_LIB_QUERY = '//:protoc_lib'
+BAZEL_DEPS_COMMON_PROTOS_QUERY = '//:well_known_protos'
+
 
+def bazel_query(query):
+  output = subprocess.check_output([BAZEL_DEPS, query])
+  return output.splitlines()
 
-def get_deps(query):
+def get_deps():
   """Write the result of the bazel query `query` against protobuf to
      `out_file`."""
-  output = subprocess.check_output([BAZEL_DEPS, query])
-  output = output.splitlines()
+  cc_files_output = bazel_query(BAZEL_DEPS_PROTOC_LIB_QUERY)
   cc_files = [
-      name for name in output
+      name[len(PROTOBUF_CC_PREFIX):] for name in cc_files_output
       if name.endswith('.cc') and name.startswith(PROTOBUF_CC_PREFIX)]
-  cc_files = [cc_file[len(PROTOBUF_CC_PREFIX):] for cc_file in cc_files]
-  deps_file_content = DEPS_FILE_CONTENT.format(cc_files)
+  proto_files_output = bazel_query(BAZEL_DEPS_COMMON_PROTOS_QUERY)
+  proto_files = [
+      name[len(PROTOBUF_PROTO_PREFIX):] for name in proto_files_output
+      if name.endswith('.proto') and name.startswith(PROTOBUF_PROTO_PREFIX)]
+  deps_file_content = DEPS_FILE_CONTENT.format(
+      cc_files=cc_files,
+      proto_files=proto_files,
+      cc_include=repr(GRPC_PYTHON_PROTOBUF_RELATIVE_ROOT),
+      proto_include=repr(GRPC_PYTHON_PROTOBUF_RELATIVE_ROOT))
   return deps_file_content
 
 
@@ -123,7 +140,7 @@ def main():
   shutil.copytree(GRPC_INCLUDE, GRPC_PYTHON_INCLUDE)
 
   try:
-    protoc_lib_deps_content = get_deps(BAZEL_DEPS_PROTOC_LIB_QUERY)
+    protoc_lib_deps_content = get_deps()
   except Exception as error:
     # We allow this script to succeed even if we couldn't get the dependencies,
     # as then we can assume that even without a successful bazel run the