Merge github.com:grpc/grpc into fix-proto-docker

9 files changed, 192 insertions, 126 deletions

diff --git a/src/python/grpcio/README.rst b/src/python/grpcio/README.rst
index c7b5a3bde4..698c760ebe 100644
--- a/src/python/grpcio/README.rst
+++ b/src/python/grpcio/README.rst
@@ -1,22 +1,40 @@
 gRPC Python
 ===========
 
-Package for GRPC Python.
+Package for gRPC Python.
 
-Dependencies
+Installation
 ------------
 
-Ensure you have installed the gRPC core.  On Mac OS X, install homebrew_.
-Run the following command to install gRPC Python.
+gRPC Python is available for Linux and Mac OS X running Python 2.7.
+
+From PyPI
+~~~~~~~~~
+
+If you are installing locally...
 
 ::
 
-  $ curl -fsSL https://goo.gl/getgrpc | bash -s python
+  $ pip install grpcio
+
+Else system wide (on Ubuntu)...
+
+::
 
-This will download and run the [gRPC install script][] to install grpc core. The script then uses pip to install this package.  It also installs the Protocol Buffers compiler (_protoc_) and the gRPC _protoc_ plugin for python.
+  $ sudo pip install grpcio
+
+From Source
+~~~~~~~~~~~
+
+Building from source requires that you have the Python headers (usually a
+package named `python-dev`).
+
+::
 
-Otherwise, `install from source`_
+  $ export REPO_ROOT=grpc
+  $ git clone https://github.com/grpc/grpc.git $REPO_ROOT
+  $ cd $REPO_ROOT
+  $ pip install .
 
-.. _`install from source`: https://github.com/grpc/grpc/blob/master/src/python/README.md#building-from-source
-.. _homebrew: http://brew.sh
-.. _`gRPC install script`: https://raw.githubusercontent.com/grpc/homebrew-grpc/master/scripts/install
+Note that `$REPO_ROOT` can be assigned to whatever directory name floats your
+fancy.
diff --git a/src/python/grpcio/commands.py b/src/python/grpcio/commands.py
index 774e7ad6a1..aa29c728f2 100644
--- a/src/python/grpcio/commands.py
+++ b/src/python/grpcio/commands.py
@@ -41,7 +41,6 @@ import sys
 import traceback
 
 import setuptools
-from setuptools.command import bdist_egg
 from setuptools.command import build_ext
 from setuptools.command import build_py
 from setuptools.command import easy_install
@@ -52,13 +51,6 @@ import support
 
 PYTHON_STEM = os.path.dirname(os.path.abspath(__file__))
 
-BINARIES_REPOSITORY = os.environ.get(
-    'GRPC_PYTHON_BINARIES_REPOSITORY',
-    'https://storage.googleapis.com/grpc-precompiled-binaries/python')
-
-USE_GRPC_CUSTOM_BDIST = bool(int(os.environ.get(
-    'GRPC_PYTHON_USE_CUSTOM_BDIST', '1')))
-
 CONF_PY_ADDENDUM = """
 extensions.append('sphinx.ext.napoleon')
 napoleon_google_docstring = True
@@ -74,126 +66,39 @@ class CommandError(Exception):
 
 # TODO(atash): Remove this once PyPI has better Linux bdist support. See
 # https://bitbucket.org/pypa/pypi/issues/120/binary-wheels-for-linux-are-not-supported
-def _get_grpc_custom_bdist_egg(decorated_basename, target_egg_basename):
-  """Returns a string path to a .egg file for Linux to install.
+def _get_grpc_custom_bdist(decorated_basename, target_bdist_basename):
+  """Returns a string path to a bdist file for Linux to install.
 
-  If we can retrieve a pre-compiled egg from online, uses it. Else, emits a
+  If we can retrieve a pre-compiled bdist from online, uses it. Else, emits a
   warning and builds from source.
   """
+  # TODO(atash): somehow the name that's returned from `wheel` is different
+  # between different versions of 'wheel' (but from a compatibility standpoint,
+  # the names are compatible); we should have some way of determining name
+  # compatibility in the same way `wheel` does to avoid having to rename all of
+  # the custom wheels that we build/upload to GCS.
+
   # Break import style to ensure that setup.py has had a chance to install the
-  # relevant package eggs.
+  # relevant package.
   from six.moves.urllib import request
-  decorated_path = decorated_basename + '.egg'
+  decorated_path = decorated_basename + GRPC_CUSTOM_BDIST_EXT
   try:
     url = BINARIES_REPOSITORY + '/{target}'.format(target=decorated_path)
-    egg_data = request.urlopen(url).read()
+    bdist_data = request.urlopen(url).read()
   except IOError as error:
     raise CommandError(
-        '{}\n\nCould not find the bdist egg {}: {}'
+        '{}\n\nCould not find the bdist {}: {}'
             .format(traceback.format_exc(), decorated_path, error.message))
-  # Our chosen local egg path.
-  egg_path = target_egg_basename + '.egg'
+  # Our chosen local bdist path.
+  bdist_path = target_bdist_basename + GRPC_CUSTOM_BDIST_EXT
   try:
-    with open(egg_path, 'w') as egg_file:
-      egg_file.write(egg_data)
+    with open(bdist_path, 'w') as bdist_file:
+      bdist_file.write(bdist_data)
   except IOError as error:
     raise CommandError(
-        '{}\n\nCould not write grpcio egg: {}'
+        '{}\n\nCould not write grpcio bdist: {}'
             .format(traceback.format_exc(), error.message))
-  return egg_path
-
-
-class EggNameMixin(object):
-  """Mixin for setuptools.Command classes to enable acquiring the egg name."""
-
-  def egg_name(self, with_custom):
-    """
-    Args:
-      with_custom: Boolean describing whether or not to decorate the egg name
-        with custom gRPC-specific target information.
-    """
-    egg_command = self.get_finalized_command('bdist_egg')
-    base = os.path.splitext(os.path.basename(egg_command.egg_output))[0]
-    if with_custom:
-      flavor = 'ucs2' if sys.maxunicode == 65535 else 'ucs4'
-      return '{base}-{flavor}'.format(base=base, flavor=flavor)
-    else:
-      return base
-
-
-class Install(install.install, EggNameMixin):
-  """Custom Install command for gRPC Python.
-
-  This is for bdist shims and whatever else we might need a custom install
-  command for.
-  """
-
-  user_options = install.install.user_options + [
-      # TODO(atash): remove this once PyPI has better Linux bdist support. See
-      # https://bitbucket.org/pypa/pypi/issues/120/binary-wheels-for-linux-are-not-supported
-      ('use-grpc-custom-bdist', None,
-       'Whether to retrieve a binary from the gRPC binary repository instead '
-       'of building from source.'),
-  ]
-
-  def initialize_options(self):
-    install.install.initialize_options(self)
-    self.use_grpc_custom_bdist = USE_GRPC_CUSTOM_BDIST
-
-  def finalize_options(self):
-    install.install.finalize_options(self)
-
-  def run(self):
-    if self.use_grpc_custom_bdist:
-      try:
-        try:
-          egg_path = _get_grpc_custom_bdist_egg(self.egg_name(True),
-                                                self.egg_name(False))
-        except CommandError as error:
-          sys.stderr.write(
-              '\nWARNING: Failed to acquire grpcio prebuilt binary:\n'
-              '{}.\n\n'.format(error.message))
-          raise
-        try:
-          self._run_bdist_retrieval_install(egg_path)
-        except Exception as error:
-          # if anything else happens (and given how there's no way to really know
-          # what's happening in setuptools here, I mean *anything*), warn the user
-          # and fall back to building from source.
-          sys.stderr.write(
-              '{}\nWARNING: Failed to install grpcio prebuilt binary.\n\n'
-                  .format(traceback.format_exc()))
-          raise
-      except Exception:
-        install.install.run(self)
-    else:
-      install.install.run(self)
-
-  # TODO(atash): Remove this once PyPI has better Linux bdist support. See
-  # https://bitbucket.org/pypa/pypi/issues/120/binary-wheels-for-linux-are-not-supported
-  def _run_bdist_retrieval_install(self, bdist_egg):
-    easy_install = self.distribution.get_command_class('easy_install')
-    easy_install_command = easy_install(
-        self.distribution, args='x', root=self.root, record=self.record,
-    )
-    easy_install_command.ensure_finalized()
-    easy_install_command.always_copy_from = '.'
-    easy_install_command.package_index.scan(glob.glob('*.egg'))
-    arguments = [bdist_egg]
-    if setuptools.bootstrap_install_from:
-      args.insert(0, setuptools.bootstrap_install_from)
-    easy_install_command.args = arguments
-    easy_install_command.run()
-    setuptools.bootstrap_install_from = None
-
-
-class BdistEggCustomName(bdist_egg.bdist_egg, EggNameMixin):
-  """Thin wrapper around the bdist_egg command to build with our custom name."""
-
-  def run(self):
-    bdist_egg.bdist_egg.run(self)
-    target = os.path.join(self.dist_dir, '{}.egg'.format(self.egg_name(True)))
-    shutil.move(self.get_outputs()[0], target)
+  return bdist_path
 
 
 class SphinxDocumentation(setuptools.Command):
diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/grpc.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/grpc.pxi
index bbeed9ad40..800d0ea2f6 100644
--- a/src/python/grpcio/grpc/_cython/_cygrpc/grpc.pxi
+++ b/src/python/grpcio/grpc/_cython/_cygrpc/grpc.pxi
@@ -36,7 +36,7 @@ cdef extern from "grpc/_cython/loader.h":
   ctypedef unsigned uint32_t
   ctypedef long int64_t
 
-  int pygrpc_load_core(const char*)
+  int pygrpc_load_core(char*)
 
   void *gpr_malloc(size_t size)
   void gpr_free(void *ptr)
diff --git a/src/python/grpcio/grpc/_cython/imports.generated.c b/src/python/grpcio/grpc/_cython/imports.generated.c
index 817303c8a4..4b1860ce8c 100644
--- a/src/python/grpcio/grpc/_cython/imports.generated.c
+++ b/src/python/grpcio/grpc/_cython/imports.generated.c
@@ -137,6 +137,7 @@ grpc_auth_context_add_cstring_property_type grpc_auth_context_add_cstring_proper
 grpc_auth_context_set_peer_identity_property_name_type grpc_auth_context_set_peer_identity_property_name_import;
 grpc_channel_credentials_release_type grpc_channel_credentials_release_import;
 grpc_google_default_credentials_create_type grpc_google_default_credentials_create_import;
+grpc_set_ssl_roots_override_callback_type grpc_set_ssl_roots_override_callback_import;
 grpc_ssl_credentials_create_type grpc_ssl_credentials_create_import;
 grpc_call_credentials_release_type grpc_call_credentials_release_import;
 grpc_composite_channel_credentials_create_type grpc_composite_channel_credentials_create_import;
@@ -296,6 +297,10 @@ gpr_thd_options_is_joinable_type gpr_thd_options_is_joinable_import;
 gpr_thd_currentid_type gpr_thd_currentid_import;
 gpr_thd_join_type gpr_thd_join_import;
 
+#ifdef __cplusplus
+extern "C" {
+#endif  /* __cpluslus */
+
 void pygrpc_load_imports(HMODULE library) {
   census_initialize_import = (census_initialize_type) GetProcAddress(library, "census_initialize");
   census_shutdown_import = (census_shutdown_type) GetProcAddress(library, "census_shutdown");
@@ -397,6 +402,7 @@ void pygrpc_load_imports(HMODULE library) {
   grpc_auth_context_set_peer_identity_property_name_import = (grpc_auth_context_set_peer_identity_property_name_type) GetProcAddress(library, "grpc_auth_context_set_peer_identity_property_name");
   grpc_channel_credentials_release_import = (grpc_channel_credentials_release_type) GetProcAddress(library, "grpc_channel_credentials_release");
   grpc_google_default_credentials_create_import = (grpc_google_default_credentials_create_type) GetProcAddress(library, "grpc_google_default_credentials_create");
+  grpc_set_ssl_roots_override_callback_import = (grpc_set_ssl_roots_override_callback_type) GetProcAddress(library, "grpc_set_ssl_roots_override_callback");
   grpc_ssl_credentials_create_import = (grpc_ssl_credentials_create_type) GetProcAddress(library, "grpc_ssl_credentials_create");
   grpc_call_credentials_release_import = (grpc_call_credentials_release_type) GetProcAddress(library, "grpc_call_credentials_release");
   grpc_composite_channel_credentials_create_import = (grpc_composite_channel_credentials_create_type) GetProcAddress(library, "grpc_composite_channel_credentials_create");
@@ -557,4 +563,8 @@ void pygrpc_load_imports(HMODULE library) {
   gpr_thd_join_import = (gpr_thd_join_type) GetProcAddress(library, "gpr_thd_join");
 }
 
+#ifdef __cplusplus
+}
+#endif  /* __cpluslus */
+
 #endif /* !GPR_WIN32 */
diff --git a/src/python/grpcio/grpc/_cython/imports.generated.h b/src/python/grpcio/grpc/_cython/imports.generated.h
index 6d0a6e06c0..ca30742abc 100644
--- a/src/python/grpcio/grpc/_cython/imports.generated.h
+++ b/src/python/grpcio/grpc/_cython/imports.generated.h
@@ -361,6 +361,9 @@ extern grpc_channel_credentials_release_type grpc_channel_credentials_release_im
 typedef grpc_channel_credentials *(*grpc_google_default_credentials_create_type)(void);
 extern grpc_google_default_credentials_create_type grpc_google_default_credentials_create_import;
 #define grpc_google_default_credentials_create grpc_google_default_credentials_create_import
+typedef void(*grpc_set_ssl_roots_override_callback_type)(grpc_ssl_roots_override_callback cb);
+extern grpc_set_ssl_roots_override_callback_type grpc_set_ssl_roots_override_callback_import;
+#define grpc_set_ssl_roots_override_callback grpc_set_ssl_roots_override_callback_import
 typedef grpc_channel_credentials *(*grpc_ssl_credentials_create_type)(const char *pem_root_certs, grpc_ssl_pem_key_cert_pair *pem_key_cert_pair, void *reserved);
 extern grpc_ssl_credentials_create_type grpc_ssl_credentials_create_import;
 #define grpc_ssl_credentials_create grpc_ssl_credentials_create_import
@@ -836,8 +839,16 @@ typedef void(*gpr_thd_join_type)(gpr_thd_id t);
 extern gpr_thd_join_type gpr_thd_join_import;
 #define gpr_thd_join gpr_thd_join_import
 
+#ifdef __cplusplus
+extern "C" {
+#endif  /* __cpluslus */
+
 void pygrpc_load_imports(HMODULE library);
 
+#ifdef __cplusplus
+}
+#endif  /* __cpluslus */
+
 #else /* !GPR_WIN32 */
 
 #include <grpc/support/alloc.h>
diff --git a/src/python/grpcio/grpc/_cython/loader.c b/src/python/grpcio/grpc/_cython/loader.c
index cdd47deed3..3b72806ea1 100644
--- a/src/python/grpcio/grpc/_cython/loader.c
+++ b/src/python/grpcio/grpc/_cython/loader.c
@@ -33,6 +33,10 @@
 
 #include "loader.h"
 
+#ifdef __cplusplus
+extern "C" {
+#endif  /* __cpluslus  */
+
 #if GPR_WIN32
 
 int pygrpc_load_core(char *path) {
@@ -56,4 +60,9 @@ int pygrpc_load_core(char *path) {
 
 int pygrpc_load_core(char *path) { return 1; }
 
-#endif
+#endif  /* !GPR_WIN32 */
+
+#ifdef __cplusplus
+}
+#endif  /* __cpluslus */
+
diff --git a/src/python/grpcio/grpc/_cython/loader.h b/src/python/grpcio/grpc/_cython/loader.h
index dd31e1561b..3b8796d39f 100644
--- a/src/python/grpcio/grpc/_cython/loader.h
+++ b/src/python/grpcio/grpc/_cython/loader.h
@@ -39,7 +39,16 @@
 /* Additional inclusions not covered by "imports.generated.h" */
 #include <grpc/byte_buffer_reader.h>
 
+#ifdef __cplusplus
+extern "C" {
+#endif  /* __cpluslus */
+
 /* Attempts to load the core if necessary, and return non-zero upon succes. */
 int pygrpc_load_core(char *path);
 
+#ifdef __cplusplus
+}
+#endif  /* __cpluslus */
+
 #endif /* GRPC_RB_BYTE_BUFFER_H_ */
+
diff --git a/src/python/grpcio/grpc_core_dependencies.py b/src/python/grpcio/grpc_core_dependencies.py
index 60af862305..5d7e31ba01 100644
--- a/src/python/grpcio/grpc_core_dependencies.py
+++ b/src/python/grpcio/grpc_core_dependencies.py
@@ -122,6 +122,7 @@ CORE_SOURCE_FILES = [
   'src/core/client_config/resolvers/sockaddr_resolver.c',
   'src/core/client_config/subchannel.c',
   'src/core/client_config/subchannel_factory.c',
+  'src/core/client_config/subchannel_index.c',
   'src/core/client_config/uri_parser.c',
   'src/core/compression/algorithm.c',
   'src/core/compression/message_compress.c',
@@ -225,6 +226,7 @@ CORE_SOURCE_FILES = [
   'src/core/transport/transport_op_string.c',
   'src/core/census/context.c',
   'src/core/census/initialize.c',
+  'src/core/census/log.c',
   'src/core/census/operation.c',
   'src/core/census/placeholders.c',
   'src/core/census/tracing.c',
diff --git a/src/python/grpcio/precompiled.py b/src/python/grpcio/precompiled.py
new file mode 100644
index 0000000000..05c651b506
--- /dev/null
+++ b/src/python/grpcio/precompiled.py
@@ -0,0 +1,102 @@
+# Copyright 2015-2016, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import os
+import platform
+import shutil
+import sys
+
+import setuptools
+
+import commands
+import grpc_version
+
+try:
+  from urllib2 import urlopen
+except ImportError:
+  from urllib.request import urlopen
+
+PYTHON_STEM = os.path.dirname(os.path.abspath(__file__))
+BINARIES_REPOSITORY = os.environ.get(
+    'GRPC_PYTHON_BINARIES_REPOSITORY',
+    'https://storage.googleapis.com/grpc-precompiled-binaries/python')
+USE_PRECOMPILED_BINARIES = bool(int(os.environ.get(
+    'GRPC_PYTHON_USE_PRECOMPILED_BINARIES', '1')))
+
+def _tagged_ext_name(base):
+  uname = platform.uname()
+  tags = '-'.join((grpc_version.VERSION, uname[0], uname[4]))
+  flavor = 'ucs2' if sys.maxunicode == 65535 else 'ucs4'
+  return '{base}-{tags}-{flavor}'.format(base=base, tags=tags, flavor=flavor)
+
+
+class BuildTaggedExt(setuptools.Command):
+
+  description = 'build the gRPC tagged extensions'
+  user_options = []
+
+  def initialize_options(self):
+    # distutils requires this override.
+    pass
+
+  def finalize_options(self):
+    # distutils requires this override.
+    pass
+
+  def run(self):
+    if 'linux' in sys.platform:
+      self.run_command('build_ext')
+      try:
+        os.makedirs('dist/')
+      except OSError:
+        pass
+      shutil.copyfile(
+          os.path.join(PYTHON_STEM, 'grpc/_cython/cygrpc.so'),
+          'dist/{}.so'.format(_tagged_ext_name('cygrpc')))
+    else:
+      sys.stderr.write('nothing to do for build_tagged_ext\n')
+
+
+def update_setup_arguments(setup_arguments):
+  url = '{}/{}.so'.format(BINARIES_REPOSITORY, _tagged_ext_name('cygrpc'))
+  target_path = os.path.join(PYTHON_STEM, 'grpc/_cython/cygrpc.so')
+  try:
+    extension = urlopen(url).read()
+  except:
+    sys.stderr.write(
+        'could not download precompiled extension: {}\n'.format(url))
+    return
+  try:
+    with open(target_path, 'w') as target:
+      target.write(extension)
+    setup_arguments['ext_modules'] = []
+  except:
+    sys.stderr.write(
+        'could not write precompiled extension to directory: {} -> {}\n'
+            .format(url, target_path))