Merge pull request #6475 from grpc/release-0_14

Upmerge from 0.14.0

13 files changed, 178 insertions, 48 deletions

diff --git a/tools/distrib/python/grpcio_tools/MANIFEST.in b/tools/distrib/python/grpcio_tools/MANIFEST.in
index e6ab312f09..7712834d64 100644
--- a/tools/distrib/python/grpcio_tools/MANIFEST.in
+++ b/tools/distrib/python/grpcio_tools/MANIFEST.in
@@ -1,5 +1,7 @@
+include grpc_version.py
 include protoc_deps.py
 include protoc_lib_deps.py
+include README.rst
 graft grpc
 graft grpc_root
 graft third_party
diff --git a/tools/distrib/python/grpcio_tools/README.rst b/tools/distrib/python/grpcio_tools/README.rst
new file mode 100644
index 0000000000..10d2fe8c30
--- /dev/null
+++ b/tools/distrib/python/grpcio_tools/README.rst
@@ -0,0 +1,128 @@
+gRPC Python Tools
+=================
+
+Package for gRPC Python tools.
+
+Installation
+------------
+
+The gRPC Python tools package is available for Linux, Mac OS X, and Windows
+running Python 2.7.
+
+From PyPI
+~~~~~~~~~
+
+If you are installing locally...
+
+::
+
+  $ pip install grpcio-tools
+
+Else system wide (on Ubuntu)...
+
+::
+
+  $ sudo pip install grpcio-tools
+
+If you're on Windows make sure that you installed the :code:`pip.exe` component
+when you installed Python (if not go back and install it!) then invoke:
+
+::
+
+  $ pip.exe install grpcio-tools
+
+Windows users may need to invoke :code:`pip.exe` from a command line ran as
+administrator.
+
+n.b. On Windows and on Mac OS X one *must* have a recent release of :code:`pip`
+to retrieve the proper wheel from PyPI. Be sure to upgrade to the latest
+version!
+
+You might also need to install Cython to handle installation via the source
+distribution if gRPC Python's system coverage with wheels does not happen to
+include your system.
+
+From Source
+~~~~~~~~~~~
+
+Building from source requires that you have the Python headers (usually a
+package named :code:`python-dev`) and Cython installed. It further requires a
+GCC-like compiler to go smoothly; you can probably get it to work without
+GCC-like stuff, but you may end up having a bad time.
+
+::
+
+  $ export REPO_ROOT=grpc  # REPO_ROOT can be any directory of your choice
+  $ git clone https://github.com/grpc/grpc.git $REPO_ROOT
+  $ cd $REPO_ROOT
+  $ git submodule update --init
+
+  $ cd tools/distrib/python/grpcio_tools
+  $ python ../make_grpcio_tools.py
+
+  # For the next command do `sudo pip install` if you get permission-denied errors
+  $ pip install .
+
+You cannot currently install Python from source on Windows. Things might work
+out for you in MSYS2 (follow the Linux instructions), but it isn't officially
+supported at the moment.
+
+Troubleshooting
+~~~~~~~~~~~~~~~
+
+Help, I ...
+
+* **... see a** :code:`pkg_resources.VersionConflict` **when I try to install
+  grpc**
+
+  This is likely because :code:`pip` doesn't own the offending dependency,
+  which in turn is likely because your operating system's package manager owns
+  it. You'll need to force the installation of the dependency:
+
+  :code:`pip install --ignore-installed $OFFENDING_DEPENDENCY`
+
+  For example, if you get an error like the following:
+
+  ::
+
+    Traceback (most recent call last):
+    File "<string>", line 17, in <module>
+     ...
+    File "/usr/lib/python2.7/dist-packages/pkg_resources.py", line 509, in find
+      raise VersionConflict(dist, req)
+    pkg_resources.VersionConflict: (six 1.8.0 (/usr/lib/python2.7/dist-packages), Requirement.parse('six>=1.10'))
+
+  You can fix it by doing:
+
+  ::
+
+    sudo pip install --ignore-installed six
+
+* **... see compiler errors on some platforms when either installing from source or from the source distribution**
+
+  If you see
+
+  ::
+
+    /tmp/pip-build-U8pSsr/cython/Cython/Plex/Scanners.c:4:20: fatal error: Python.h: No such file or directory
+    #include "Python.h"
+                    ^
+    compilation terminated.
+
+  You can fix it by installing `python-dev` package. i.e
+
+  ::
+
+    sudo apt-get install python-dev
+
+  If you see something similar to:
+
+  ::
+
+    third_party/protobuf/src/google/protobuf/stubs/mathlimits.h:173:31: note: in expansion of macro 'SIGNED_INT_MAX'
+    static const Type kPosMax = SIGNED_INT_MAX(Type); \\
+                               ^
+  And your toolchain is GCC (at the time of this writing, up through at least
+  GCC 6.0), this is probably a bug where GCC chokes on constant expressions
+  when the :code:`-fwrapv` flag is specified. You should consider setting your
+  environment with :code:`CFLAGS=-fno-wrapv` or using clang (:code:`CC=clang`).
diff --git a/tools/distrib/python/grpcio_tools/grpc/protoc/__init__.py b/tools/distrib/python/grpcio_tools/grpc/tools/__init__.py
index d5ad73a74a..d5ad73a74a 100644
--- a/tools/distrib/python/grpcio_tools/grpc/protoc/__init__.py
+++ b/tools/distrib/python/grpcio_tools/grpc/tools/__init__.py
diff --git a/tools/distrib/python/grpcio_tools/grpc/protoc/main.cc b/tools/distrib/python/grpcio_tools/grpc/tools/main.cc
index c9936a3a6b..81675b4e6f 100644
--- a/tools/distrib/python/grpcio_tools/grpc/protoc/main.cc
+++ b/tools/distrib/python/grpcio_tools/grpc/tools/main.cc
@@ -32,7 +32,7 @@
 
 #include "src/compiler/python_generator.h"
 
-#include "grpc/protoc/main.h"
+#include "grpc/tools/main.h"
 
 int protoc_main(int argc, char* argv[]) {
   google::protobuf::compiler::CommandLineInterface cli;
diff --git a/tools/distrib/python/grpcio_tools/grpc/protoc/main.h b/tools/distrib/python/grpcio_tools/grpc/tools/main.h
index ea2860ff02..ea2860ff02 100644
--- a/tools/distrib/python/grpcio_tools/grpc/protoc/main.h
+++ b/tools/distrib/python/grpcio_tools/grpc/tools/main.h
diff --git a/tools/distrib/python/grpcio_tools/grpc/protoc/compiler.py b/tools/distrib/python/grpcio_tools/grpc/tools/protoc.py
index caafc544b2..b4dd0ecae2 100644
--- a/tools/distrib/python/grpcio_tools/grpc/protoc/compiler.py
+++ b/tools/distrib/python/grpcio_tools/grpc/tools/protoc.py
@@ -31,7 +31,7 @@
 
 import sys
 
-from grpc.protoc import protoc_compiler
+from grpc.tools import protoc_compiler
 
 
 if __name__ == '__main__':
diff --git a/tools/distrib/python/grpcio_tools/grpc/protoc/protoc_compiler.pyx b/tools/distrib/python/grpcio_tools/grpc/tools/protoc_compiler.pyx
index af15f3db30..a6530127c0 100644
--- a/tools/distrib/python/grpcio_tools/grpc/protoc/protoc_compiler.pyx
+++ b/tools/distrib/python/grpcio_tools/grpc/tools/protoc_compiler.pyx
@@ -29,7 +29,7 @@
 
 from libc cimport stdlib
 
-cdef extern from "grpc/protoc/main.h":
+cdef extern from "grpc/tools/main.h":
   int protoc_main(int argc, char *argv[])
 
 def run_main(list args not None):
diff --git a/tools/distrib/python/grpcio_tools/setup.py b/tools/distrib/python/grpcio_tools/setup.py
index 0281c01796..576f7ae32a 100644
--- a/tools/distrib/python/grpcio_tools/setup.py
+++ b/tools/distrib/python/grpcio_tools/setup.py
@@ -30,6 +30,7 @@
 from distutils import extension
 import os
 import os.path
+import shlex
 import sys
 
 import setuptools
@@ -40,18 +41,28 @@ from setuptools.command import build_ext
 os.chdir(os.path.dirname(os.path.abspath(__file__)))
 sys.path.insert(0, os.path.abspath('.'))
 
+# There are some situations (like on Windows) where CC, CFLAGS, and LDFLAGS are
+# entirely ignored/dropped/forgotten by distutils and its Cygwin/MinGW support.
+# We use these environment variables to thus get around that without locking
+# ourselves in w.r.t. the multitude of operating systems this ought to build on.
+# By default we assume a GCC-like compiler.
+EXTRA_COMPILE_ARGS = shlex.split(os.environ.get('GRPC_PYTHON_CFLAGS',
+                                                '-frtti -std=c++11'))
+EXTRA_LINK_ARGS = shlex.split(os.environ.get('GRPC_PYTHON_LDFLAGS',
+                                             '-lpthread'))
+
 import protoc_lib_deps
 import grpc_version
 
 def protoc_ext_module():
   plugin_sources = [
-      'grpc/protoc/main.cc',
+      'grpc/tools/main.cc',
       'grpc_root/src/compiler/python_generator.cc'] + [
       os.path.join('third_party/protobuf/src', cc_file)
       for cc_file in protoc_lib_deps.CC_FILES]
   plugin_ext = extension.Extension(
-      name='grpc.protoc.protoc_compiler',
-      sources=['grpc/protoc/protoc_compiler.pyx'] + plugin_sources,
+      name='grpc.tools.protoc_compiler',
+      sources=['grpc/tools/protoc_compiler.pyx'] + plugin_sources,
       include_dirs=[
           '.',
           'grpc_root',
@@ -60,7 +71,8 @@ def protoc_ext_module():
       ],
       language='c++',
       define_macros=[('HAVE_PTHREAD', 1)],
-      extra_compile_args=['-lpthread', '-frtti', '-std=c++11'],
+      extra_compile_args=EXTRA_COMPILE_ARGS,
+      extra_link_args=EXTRA_LINK_ARGS,
   )
   return plugin_ext
 
@@ -71,7 +83,7 @@ def maybe_cythonize(exts):
 setuptools.setup(
   name='grpcio_tools',
   version=grpc_version.VERSION,
-  license='',
+  license='3-clause BSD',
   ext_modules=maybe_cythonize([
       protoc_ext_module(),
   ]),
diff --git a/tools/run_tests/artifact_targets.py b/tools/run_tests/artifact_targets.py
index 477bd46888..bd1269ceb7 100644
--- a/tools/run_tests/artifact_targets.py
+++ b/tools/run_tests/artifact_targets.py
@@ -113,37 +113,7 @@ class PythonArtifact:
       # defines ourselves.
       # TODO(atash) get better platform-detection support in core so we don't
       # need to do this manually...
-      environ['CFLAGS'] = " ".join([
-        '-DGPR_NO_AUTODETECT_PLATFORM',
-        '-DGPR_PLATFORM_STRING=\\"manylinux\\"',
-        '-DGPR_POSIX_CRASH_HANDLER=1',
-        '-DGPR_CPU_LINUX=1',
-        '-DGPR_GCC_ATOMIC=1',
-        '-DGPR_GCC_TLS=1',
-        '-DGPR_LINUX=1',
-        '-DGPR_LINUX_LOG=1',
-        #'-DGPR_LINUX_MULTIPOLL_WITH_EPOLL=1',
-        '-DGPR_POSIX_SOCKET=1',
-        '-DGPR_POSIX_WAKEUP_FD=1',
-        '-DGPR_POSIX_SOCKETADDR=1',
-        #'-DGPR_LINUX_EVENTFD=1',
-        '-DGPR_POSIX_NO_SPECIAL_WAKEUP_FD=1',
-        #'-DGPR_LINUX_SOCKETUTILS=1',
-        '-DGPR_POSIX_SOCKETUTILS=1',
-        '-DGPR_HAVE_UNIX_SOCKET=1',
-        '-DGPR_HAVE_IP_PKTINFO=1',
-        '-DGPR_HAVE_IPV6_RECVPKTINFO=1',
-        '-DGPR_LINUX_ENV=1',
-        '-DGPR_POSIX_FILE=1',
-        '-DGPR_POSIX_TMPFILE=1',
-        '-DGPR_POSIX_STRING=1',
-        '-DGPR_POSIX_SUBPROCESS=1',
-        '-DGPR_POSIX_SYNC=1',
-        '-DGPR_POSIX_TIME=1',
-        '-DGPR_GETPID_IN_UNISTD_H=1',
-        '-DGPR_HAVE_MSG_NOSIGNAL=1',
-        '-DGPR_ARCH_{arch}=1'.format(arch=('32' if self.arch == 'x86' else '64')),
-      ])
+      environ['CFLAGS'] = '-DGPR_MANYLINUX1=1'
       return create_docker_jobspec(self.name,
           'tools/dockerfile/grpc_artifact_python_manylinux_%s' % self.arch,
           'tools/run_tests/build_artifact_python.sh',
diff --git a/tools/run_tests/build_artifact_node.bat b/tools/run_tests/build_artifact_node.bat
index 84c63c28a2..c5bd726db7 100644
--- a/tools/run_tests/build_artifact_node.bat
+++ b/tools/run_tests/build_artifact_node.bat
@@ -27,7 +27,7 @@
 @rem (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 @rem OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-set node_versions=0.12.0 1.0.0 1.1.0 2.0.0 3.0.0 4.0.0 5.0.0
+set node_versions=0.12.0 1.0.0 1.1.0 2.0.0 3.0.0 4.0.0 5.0.0 6.0.0
 
 set PATH=%PATH%;C:\Program Files\nodejs\;%APPDATA%\npm
 
@@ -38,12 +38,12 @@ call npm update || goto :error
 mkdir artifacts
 
 for %%v in (%node_versions%) do (
-  call node-pre-gyp configure build --target=%%v --target_arch=%1
+  call .\node_modules\.bin\node-pre-gyp.cmd configure build --target=%%v --target_arch=%1
 
 @rem Try again after removing openssl headers
   rmdir "%HOMEDRIVE%%HOMEPATH%\.node-gyp\%%v\include\node\openssl" /S /Q
   rmdir "%HOMEDRIVE%%HOMEPATH%\.node-gyp\iojs-%%v\include\node\openssl" /S /Q
-  call node-pre-gyp build package testpackage --target=%%v --target_arch=%1 || goto :error
+  call .\node_modules\.bin\node-pre-gyp.cmd build package testpackage --target=%%v --target_arch=%1 || goto :error
 
   xcopy /Y /I /S build\stage\* artifacts\ || goto :error
 )
diff --git a/tools/run_tests/build_artifact_node.sh b/tools/run_tests/build_artifact_node.sh
index ef3476a038..9d06472aa4 100755
--- a/tools/run_tests/build_artifact_node.sh
+++ b/tools/run_tests/build_artifact_node.sh
@@ -42,7 +42,7 @@ mkdir -p artifacts
 
 npm update
 
-node_versions=( 0.12.0 1.0.0 1.1.0 2.0.0 3.0.0 4.0.0 5.0.0 )
+node_versions=( 0.12.0 1.0.0 1.1.0 2.0.0 3.0.0 4.0.0 5.0.0 6.0.0 )
 
 for version in ${node_versions[@]}
 do
diff --git a/tools/run_tests/build_artifact_python.bat b/tools/run_tests/build_artifact_python.bat
index 636ae0d393..fea0275426 100644
--- a/tools/run_tests/build_artifact_python.bat
+++ b/tools/run_tests/build_artifact_python.bat
@@ -41,7 +41,7 @@ copy /Y vsprojects\Release\grpc_dll.dll src\python\grpcio\grpc\_cython\_windows\
 copy /Y vsprojects\x64\Release\grpc_dll.dll src\python\grpcio\grpc\_cython\_windows\grpc_c.64.python || goto :error
 
 
-set PATH=C:\%1;C:\%1\scripts;%PATH%
+set PATH=C:\%1;C:\%1\scripts;C:\msys64\mingw%2\bin;%PATH%
 
 pip install --upgrade six
 pip install --upgrade setuptools
@@ -50,12 +50,26 @@ pip install -rrequirements.txt
 set GRPC_PYTHON_USE_CUSTOM_BDIST=0
 set GRPC_PYTHON_BUILD_WITH_CYTHON=1
 
+@rem TODO(atash): maybe we could avoid the grpc_c.(32|64).python shim above if
+@rem this used the right python build?
 python setup.py bdist_wheel
 
 @rem Build gRPC Python tools
-set PATH=C:\msys64\mingw%2\bin;%PATH%
-set CC=C:\msys64\mingw%2\bin\g++.exe
-set CFLAGS=-fno-wrapv
+@rem
+@rem Because this is windows and *everything seems to hate Windows* we have to
+@rem set all of these flags ourselves because Python won't help us (see the
+@rem setup.py of the grpcio_tools project).
+set GRPC_PYTHON_CFLAGS=-fno-wrapv -frtti -std=c++11
+@rem Further confusing things, MSYS2's mingw64 tries to dynamically link
+@rem libgcc, libstdc++, and winpthreads. We have to override this or our
+@rem extensions end up linking to MSYS2 DLLs, which the normal Python on
+@rem Windows user won't have... and ON TOP OF THIS, there's MinGW's GCC default
+@rem behavior of linking msvcrt.dll as the C runtime library, which we need to
+@rem override so that Python's distutils doesn't link us against multiple C
+@rem runtimes.
+python -c "from distutils.cygwinccompiler import get_msvcr; print(get_msvcr()[0])" > temp.txt
+set /p PYTHON_MSVCR=<temp.txt
+set GRPC_PYTHON_LDFLAGS=-static-libgcc -static-libstdc++ -mcrtdll=%PYTHON_MSVCR% -static -lpthread
 python tools\distrib\python\make_grpcio_tools.py
 if %2 == 32 (
   python tools\distrib\python\grpcio_tools\setup.py build_ext -c mingw32
diff --git a/tools/run_tests/build_artifact_python.sh b/tools/run_tests/build_artifact_python.sh
index 35c3a48afa..4320f978e3 100755
--- a/tools/run_tests/build_artifact_python.sh
+++ b/tools/run_tests/build_artifact_python.sh
@@ -59,7 +59,11 @@ ${SETARCH_CMD} ${PYTHON} setup.py  \
 ${SETARCH_CMD} ${PYTHON} setup.py  \
     bdist_wheel
 
-# Build gRPC tools package
+# Build gRPC tools package source distribution
+${SETARCH_CMD} ${PYTHON} tools/distrib/python/grpcio_tools/setup.py  \
+    sdist
+
+# Build gRPC tools package binary distribution
 ${PYTHON} tools/distrib/python/make_grpcio_tools.py
 CFLAGS="$CFLAGS -fno-wrapv" ${SETARCH_CMD} \
   ${PYTHON} tools/distrib/python/grpcio_tools/setup.py bdist_wheel