Merge pull request #7802 from nicolasnoble/upmerge-from-v1.0.x

Upmerge from v1.0.x

4 files changed, 84 insertions, 61 deletions

diff --git a/tools/run_tests/artifact_targets.py b/tools/run_tests/artifact_targets.py
index 8550ee7b84..d36f963a7c 100644
--- a/tools/run_tests/artifact_targets.py
+++ b/tools/run_tests/artifact_targets.py
@@ -31,6 +31,8 @@
 """Definition of targets to build artifacts."""
 
 import os.path
+import random
+import string
 import sys
 
 import jobset
@@ -79,27 +81,16 @@ _ARCH_FLAG_MAP = {
   'x64': '-m64'
 }
 
-python_windows_version_arch_map = {
-  ('x86', '2.7'): 'Python27_32bits',
-  ('x64', '2.7'): 'Python27',
-  ('x86', '3.4'): 'Python34_32bits',
-  ('x64', '3.4'): 'Python34',
-}
 
 class PythonArtifact:
   """Builds Python artifacts."""
 
-  def __init__(self, platform, arch, python_version, manylinux_build=None):
-    if manylinux_build:
-      self.name = 'python%s_%s_%s_%s' % (python_version, platform, arch, manylinux_build)
-    else:
-      self.name = 'python%s_%s_%s' % (python_version, platform, arch)
+  def __init__(self, platform, arch, py_version):
+    self.name = 'python_%s_%s_%s' % (platform, arch, py_version)
     self.platform = platform
     self.arch = arch
-    self.labels = ['artifact', 'python', python_version, platform, arch]
-    self.python_version = python_version
-    self.python_windows_prefix = python_windows_version_arch_map[arch, python_version]
-    self.manylinux_build = manylinux_build
+    self.labels = ['artifact', 'python', platform, arch, py_version]
+    self.py_version = py_version
 
   def pre_build_jobspecs(self):
     return []
@@ -111,8 +102,8 @@ class PythonArtifact:
         environ['SETARCH_CMD'] = 'linux32'
       # Inside the manylinux container, the python installations are located in
       # special places...
-      environ['PYTHON'] = '/opt/python/{}/bin/python'.format(self.manylinux_build)
-      environ['PIP'] = '/opt/python/{}/bin/pip'.format(self.manylinux_build)
+      environ['PYTHON'] = '/opt/python/{}/bin/python'.format(self.py_version)
+      environ['PIP'] = '/opt/python/{}/bin/pip'.format(self.py_version)
       # Platform autodetection for the manylinux1 image breaks so we set the
       # defines ourselves.
       # TODO(atash) get better platform-detection support in core so we don't
@@ -126,14 +117,24 @@ class PythonArtifact:
           environ=environ,
           timeout_seconds=60*60)
     elif self.platform == 'windows':
+      if 'Python27' in self.py_version or 'Python34' in self.py_version:
+        environ['EXT_COMPILER'] = 'mingw32'
+      else:
+        environ['EXT_COMPILER'] = 'msvc'
+      # For some reason, the batch script %random% always runs with the same
+      # seed.  We create a random temp-dir here
+      dir = ''.join(random.choice(string.ascii_uppercase) for _ in range(10))
       return create_jobspec(self.name,
                             ['tools\\run_tests\\build_artifact_python.bat',
-                             self.python_windows_prefix,
-                             '32' if self.arch == 'x86' else '64'
+                             self.py_version,
+                             '32' if self.arch == 'x86' else '64',
+                             dir
                             ],
+                            environ=environ,
                             shell=True)
     else:
-      environ['PYTHON'] = 'python{}'.format(self.python_version)
+      environ['PYTHON'] = self.py_version
+      environ['SKIP_PIP_INSTALL'] = 'TRUE'
       return create_jobspec(self.name,
                             ['tools/run_tests/build_artifact_python.sh'],
                             environ=environ)
@@ -330,18 +331,23 @@ def targets():
            for Cls in (CSharpExtArtifact, NodeExtArtifact, ProtocArtifact)
            for platform in ('linux', 'macos', 'windows')
            for arch in ('x86', 'x64')] +
-          [PythonArtifact('linux', 'x86', '2.7', 'cp27-cp27m'),
-           PythonArtifact('linux', 'x86', '2.7', 'cp27-cp27mu'),
-           PythonArtifact('linux', 'x64', '2.7', 'cp27-cp27m'),
-           PythonArtifact('linux', 'x64', '2.7', 'cp27-cp27mu'),
-           PythonArtifact('macos', 'x64', '2.7'),
-           PythonArtifact('windows', 'x86', '2.7'),
-           PythonArtifact('windows', 'x64', '2.7'),
-           PythonArtifact('linux', 'x86', '3.4', 'cp34-cp34m'),
-           PythonArtifact('linux', 'x64', '3.4', 'cp34-cp34m'),
-           PythonArtifact('macos', 'x64', '3.4'),
-           PythonArtifact('windows', 'x86', '3.4'),
-           PythonArtifact('windows', 'x64', '3.4'),
+          [PythonArtifact('linux', 'x86', 'cp27-cp27m'),
+           PythonArtifact('linux', 'x86', 'cp27-cp27mu'),
+           PythonArtifact('linux', 'x86', 'cp34-cp34m'),
+           PythonArtifact('linux', 'x86', 'cp35-cp35m'),
+           PythonArtifact('linux', 'x64', 'cp27-cp27m'),
+           PythonArtifact('linux', 'x64', 'cp27-cp27mu'),
+           PythonArtifact('linux', 'x64', 'cp34-cp34m'),
+           PythonArtifact('linux', 'x64', 'cp35-cp35m'),
+           PythonArtifact('macos', 'x64', 'python2.7'),
+           PythonArtifact('macos', 'x64', 'python3.4'),
+           PythonArtifact('macos', 'x64', 'python3.5'),
+           PythonArtifact('windows', 'x86', 'Python27_32bits'),
+           PythonArtifact('windows', 'x86', 'Python34_32bits'),
+           PythonArtifact('windows', 'x86', 'Python35_32bits'),
+           PythonArtifact('windows', 'x64', 'Python27'),
+           PythonArtifact('windows', 'x64', 'Python34'),
+           PythonArtifact('windows', 'x64', 'Python35'),
            RubyArtifact('linux', 'x86'),
            RubyArtifact('linux', 'x64'),
            RubyArtifact('macos', 'x64'),
diff --git a/tools/run_tests/build_artifact_python.bat b/tools/run_tests/build_artifact_python.bat
index 074a3c6781..246713a6ce 100644
--- a/tools/run_tests/build_artifact_python.bat
+++ b/tools/run_tests/build_artifact_python.bat
@@ -36,31 +36,43 @@ pip install -rrequirements.txt
 
 set GRPC_PYTHON_BUILD_WITH_CYTHON=1
 
+@rem Multiple builds are running simultaneously, so to avoid distutils
+@rem file collisions, we build everything in a tmp directory
+if not exist "artifacts" mkdir "artifacts"
+set ARTIFACT_DIR=%cd%\artifacts
+set BUILD_DIR=C:\Windows\Temp\pygrpc-%3\
+mkdir %BUILD_DIR%
+xcopy /s/e/q %cd%\* %BUILD_DIR%
+pushd %BUILD_DIR%
+
 
 @rem Set up gRPC Python tools
 python tools\distrib\python\make_grpcio_tools.py
 
 @rem Build gRPC Python extensions
-python setup.py build_ext -c mingw32
+python setup.py build_ext -c %EXT_COMPILER% || goto :error
 
 pushd tools\distrib\python\grpcio_tools
-python setup.py build_ext -c mingw32
+python setup.py build_ext -c %EXT_COMPILER% || goto :error
 popd
 
-
 @rem Build gRPC Python distributions
-python setup.py bdist_wheel
+python setup.py bdist_wheel || goto :error
 
 pushd tools\distrib\python\grpcio_tools
-python setup.py bdist_wheel
+python setup.py bdist_wheel || goto :error
 popd
 
 
-mkdir artifacts
-xcopy /Y /I /S dist\* artifacts\ || goto :error
-xcopy /Y /I /S tools\distrib\python\grpcio_tools\dist\* artifacts\ || goto :error
+xcopy /Y /I /S dist\* %ARTIFACT_DIR% || goto :error
+xcopy /Y /I /S tools\distrib\python\grpcio_tools\dist\* %ARTIFACT_DIR% || goto :error
+
+popd
+rmdir /s /q %BUILD_DIR%
 
 goto :EOF
 
 :error
+popd
+rmdir /s /q %BUILD_DIR%
 exit /b 1
diff --git a/tools/run_tests/build_artifact_python.sh b/tools/run_tests/build_artifact_python.sh
index 8f8330ef24..9fed7c5028 100755
--- a/tools/run_tests/build_artifact_python.sh
+++ b/tools/run_tests/build_artifact_python.sh
@@ -38,38 +38,42 @@ export PYTHON=${PYTHON:-python}
 export PIP=${PIP:-pip}
 export AUDITWHEEL=${AUDITWHEEL:-auditwheel}
 
+# Because multiple builds run in parallel, some distutils file
+# operations may collide.  To avoid this, each build is run in
+# a temp directory
+mkdir -p artifacts
+ARTIFACT_DIR="$PWD/artifacts"
+BUILD_DIR=`mktemp -d "${TMPDIR:-/tmp}/pygrpc.XXXXXX"`
+trap "rm -rf $BUILD_DIR" EXIT
+cp -r * "$BUILD_DIR"
+cd "$BUILD_DIR"
 
 # Build the source distribution first because MANIFEST.in cannot override
 # exclusion of built shared objects among package resources (for some
 # inexplicable reason).
-${SETARCH_CMD} ${PYTHON} setup.py  \
-    sdist
+${SETARCH_CMD} ${PYTHON} setup.py sdist
 
 # Wheel has a bug where directories don't get excluded.
 # https://bitbucket.org/pypa/wheel/issues/99/cannot-exclude-directory
-${SETARCH_CMD} ${PYTHON} setup.py  \
-    bdist_wheel
+${SETARCH_CMD} ${PYTHON} setup.py bdist_wheel
 
 # Build gRPC tools package distribution
 ${PYTHON} tools/distrib/python/make_grpcio_tools.py
 
 # Build gRPC tools package source distribution
-${SETARCH_CMD} ${PYTHON} tools/distrib/python/grpcio_tools/setup.py  \
-    sdist
+${SETARCH_CMD} ${PYTHON} tools/distrib/python/grpcio_tools/setup.py sdist
 
 # Build gRPC tools package binary distribution
-CFLAGS="$CFLAGS -fno-wrapv" ${SETARCH_CMD} \
-  ${PYTHON} tools/distrib/python/grpcio_tools/setup.py bdist_wheel
+${SETARCH_CMD} ${PYTHON} tools/distrib/python/grpcio_tools/setup.py bdist_wheel
 
-mkdir -p artifacts
 if [ "$BUILD_MANYLINUX_WHEEL" != "" ]
 then
   for wheel in dist/*.whl; do
-    ${AUDITWHEEL} repair $wheel -w artifacts/
+    ${AUDITWHEEL} repair $wheel -w "$ARTIFACT_DIR"
     rm $wheel
   done
   for wheel in tools/distrib/python/grpcio_tools/dist/*.whl; do
-    ${AUDITWHEEL} repair $wheel -w artifacts/
+    ${AUDITWHEEL} repair $wheel -w "$ARTIFACT_DIR"
     rm $wheel
   done
 fi
@@ -81,14 +85,14 @@ fi
 if [ "$BUILD_HEALTH_CHECKING" != "" ]
 then
   ${PIP} install -rrequirements.txt
-  ${PIP} install grpcio --no-index --find-links "file://${PWD}/artifacts/"
-  ${PIP} install grpcio-tools --no-index --find-links "file://${PWD}/artifacts/"
+  ${PIP} install grpcio --no-index --find-links "file://$ARTIFACT_DIR/"
+  ${PIP} install grpcio-tools --no-index --find-links "file://$ARTIFACT_DIR/"
 
   # Build gRPC health check source distribution
   ${SETARCH_CMD} ${PYTHON} src/python/grpcio_health_checking/setup.py \
       preprocess build_package_protos sdist
-  cp -r src/python/grpcio_health_checking/dist/* artifacts
+  cp -r src/python/grpcio_health_checking/dist/* "$ARTIFACT_DIR"
 fi
 
-cp -r dist/* artifacts
-cp -r tools/distrib/python/grpcio_tools/dist/* artifacts
+cp -r dist/* "$ARTIFACT_DIR"
+cp -r tools/distrib/python/grpcio_tools/dist/* "$ARTIFACT_DIR"
diff --git a/tools/run_tests/sources_and_headers.json b/tools/run_tests/sources_and_headers.json
index e5946cc14a..11daf24e0a 100644
--- a/tools/run_tests/sources_and_headers.json
+++ b/tools/run_tests/sources_and_headers.json
@@ -4413,6 +4413,7 @@
   }, 
   {
     "deps": [
+      "gpr", 
       "grpc", 
       "grpc++_base", 
       "grpc++_codegen_base", 
@@ -4533,7 +4534,6 @@
   {
     "deps": [
       "gpr", 
-      "grpc", 
       "grpc++_base", 
       "grpc++_codegen_base", 
       "grpc++_codegen_base_src", 
@@ -5815,6 +5815,7 @@
       "include/grpc/compression.h", 
       "include/grpc/grpc.h", 
       "include/grpc/grpc_posix.h", 
+      "include/grpc/grpc_security_constants.h", 
       "include/grpc/status.h", 
       "src/core/lib/channel/channel_args.h", 
       "src/core/lib/channel/channel_stack.h", 
@@ -5905,6 +5906,7 @@
       "include/grpc/compression.h", 
       "include/grpc/grpc.h", 
       "include/grpc/grpc_posix.h", 
+      "include/grpc/grpc_security_constants.h", 
       "include/grpc/status.h", 
       "src/core/lib/channel/channel_args.c", 
       "src/core/lib/channel/channel_args.h", 
@@ -6285,7 +6287,6 @@
     ], 
     "headers": [
       "include/grpc/grpc_security.h", 
-      "include/grpc/grpc_security_constants.h", 
       "src/core/lib/security/context/security_context.h", 
       "src/core/lib/security/credentials/composite/composite_credentials.h", 
       "src/core/lib/security/credentials/credentials.h", 
@@ -6310,7 +6311,6 @@
     "name": "grpc_secure", 
     "src": [
       "include/grpc/grpc_security.h", 
-      "include/grpc/grpc_security_constants.h", 
       "src/core/lib/http/httpcli_security_connector.c", 
       "src/core/lib/security/context/security_context.c", 
       "src/core/lib/security/context/security_context.h", 
@@ -6631,8 +6631,9 @@
   }, 
   {
     "deps": [
-      "grpc", 
-      "grpc++_codegen_base"
+      "gpr", 
+      "grpc++_codegen_base", 
+      "grpc_base"
     ], 
     "headers": [
       "include/grpc++/alarm.h",