Merge remote-tracking branch 'upstream/master' into fix_mac_build

6 files changed, 204 insertions, 85 deletions

diff --git a/tools/run_tests/build_artifact_python.bat b/tools/run_tests/build_artifact_python.bat
index 295347e947..7c8c2aa12d 100644
--- a/tools/run_tests/build_artifact_python.bat
+++ b/tools/run_tests/build_artifact_python.bat
@@ -28,33 +28,24 @@
 @rem OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
-set NUGET=C:\nuget\nuget.exe
-%NUGET% restore vsprojects\grpc.sln || goto :error
-
-
-@call vsprojects\build_vs2013.bat vsprojects\grpc.sln /t:grpc_dll /p:Configuration=Release /p:PlatformToolset=v120 /p:Platform=Win32 || goto :error
-@call vsprojects\build_vs2013.bat vsprojects\grpc.sln /t:grpc_dll /p:Configuration=Release /p:PlatformToolset=v120 /p:Platform=x64   || goto :error
-
-mkdir src\python\grpcio\grpc\_cython\_windows
-
-@rem TODO(atash): maybe we could avoid the grpc_c.(32|64).python shim below if
-@rem this used the right python build?
-copy /Y vsprojects\Release\grpc_dll.dll src\python\grpcio\grpc\_cython\_windows\grpc_c.32.python || goto :error
-copy /Y vsprojects\x64\Release\grpc_dll.dll src\python\grpcio\grpc\_cython\_windows\grpc_c.64.python || goto :error
-
 set PATH=C:\%1;C:\%1\scripts;C:\msys64\mingw%2\bin;%PATH%
 
 pip install --upgrade six
 pip install --upgrade setuptools
 pip install -rrequirements.txt
 
-set GRPC_PYTHON_USE_CUSTOM_BDIST=0
-set GRPC_PYTHON_BUILD_WITH_CYTHON=1
-
 @rem Because this is windows and *everything seems to hate Windows* we have to
 @rem set all of these flags ourselves because Python won't help us (see the
 @rem setup.py of the grpcio_tools project).
 set GRPC_PYTHON_CFLAGS=-fno-wrapv -frtti -std=c++11
+
+@rem See https://sourceforge.net/p/mingw-w64/bugs/363/
+if %2 == 32 (
+  set GRPC_PYTHON_CFLAGS=%GRPC_PYTHON_CFLAGS% -D_ftime=_ftime32 -D_timeb=__timeb32 -D_ftime_s=_ftime32_s
+) else (
+  set GRPC_PYTHON_CFLAGS=%GRPC_PYTHON_CFLAGS% -D_ftime=_ftime64 -D_timeb=__timeb64
+)
+
 @rem Further confusing things, MSYS2's mingw64 tries to dynamically link
 @rem libgcc, libstdc++, and winpthreads. We have to override this or our
 @rem extensions end up linking to MSYS2 DLLs, which the normal Python on
@@ -66,23 +57,18 @@ python -c "from distutils.cygwinccompiler import get_msvcr; print(get_msvcr()[0]
 set /p PYTHON_MSVCR=<temp.txt
 set GRPC_PYTHON_LDFLAGS=-static-libgcc -static-libstdc++ -mcrtdll=%PYTHON_MSVCR% -static -lpthread
 
-
-@rem Build gRPC
-if %2 == 32 (
-  python setup.py build_ext -c mingw32
-) else (
-  python setup.py build_ext -c mingw32 -DMS_WIN64
-)
-python setup.py bdist_wheel
+set GRPC_PYTHON_BUILD_WITH_CYTHON=1
 
 
-@rem Build gRPC Python tools
+@rem Set up gRPC Python tools
 python tools\distrib\python\make_grpcio_tools.py
-if %2 == 32 (
-  python tools\distrib\python\grpcio_tools\setup.py build_ext -c mingw32
-) else (
-  python tools\distrib\python\grpcio_tools\setup.py build_ext -c mingw32 -DMS_WIN64
-)
+
+@rem Build gRPC Python extensions
+python setup.py build_ext -c mingw32
+python tools\distrib\python\grpcio_tools\setup.py build_ext -c mingw32
+
+@rem Build gRPC Python distributions
+python setup.py bdist_wheel
 python tools\distrib\python\grpcio_tools\setup.py bdist_wheel
 
 mkdir artifacts
diff --git a/tools/run_tests/build_python.sh b/tools/run_tests/build_python.sh
index 687b04e954..a3fa8200d5 100755
--- a/tools/run_tests/build_python.sh
+++ b/tools/run_tests/build_python.sh
@@ -33,25 +33,91 @@ set -ex
 # change to grpc repo root
 cd $(dirname $0)/../..
 
-# Arguments
+##########################
+# Portability operations #
+##########################
+
+PLATFORM=`uname -s`
+
+function is_mingw() {
+  if [ "${PLATFORM/MINGW}" != "$PLATFORM" ]; then
+    echo true
+  else
+    exit 1
+  fi
+}
+
+function is_darwin() {
+  if [ "${PLATFORM/Darwin}" != "$PLATFORM" ]; then
+    echo true
+  else
+    exit 1
+  fi
+}
+
+function is_linux() {
+  if [ "${PLATFORM/Linux}" != "$PLATFORM" ]; then
+    echo true
+  else
+    exit 1
+  fi
+}
+
+# Associated virtual environment name for the given python command.
+function venv() {
+  $1 -c "import sys; print('py{}{}'.format(*sys.version_info[:2]))"
+}
+
+# Path to python executable within a virtual environment depending on the
+# system.
+function venv_relative_python() {
+  if [ $(is_mingw) ]; then
+    echo 'Scripts/python.exe'
+  else
+    echo 'bin/python'
+  fi
+}
+
+# Distutils toolchain to use depending on the system.
+function toolchain() {
+  if [ $(is_mingw) ]; then
+    echo 'mingw32'
+  else
+    echo 'unix'
+  fi
+}
+
+# Command to invoke the linux command `realpath` or equivalent.
+function script_realpath() {
+  # Find `realpath`
+  if [ -x "$(command -v realpath)" ]; then
+    realpath "$@"
+  elif [ -x "$(command -v grealpath)" ]; then
+    grealpath "$@"
+  else
+    exit 1
+  fi
+}
+
+####################
+# Script Arguments #
+####################
+
 PYTHON=${1:-python2.7}
-VENV=${2:-py27}
-VENV_RELATIVE_PYTHON=${3:-bin/python}
-TOOLCHAIN=${4:-unix}
+VENV=${2:-$(venv $PYTHON)}
+VENV_RELATIVE_PYTHON=${3:-$(venv_relative_python)}
+TOOLCHAIN=${4:-$(toolchain)}
 
 ROOT=`pwd`
-export CFLAGS="-I$ROOT/include -std=gnu99 -fno-wrapv"
+export CFLAGS="-I$ROOT/include -std=gnu99 -fno-wrapv $CFLAGS"
 export GRPC_PYTHON_BUILD_WITH_CYTHON=1
 
 # Default python on the host to fall back to when instantiating e.g. the
 # virtualenv.
 HOST_PYTHON=${HOST_PYTHON:-python}
 
-# If ccache is available, use it... unless we're on Mac, then all hell breaks
-# loose because Python does hacky things to support other hacky things done to
-# hacky things on Mac OS X
-PLATFORM=`uname -s`
-if [ "${PLATFORM/Darwin}" = "$PLATFORM" ]; then
+# If ccache is available on Linux, use it.
+if [ $(is_linux) ]; then
   # We're not on Darwin (Mac OS X)
   if [ -x "$(command -v ccache)" ]; then
     if [ -x "$(command -v gcc)" ]; then
@@ -61,17 +127,24 @@ if [ "${PLATFORM/Darwin}" = "$PLATFORM" ]; then
     fi
   fi
 fi
-
-# Find `realpath`
-if [ -x "$(command -v realpath)" ]; then
-  export REALPATH=realpath
-elif [ -x "$(command -v grealpath)" ]; then
-  export REALPATH=grealpath
-else
-  echo 'Couldn'"'"'t find `realpath` or `grealpath`'
-  exit 1
+# TODO(atash) consider conceptualizing MinGW as a first-class platform and move
+# these flags into our `setup.py`s
+if [ $(is_mingw) ]; then
+  # We're on MinGW, and our CFLAGS and LDFLAGS will be eaten by the void. Use
+  # our work-around environment variables instead.
+  PYTHON_MSVCR=`$PYTHON -c "from distutils.cygwinccompiler import get_msvcr; print(get_msvcr()[0])"`
+  export GRPC_PYTHON_LDFLAGS="-static-libgcc -static-libstdc++ -mcrtdll=$PYTHON_MSVCR -static -lpthread"
+  # See https://sourceforge.net/p/mingw-w64/bugs/363/
+  export GRPC_PYTHON_CFLAGS="-D_ftime=_ftime64 -D_timeb=__timeb64"
+  # TODO(atash) set these flags for only grpcio-tools (they don't do any harm to
+  # grpcio, but they result in noisy warnings).
+  export GRPC_PYTHON_CFLAGS="-frtti -std=c++11 $GRPC_PYTHON_CFLAGS"
 fi
 
+############################
+# Perform build operations #
+############################
+
 # Instnatiate the virtualenv, preferring to do so from the relevant python
 # version. Even if these commands fail (e.g. on Windows due to name conflicts)
 # it's possible that the virtualenv is still usable and we trust the tester to
@@ -80,7 +153,7 @@ fi
 ($PYTHON -m virtualenv $VENV ||
  $HOST_PYTHON -m virtualenv -p $PYTHON $VENV ||
  true)
-VENV_PYTHON=`$REALPATH -s "$VENV/$VENV_RELATIVE_PYTHON"`
+VENV_PYTHON=`script_realpath -s "$VENV/$VENV_RELATIVE_PYTHON"`
 
 # pip-installs the directory specified. Used because on MSYS the vanilla Windows
 # Python gets confused when parsing paths.
diff --git a/tools/run_tests/build_python_msys2.sh b/tools/run_tests/build_python_msys2.sh
new file mode 100644
index 0000000000..6e9d369018
--- /dev/null
+++ b/tools/run_tests/build_python_msys2.sh
@@ -0,0 +1,36 @@
+#!/bin/bash
+# Copyright 2016, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+set -ex
+
+BUILD_PYTHON=`realpath "$(dirname $0)/build_python.sh"`
+export MSYSTEM=$1
+shift 1
+bash --login $BUILD_PYTHON "$@"
diff --git a/tools/run_tests/jobset.py b/tools/run_tests/jobset.py
index 4fe77487f9..3999537c40 100755
--- a/tools/run_tests/jobset.py
+++ b/tools/run_tests/jobset.py
@@ -47,6 +47,12 @@ measure_cpu_costs = False
 _DEFAULT_MAX_JOBS = 16 * multiprocessing.cpu_count()
 _MAX_RESULT_SIZE = 8192
 
+def sanitized_environment(env):
+  sanitized = {}
+  for key, value in env.items():
+    sanitized[str(key).encode()] = str(value).encode()
+  return sanitized
+
 def platform_string():
   if platform.system() == 'Windows':
     return 'windows'
@@ -219,6 +225,7 @@ class Job(object):
     env = dict(os.environ)
     env.update(self._spec.environ)
     env.update(self._add_env)
+    env = sanitized_environment(env)
     self._start = time.time()
     cmdline = self._spec.cmdline
     if measure_cpu_costs:
diff --git a/tools/run_tests/port_server.py b/tools/run_tests/port_server.py
index e2be26d182..83f8e6cd35 100755
--- a/tools/run_tests/port_server.py
+++ b/tools/run_tests/port_server.py
@@ -42,7 +42,7 @@ import time
 # increment this number whenever making a change to ensure that
 # the changes are picked up by running CI servers
 # note that all changes must be backwards compatible
-_MY_VERSION = 8
+_MY_VERSION = 9
 
 
 if len(sys.argv) == 2 and sys.argv[1] == 'dump_version':
@@ -110,6 +110,11 @@ keep_running = True
 
 
 class Handler(BaseHTTPServer.BaseHTTPRequestHandler):
+  
+  def setup(self):
+    # If the client is unreachable for 5 seconds, close the connection
+    self.timeout = 5
+    BaseHTTPServer.BaseHTTPRequestHandler.setup(self)
 
   def do_GET(self):
     global keep_running
diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py
index cbe17ee2ad..b0e20698bd 100755
--- a/tools/run_tests/run_tests.py
+++ b/tools/run_tests/run_tests.py
@@ -375,19 +375,15 @@ class PhpLanguage(object):
 
 
 class PythonConfig(collections.namedtuple('PythonConfig', [
-    'python', 'venv', 'venv_relative_python', 'toolchain',])):
-
-  @property
-  def venv_python(self):
-    return os.path.abspath('{}/{}'.format(self.venv, self.venv_relative_python))
-
+    'name', 'build', 'run'])):
+  """Tuple of commands (named s.t. 'what it says on the tin' applies)"""
 
 class PythonLanguage(object):
 
   def configure(self, config, args):
     self.config = config
     self.args = args
-    self.pythons = self._get_pythons(self.args.compiler)
+    self.pythons = self._get_pythons(self.args)
 
   def test_specs(self):
     # load list of known test suites
@@ -395,11 +391,11 @@ class PythonLanguage(object):
       tests_json = json.load(tests_json_file)
     environment = dict(_FORCE_ENVIRON_FOR_WRAPPERS)
     return [self.config.job_spec(
-        ['tools/run_tests/run_python.sh', config.venv_python],
+        config.run,
         timeout_seconds=5*60,
         environ=dict(environment.items() +
                      [('GRPC_PYTHON_TESTRUNNER_FILTER', suite_name)]),
-        shortname='%s.test.%s' % (config.venv, suite_name),)
+        shortname='%s.test.%s' % (config.name, suite_name),)
         for suite_name in tests_json
         for config in self.pythons]
 
@@ -413,14 +409,7 @@ class PythonLanguage(object):
     return []
 
   def build_steps(self):
-    return [
-        [
-            'tools/run_tests/build_python.sh',
-            config.python, config.venv,
-            config.venv_relative_python, config.toolchain
-        ]
-        for config in self.pythons
-    ]
+    return [config.build for config in self.pythons]
 
   def post_tests_steps(self):
     return []
@@ -431,23 +420,50 @@ class PythonLanguage(object):
   def dockerfile_dir(self):
     return 'tools/dockerfile/test/python_jessie_%s' % _docker_arch_suffix(self.args.arch)
 
-  def _get_pythons(self, compiler):
+  def _get_pythons(self, args):
+    if args.arch == 'x86':
+      bits = '32'
+    else:
+      bits = '64'
     if os.name == 'nt':
-      venv_relative_python = 'Scripts/python.exe'
-      toolchain = 'mingw32'
+      shell = ['bash']
+      builder = [os.path.abspath('tools/run_tests/build_python_msys2.sh')]
+      builder_prefix_arguments = ['MINGW{}'.format(bits)]
+      venv_relative_python = ['Scripts/python.exe']
+      toolchain = ['mingw32']
+      python_pattern_function = lambda major, minor, bits: (
+          '/c/Python{major}{minor}/python.exe'.format(major=major, minor=minor, bits=bits)
+	  if bits == '64' else
+	  '/c/Python{major}{minor}_{bits}bits/python.exe'.format(
+              major=major, minor=minor, bits=bits))
     else:
-      venv_relative_python = 'bin/python'
-      toolchain = 'unix'
-    python27_config = PythonConfig('python2.7', 'py27', venv_relative_python, toolchain)
-    python34_config = PythonConfig('python3.4', 'py34', venv_relative_python, toolchain)
-    if compiler == 'default':
-      return (python27_config, python34_config,)
-    elif compiler == 'python2.7':
+      shell = []
+      builder = [os.path.abspath('tools/run_tests/build_python.sh')]
+      builder_prefix_arguments = []
+      venv_relative_python = ['bin/python']
+      toolchain = ['unix']
+      # Bit-ness is handled by the test machine's environment
+      python_pattern_function = lambda major, minor, bits: 'python{major}.{minor}'.format(major=major, minor=minor)
+    runner = [os.path.abspath('tools/run_tests/run_python.sh')]
+    python_config_generator = lambda name, major, minor, bits: PythonConfig(
+        name,
+        shell + builder + builder_prefix_arguments
+	    + [python_pattern_function(major=major, minor=minor, bits=bits)]
+	    + [name] + venv_relative_python + toolchain,
+        shell + runner + [os.path.join(name, venv_relative_python[0])])
+    python27_config = python_config_generator(name='py27', major='2', minor='7', bits=bits)
+    python34_config = python_config_generator(name='py34', major='3', minor='4', bits=bits)
+    if args.compiler == 'default':
+      if os.name == 'nt':
+        return (python27_config,)
+      else:
+        return (python27_config, python34_config,)
+    elif args.compiler == 'python2.7':
       return (python27_config,)
-    elif compiler == 'python3.4':
+    elif args.compiler == 'python3.4':
       return (python34_config,)
     else:
-      raise Exception('Compiler %s not supported.' % compiler)
+      raise Exception('Compiler %s not supported.' % args.compiler)
 
   def __str__(self):
     return 'python'
@@ -1078,10 +1094,6 @@ def _shut_down_legacy_server(legacy_server_port):
 
 
 def _start_port_server(port_server_port):
-  # Temporary patch to switch the port_server port
-  # see https://github.com/grpc/grpc/issues/7145
-  _shut_down_legacy_server(32767)
-
   # check if a compatible port server is running
   # if incompatible (version mismatch) ==> start a new one
   # if not running ==> start a new one