Merge github.com:grpc/grpc into footprints-on-the-sands-of-time

13 files changed, 288 insertions, 68 deletions

diff --git a/tools/buildgen/generate_projects.py b/tools/buildgen/generate_projects.py
new file mode 100755
index 0000000000..1964ceb665
--- /dev/null
+++ b/tools/buildgen/generate_projects.py
@@ -0,0 +1,77 @@
+#!/usr/bin/env python2.7
+
+# Copyright 2015, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import glob
+import os
+import sys
+import tempfile
+sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '..', 'run_tests'))
+
+assert sys.argv[1:], 'run generate_projects.sh instead of this directly'
+
+import jobset
+
+os.chdir(os.path.join(os.path.dirname(sys.argv[0]), '..', '..'))
+json = sys.argv[1:]
+
+test = {} if 'TEST' in os.environ else None
+
+plugins = sorted(glob.glob('tools/buildgen/plugins/*.py'))
+
+jobs = []
+for root, dirs, files in os.walk('templates'):
+	for f in files:
+		if os.path.splitext(f)[1] == '.template':
+			out = '.' + root[len('templates'):] + '/' + os.path.splitext(f)[0]
+			cmd = ['tools/buildgen/mako_renderer.py']
+			for plugin in plugins:
+				cmd.append('-p')
+				cmd.append(plugin)
+			for js in json:
+				cmd.append('-d')
+				cmd.append(js)
+			cmd.append('-o')
+			if test is None:
+				cmd.append(out)
+			else:
+				tf = tempfile.mkstemp()
+				test[out] = tf[1]
+				os.close(tf[0])
+				cmd.append(test[out])
+			cmd.append(root + '/' + f)
+			jobs.append(jobset.JobSpec(cmd, shortname=out))
+
+jobset.run(jobs)
+
+if test is not None:
+	for s, g in test.iteritems():
+		assert(0 == os.system('diff %s %s' % (s, g)))
+		os.unlink(g)
diff --git a/tools/buildgen/generate_projects.sh b/tools/buildgen/generate_projects.sh
index 5399867746..32fc90fef5 100755
--- a/tools/buildgen/generate_projects.sh
+++ b/tools/buildgen/generate_projects.sh
@@ -45,32 +45,6 @@ fi
 
 . tools/buildgen/generate_build_additions.sh
 
-global_plugins=`find ./tools/buildgen/plugins -name '*.py' |
-  sort | grep -v __init__ | awk ' { printf "-p %s ", $0 } '`
-
-for dir in . ; do
-  local_plugins=`find $dir/templates -name '*.py' |
-    sort | grep -v __init__ | awk ' { printf "-p %s ", $0 } '`
-
-  plugins="$global_plugins $local_plugins"
-
-  find -L $dir/templates -type f -and -name *.template | while read file ; do
-    out=${dir}/${file#$dir/templates/}  # strip templates dir prefix
-    out=${out%.*}  # strip template extension
-    echo "generating file: $out"
-    json_files="build.json $gen_build_files"
-    data=`for i in $json_files ; do echo $i ; done | awk ' { printf "-d %s ", $0 } '`
-    if [ "x$TEST" = "xtrue" ] ; then
-      actual_out=$out
-      out=`mktemp /tmp/gentXXXXXX`
-    fi
-    mkdir -p `dirname $out`  # make sure dest directory exist
-    $mako_renderer $plugins $data -o $out $file
-    if [ "x$TEST" = "xtrue" ] ; then
-      diff -q $out $actual_out
-      rm $out
-    fi
-  done
-done
+tools/buildgen/generate_projects.py build.json $gen_build_files
 
 rm $gen_build_files
diff --git a/tools/distrib/python/submit.py b/tools/distrib/python/submit.py
index dd48f440ba..a3615b3640 100755
--- a/tools/distrib/python/submit.py
+++ b/tools/distrib/python/submit.py
@@ -66,6 +66,12 @@ try:
 except:
   pass
 
+# Build the Cython C files
+build_env = os.environ.copy()
+build_env['GRPC_PYTHON_BUILD_WITH_CYTHON'] = "1"
+cmd = ['python', 'setup.py', 'build_ext', '--inplace']
+subprocess.call(cmd, cwd=pkgdir, env=build_env)
+
 # Make the push.
 cmd = ['python', 'setup.py', 'sdist']
 subprocess.call(cmd, cwd=pkgdir)
diff --git a/tools/doxygen/Doxyfile.c++ b/tools/doxygen/Doxyfile.c++
index d782dc18f7..feb7ad8bb9 100644
--- a/tools/doxygen/Doxyfile.c++
+++ b/tools/doxygen/Doxyfile.c++
@@ -762,6 +762,7 @@ WARN_LOGFILE           =
 
 INPUT                  = include/grpc++/async_generic_service.h \
 include/grpc++/async_unary_call.h \
+include/grpc++/auth_context.h \
 include/grpc++/byte_buffer.h \
 include/grpc++/channel_arguments.h \
 include/grpc++/channel_interface.h \
diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal
index 575cd95753..67718d8976 100644
--- a/tools/doxygen/Doxyfile.c++.internal
+++ b/tools/doxygen/Doxyfile.c++.internal
@@ -762,6 +762,7 @@ WARN_LOGFILE           =
 
 INPUT                  = include/grpc++/async_generic_service.h \
 include/grpc++/async_unary_call.h \
+include/grpc++/auth_context.h \
 include/grpc++/byte_buffer.h \
 include/grpc++/channel_arguments.h \
 include/grpc++/channel_interface.h \
@@ -798,11 +799,15 @@ include/grpc++/stream.h \
 include/grpc++/thread_pool_interface.h \
 include/grpc++/time.h \
 src/cpp/client/secure_credentials.h \
+src/cpp/common/secure_auth_context.h \
 src/cpp/server/secure_server_credentials.h \
 src/cpp/client/channel.h \
+src/cpp/common/create_auth_context.h \
 src/cpp/server/thread_pool.h \
 src/cpp/client/secure_channel_arguments.cc \
 src/cpp/client/secure_credentials.cc \
+src/cpp/common/secure_auth_context.cc \
+src/cpp/common/secure_create_auth_context.cc \
 src/cpp/server/secure_server_credentials.cc \
 src/cpp/client/channel.cc \
 src/cpp/client/channel_arguments.cc \
diff --git a/tools/run_tests/build_python.sh b/tools/run_tests/build_python.sh
index d9b7644f44..ae0fb42241 100755
--- a/tools/run_tests/build_python.sh
+++ b/tools/run_tests/build_python.sh
@@ -34,9 +34,32 @@ set -ex
 cd $(dirname $0)/../..
 
 root=`pwd`
-rm -rf python2.7_virtual_environment
-virtualenv -p /usr/bin/python2.7 python2.7_virtual_environment
-source python2.7_virtual_environment/bin/activate
-pip install -r src/python/requirements.txt
-CFLAGS="-I$root/include -std=c89 -Werror" LDFLAGS=-L$root/libs/$CONFIG pip install src/python/src
-pip install src/python/interop
+
+make_virtualenv() {
+  virtualenv_name="python"$1"_virtual_environment"
+  if [ ! -d $virtualenv_name ]
+  then
+    # Build the entire virtual environment
+    virtualenv -p `which "python"$1` $virtualenv_name
+    source $virtualenv_name/bin/activate
+    pip install -r src/python/requirements.txt
+    CFLAGS="-I$root/include -std=c89" LDFLAGS=-L$root/libs/$CONFIG GRPC_PYTHON_BUILD_WITH_CYTHON=1 pip install src/python/src
+    pip install src/python/interop
+  else
+    source $virtualenv_name/bin/activate
+    # Uninstall and re-install the packages we care about. Don't use
+    # --force-reinstall or --ignore-installed to avoid propagating this
+    # unnecessarily to dependencies. Don't use --no-deps to avoid missing
+    # dependency upgrades.
+    (yes | pip uninstall grpcio) || true
+    (yes | pip uninstall interop) || true
+    (CFLAGS="-I$root/include -std=c89" LDFLAGS=-L$root/libs/$CONFIG GRPC_PYTHON_BUILD_WITH_CYTHON=1 pip install src/python/src) || (
+      # Fall back to rebuilding the entire environment
+      rm -rf $virtualenv_name
+      make_virtualenv $1
+    )
+    pip install src/python/interop
+  fi
+}
+
+make_virtualenv $1
diff --git a/tools/run_tests/jobset.py b/tools/run_tests/jobset.py
index baa126ba5f..e5138147d3 100755
--- a/tools/run_tests/jobset.py
+++ b/tools/run_tests/jobset.py
@@ -81,6 +81,7 @@ _CLEAR_LINE = '\x1b[2K'
 
 _TAG_COLOR = {
     'FAILED': 'red',
+    'WARNING': 'yellow',
     'TIMEOUT': 'red',
     'PASSED': 'green',
     'START': 'gray',
diff --git a/tools/run_tests/python_tests.json b/tools/run_tests/python_tests.json
index 6c969d765f..4da4c1b68b 100755
--- a/tools/run_tests/python_tests.json
+++ b/tools/run_tests/python_tests.json
@@ -1,56 +1,123 @@
 [
   {
-    "module": "grpc._adapter._c_test"
+    "module": "grpc._cython.cygrpc_test",
+    "pythonVersions": [
+      "2.7",
+      "3.4"
+    ]
   },
   {
-    "module": "grpc._adapter._low_test"
+    "module": "grpc._cython.adapter_low_test",
+    "pythonVersions": [
+      "2.7"
+    ]
   },
   {
-    "module": "grpc._adapter._intermediary_low_test"
+    "module": "grpc._adapter._c_test",
+    "pythonVersions": [
+      "2.7"
+    ]
   },
   {
-    "module": "grpc._adapter._links_test"
+    "module": "grpc._adapter._low_test",
+    "pythonVersions": [
+      "2.7"
+    ]
   },
   {
-    "module": "grpc._adapter._lonely_rear_link_test"
+    "module": "grpc._adapter._intermediary_low_test",
+    "pythonVersions": [
+      "2.7"
+    ]
   },
   {
-    "module": "grpc._adapter._blocking_invocation_inline_service_test"
+    "module": "grpc._adapter._links_test",
+    "pythonVersions": [
+      "2.7"
+    ]
   },
   {
-    "module": "grpc._adapter._event_invocation_synchronous_event_service_test"
+    "module": "grpc._adapter._lonely_rear_link_test",
+    "pythonVersions": [
+      "2.7"
+    ]
   },
   {
-    "module": "grpc._adapter._future_invocation_asynchronous_event_service_test"
+    "module": "grpc._adapter._blocking_invocation_inline_service_test",
+    "pythonVersions": [
+      "2.7"
+    ]
   },
   {
-    "module": "grpc.early_adopter.implementations_test"
+    "module": "grpc._adapter._event_invocation_synchronous_event_service_test",
+    "pythonVersions": [
+      "2.7"
+    ]
   },
   {
-    "module": "grpc.framework.base.implementations_test"
+    "module": "grpc._adapter._future_invocation_asynchronous_event_service_test",
+    "pythonVersions": [
+      "2.7"
+    ]
   },
   {
-    "module": "grpc.framework.face.blocking_invocation_inline_service_test"
+    "module": "grpc.early_adopter.implementations_test",
+    "pythonVersions": [
+      "2.7"
+    ]
   },
   {
-    "module": "grpc.framework.face.event_invocation_synchronous_event_service_test"
+    "module": "grpc.framework.base.implementations_test",
+    "pythonVersions": [
+      "2.7"
+    ]
   },
   {
-    "module": "grpc.framework.face.future_invocation_asynchronous_event_service_test"
+    "module": "grpc.framework.face.blocking_invocation_inline_service_test",
+    "pythonVersions": [
+      "2.7"
+    ]
   },
   {
-    "module": "grpc.framework.foundation._later_test"
+    "module": "grpc.framework.face.event_invocation_synchronous_event_service_test",
+    "pythonVersions": [
+      "2.7"
+    ]
   },
   {
-    "module": "grpc.framework.foundation._logging_pool_test"
+    "module": "grpc.framework.face.future_invocation_asynchronous_event_service_test",
+    "pythonVersions": [
+      "2.7"
+    ]
   },
   {
-    "module": "interop._insecure_interop_test"
+    "module": "grpc.framework.foundation._later_test",
+    "pythonVersions": [
+      "2.7"
+    ]
   },
   {
-    "module": "interop._secure_interop_test"
+    "module": "grpc.framework.foundation._logging_pool_test",
+    "pythonVersions": [
+      "2.7"
+    ]
   },
   {
-    "file": "test/compiler/python_plugin_test.py"
+    "module": "interop._insecure_interop_test",
+    "pythonVersions": [
+      "2.7"
+    ]
+  },
+  {
+    "module": "interop._secure_interop_test",
+    "pythonVersions": [
+      "2.7"
+    ]
+  },
+  {
+    "file": "test/compiler/python_plugin_test.py",
+    "pythonVersions": [
+      "2.7"
+    ]
   }
 ]
diff --git a/tools/run_tests/run_csharp.bat b/tools/run_tests/run_csharp.bat
index 17c622cc2d..c86136767c 100644
--- a/tools/run_tests/run_csharp.bat
+++ b/tools/run_tests/run_csharp.bat
@@ -5,7 +5,10 @@ setlocal
 @rem enter this directory
 cd /d %~dp0\..\..\src\csharp
 
-packages\NUnit.Runners.2.6.4\tools\nunit-console-x86.exe -labels "%1/bin/Debug/%1.dll" || goto :error
+@rem set UUID variable to a random GUID, we will use it to put TestResults.xml to a dedicated directory, so that parallel test runs don't collide
+for /F %%i in ('powershell -Command "[guid]::NewGuid().ToString()"') do (set UUID=%%i)
+
+packages\NUnit.Runners.2.6.4\tools\nunit-console-x86.exe -labels "%1/bin/Debug/%1.dll" -work test-results/%UUID% || goto :error
 
 endlocal
 
diff --git a/tools/run_tests/run_python.sh b/tools/run_tests/run_python.sh
index cab08f9358..4959c0241c 100755
--- a/tools/run_tests/run_python.sh
+++ b/tools/run_tests/run_python.sh
@@ -36,5 +36,5 @@ cd $(dirname $0)/../..
 root=`pwd`
 export LD_LIBRARY_PATH=$root/libs/$CONFIG
 export DYLD_LIBRARY_PATH=$root/libs/$CONFIG
-source python2.7_virtual_environment/bin/activate
-python2.7 -B $*
+source "python"$PYVER"_virtual_environment"/bin/activate
+"python"$PYVER -B $*
diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py
index fd90613ad6..1f44fc34fa 100755
--- a/tools/run_tests/run_tests.py
+++ b/tools/run_tests/run_tests.py
@@ -189,27 +189,55 @@ class PythonLanguage(object):
   def __init__(self):
     with open('tools/run_tests/python_tests.json') as f:
       self._tests = json.load(f)
+    self._build_python_versions = set([
+        python_version
+        for test in self._tests
+        for python_version in test['pythonVersions']])
+    self._has_python_versions = []
 
   def test_specs(self, config, travis):
-    modules = [config.job_spec(['tools/run_tests/run_python.sh', '-m',
-                                test['module']],
-                               None,
-                               environ=_FORCE_ENVIRON_FOR_WRAPPERS,
-                               shortname=test['module'])
-               for test in self._tests if 'module' in test]
-    files = [config.job_spec(['tools/run_tests/run_python.sh',
-                              test['file']],
-                             None,
-                             environ=_FORCE_ENVIRON_FOR_WRAPPERS,
-                             shortname=test['file'])
-            for test in self._tests if 'file' in test]
-    return files + modules
+    job_specifications = []
+    for test in self._tests:
+      command = None
+      short_name = None
+      if 'module' in test:
+        command = ['tools/run_tests/run_python.sh', '-m', test['module']]
+        short_name = test['module']
+      elif 'file' in test:
+        command = ['tools/run_tests/run_python.sh', test['file']]
+        short_name = test['file']
+      else:
+        raise ValueError('expected input to be a module or file to run '
+                         'unittests from')
+      for python_version in test['pythonVersions']:
+        if python_version in self._has_python_versions:
+          environment = dict(_FORCE_ENVIRON_FOR_WRAPPERS)
+          environment['PYVER'] = python_version
+          job_specifications.append(config.job_spec(
+              command, None, environ=environment, shortname=short_name))
+        else:
+          jobset.message(
+              'WARNING',
+              'Could not find Python {}; skipping test'.format(python_version),
+              '{}\n'.format(command), do_newline=True)
+    return job_specifications
 
   def make_targets(self):
     return ['static_c', 'grpc_python_plugin', 'shared_c']
 
   def build_steps(self):
-    return [['tools/run_tests/build_python.sh']]
+    commands = []
+    for python_version in self._build_python_versions:
+      try:
+        with open(os.devnull, 'w') as output:
+          subprocess.check_call(['which', 'python' + python_version],
+                                stdout=output, stderr=output)
+        commands.append(['tools/run_tests/build_python.sh', python_version])
+        self._has_python_versions.append(python_version)
+      except:
+        jobset.message('WARNING', 'Missing Python ' + python_version,
+                       do_newline=True)
+    return commands
 
   def supports_multi_config(self):
     return False
diff --git a/tools/run_tests/sources_and_headers.json b/tools/run_tests/sources_and_headers.json
index 3f13b7b9ad..1fc5c20fe4 100644
--- a/tools/run_tests/sources_and_headers.json
+++ b/tools/run_tests/sources_and_headers.json
@@ -1486,6 +1486,19 @@
   {
     "deps": [
       "gpr", 
+      "grpc", 
+      "grpc++"
+    ], 
+    "headers": [], 
+    "language": "c++", 
+    "name": "secure_auth_context_test", 
+    "src": [
+      "test/cpp/common/secure_auth_context_test.cc"
+    ]
+  }, 
+  {
+    "deps": [
+      "gpr", 
       "gpr_test_util", 
       "grpc", 
       "grpc++", 
@@ -9475,6 +9488,7 @@
     "headers": [
       "include/grpc++/async_generic_service.h", 
       "include/grpc++/async_unary_call.h", 
+      "include/grpc++/auth_context.h", 
       "include/grpc++/byte_buffer.h", 
       "include/grpc++/channel_arguments.h", 
       "include/grpc++/channel_interface.h", 
@@ -9512,6 +9526,8 @@
       "include/grpc++/time.h", 
       "src/cpp/client/channel.h", 
       "src/cpp/client/secure_credentials.h", 
+      "src/cpp/common/create_auth_context.h", 
+      "src/cpp/common/secure_auth_context.h", 
       "src/cpp/server/secure_server_credentials.h", 
       "src/cpp/server/thread_pool.h"
     ], 
@@ -9520,6 +9536,7 @@
     "src": [
       "include/grpc++/async_generic_service.h", 
       "include/grpc++/async_unary_call.h", 
+      "include/grpc++/auth_context.h", 
       "include/grpc++/byte_buffer.h", 
       "include/grpc++/channel_arguments.h", 
       "include/grpc++/channel_interface.h", 
@@ -9569,7 +9586,11 @@
       "src/cpp/client/secure_credentials.h", 
       "src/cpp/common/call.cc", 
       "src/cpp/common/completion_queue.cc", 
+      "src/cpp/common/create_auth_context.h", 
       "src/cpp/common/rpc_method.cc", 
+      "src/cpp/common/secure_auth_context.cc", 
+      "src/cpp/common/secure_auth_context.h", 
+      "src/cpp/common/secure_create_auth_context.cc", 
       "src/cpp/proto/proto_utils.cc", 
       "src/cpp/server/async_generic_service.cc", 
       "src/cpp/server/create_default_thread_pool.cc", 
@@ -9638,6 +9659,7 @@
     "headers": [
       "include/grpc++/async_generic_service.h", 
       "include/grpc++/async_unary_call.h", 
+      "include/grpc++/auth_context.h", 
       "include/grpc++/byte_buffer.h", 
       "include/grpc++/channel_arguments.h", 
       "include/grpc++/channel_interface.h", 
@@ -9674,6 +9696,7 @@
       "include/grpc++/thread_pool_interface.h", 
       "include/grpc++/time.h", 
       "src/cpp/client/channel.h", 
+      "src/cpp/common/create_auth_context.h", 
       "src/cpp/server/thread_pool.h"
     ], 
     "language": "c++", 
@@ -9681,6 +9704,7 @@
     "src": [
       "include/grpc++/async_generic_service.h", 
       "include/grpc++/async_unary_call.h", 
+      "include/grpc++/auth_context.h", 
       "include/grpc++/byte_buffer.h", 
       "include/grpc++/channel_arguments.h", 
       "include/grpc++/channel_interface.h", 
@@ -9727,6 +9751,8 @@
       "src/cpp/client/internal_stub.cc", 
       "src/cpp/common/call.cc", 
       "src/cpp/common/completion_queue.cc", 
+      "src/cpp/common/create_auth_context.h", 
+      "src/cpp/common/insecure_create_auth_context.cc", 
       "src/cpp/common/rpc_method.cc", 
       "src/cpp/proto/proto_utils.cc", 
       "src/cpp/server/async_generic_service.cc", 
diff --git a/tools/run_tests/tests.json b/tools/run_tests/tests.json
index 12b1f22b12..cc05870640 100644
--- a/tools/run_tests/tests.json
+++ b/tools/run_tests/tests.json
@@ -723,6 +723,15 @@
   {
     "flaky": false, 
     "language": "c++", 
+    "name": "secure_auth_context_test", 
+    "platforms": [
+      "windows", 
+      "posix"
+    ]
+  }, 
+  {
+    "flaky": false, 
+    "language": "c++", 
     "name": "server_crash_test", 
     "platforms": [
       "windows",