Merge branch 'master' of github.com:grpc/grpc into flexible_default_creds

9 files changed, 240 insertions, 43 deletions

diff --git a/tools/run_tests/build_python.sh b/tools/run_tests/build_python.sh
index d9b7644f44..ae0fb42241 100755
--- a/tools/run_tests/build_python.sh
+++ b/tools/run_tests/build_python.sh
@@ -34,9 +34,32 @@ set -ex
 cd $(dirname $0)/../..
 
 root=`pwd`
-rm -rf python2.7_virtual_environment
-virtualenv -p /usr/bin/python2.7 python2.7_virtual_environment
-source python2.7_virtual_environment/bin/activate
-pip install -r src/python/requirements.txt
-CFLAGS="-I$root/include -std=c89 -Werror" LDFLAGS=-L$root/libs/$CONFIG pip install src/python/src
-pip install src/python/interop
+
+make_virtualenv() {
+  virtualenv_name="python"$1"_virtual_environment"
+  if [ ! -d $virtualenv_name ]
+  then
+    # Build the entire virtual environment
+    virtualenv -p `which "python"$1` $virtualenv_name
+    source $virtualenv_name/bin/activate
+    pip install -r src/python/requirements.txt
+    CFLAGS="-I$root/include -std=c89" LDFLAGS=-L$root/libs/$CONFIG GRPC_PYTHON_BUILD_WITH_CYTHON=1 pip install src/python/src
+    pip install src/python/interop
+  else
+    source $virtualenv_name/bin/activate
+    # Uninstall and re-install the packages we care about. Don't use
+    # --force-reinstall or --ignore-installed to avoid propagating this
+    # unnecessarily to dependencies. Don't use --no-deps to avoid missing
+    # dependency upgrades.
+    (yes | pip uninstall grpcio) || true
+    (yes | pip uninstall interop) || true
+    (CFLAGS="-I$root/include -std=c89" LDFLAGS=-L$root/libs/$CONFIG GRPC_PYTHON_BUILD_WITH_CYTHON=1 pip install src/python/src) || (
+      # Fall back to rebuilding the entire environment
+      rm -rf $virtualenv_name
+      make_virtualenv $1
+    )
+    pip install src/python/interop
+  fi
+}
+
+make_virtualenv $1
diff --git a/tools/run_tests/build_ruby.sh b/tools/run_tests/build_ruby.sh
index de96413bc1..b6c4e32b7e 100755
--- a/tools/run_tests/build_ruby.sh
+++ b/tools/run_tests/build_ruby.sh
@@ -31,7 +31,7 @@
 
 set -ex
 
-export CONFIG=${CONFIG:-opt}
+export GRPC_CONFIG=${CONFIG:-opt}
 
 # change to grpc's ruby directory
 cd $(dirname $0)/../../src/ruby
diff --git a/tools/run_tests/jobset.py b/tools/run_tests/jobset.py
index 8694b8f6bd..e5138147d3 100755
--- a/tools/run_tests/jobset.py
+++ b/tools/run_tests/jobset.py
@@ -81,6 +81,7 @@ _CLEAR_LINE = '\x1b[2K'
 
 _TAG_COLOR = {
     'FAILED': 'red',
+    'WARNING': 'yellow',
     'TIMEOUT': 'red',
     'PASSED': 'green',
     'START': 'gray',
@@ -95,7 +96,7 @@ def message(tag, msg, explanatory_text=None, do_newline=False):
     return
   message.old_tag = tag
   message.old_msg = msg
-  if platform.system() == 'Windows':
+  if platform.system() == 'Windows' or not sys.stdout.isatty():
     if explanatory_text:
       print explanatory_text
     print '%s: %s' % (tag, msg)
diff --git a/tools/run_tests/python_tests.json b/tools/run_tests/python_tests.json
index 6c969d765f..4da4c1b68b 100755
--- a/tools/run_tests/python_tests.json
+++ b/tools/run_tests/python_tests.json
@@ -1,56 +1,123 @@
 [
   {
-    "module": "grpc._adapter._c_test"
+    "module": "grpc._cython.cygrpc_test",
+    "pythonVersions": [
+      "2.7",
+      "3.4"
+    ]
   },
   {
-    "module": "grpc._adapter._low_test"
+    "module": "grpc._cython.adapter_low_test",
+    "pythonVersions": [
+      "2.7"
+    ]
   },
   {
-    "module": "grpc._adapter._intermediary_low_test"
+    "module": "grpc._adapter._c_test",
+    "pythonVersions": [
+      "2.7"
+    ]
   },
   {
-    "module": "grpc._adapter._links_test"
+    "module": "grpc._adapter._low_test",
+    "pythonVersions": [
+      "2.7"
+    ]
   },
   {
-    "module": "grpc._adapter._lonely_rear_link_test"
+    "module": "grpc._adapter._intermediary_low_test",
+    "pythonVersions": [
+      "2.7"
+    ]
   },
   {
-    "module": "grpc._adapter._blocking_invocation_inline_service_test"
+    "module": "grpc._adapter._links_test",
+    "pythonVersions": [
+      "2.7"
+    ]
   },
   {
-    "module": "grpc._adapter._event_invocation_synchronous_event_service_test"
+    "module": "grpc._adapter._lonely_rear_link_test",
+    "pythonVersions": [
+      "2.7"
+    ]
   },
   {
-    "module": "grpc._adapter._future_invocation_asynchronous_event_service_test"
+    "module": "grpc._adapter._blocking_invocation_inline_service_test",
+    "pythonVersions": [
+      "2.7"
+    ]
   },
   {
-    "module": "grpc.early_adopter.implementations_test"
+    "module": "grpc._adapter._event_invocation_synchronous_event_service_test",
+    "pythonVersions": [
+      "2.7"
+    ]
   },
   {
-    "module": "grpc.framework.base.implementations_test"
+    "module": "grpc._adapter._future_invocation_asynchronous_event_service_test",
+    "pythonVersions": [
+      "2.7"
+    ]
   },
   {
-    "module": "grpc.framework.face.blocking_invocation_inline_service_test"
+    "module": "grpc.early_adopter.implementations_test",
+    "pythonVersions": [
+      "2.7"
+    ]
   },
   {
-    "module": "grpc.framework.face.event_invocation_synchronous_event_service_test"
+    "module": "grpc.framework.base.implementations_test",
+    "pythonVersions": [
+      "2.7"
+    ]
   },
   {
-    "module": "grpc.framework.face.future_invocation_asynchronous_event_service_test"
+    "module": "grpc.framework.face.blocking_invocation_inline_service_test",
+    "pythonVersions": [
+      "2.7"
+    ]
   },
   {
-    "module": "grpc.framework.foundation._later_test"
+    "module": "grpc.framework.face.event_invocation_synchronous_event_service_test",
+    "pythonVersions": [
+      "2.7"
+    ]
   },
   {
-    "module": "grpc.framework.foundation._logging_pool_test"
+    "module": "grpc.framework.face.future_invocation_asynchronous_event_service_test",
+    "pythonVersions": [
+      "2.7"
+    ]
   },
   {
-    "module": "interop._insecure_interop_test"
+    "module": "grpc.framework.foundation._later_test",
+    "pythonVersions": [
+      "2.7"
+    ]
   },
   {
-    "module": "interop._secure_interop_test"
+    "module": "grpc.framework.foundation._logging_pool_test",
+    "pythonVersions": [
+      "2.7"
+    ]
   },
   {
-    "file": "test/compiler/python_plugin_test.py"
+    "module": "interop._insecure_interop_test",
+    "pythonVersions": [
+      "2.7"
+    ]
+  },
+  {
+    "module": "interop._secure_interop_test",
+    "pythonVersions": [
+      "2.7"
+    ]
+  },
+  {
+    "file": "test/compiler/python_plugin_test.py",
+    "pythonVersions": [
+      "2.7"
+    ]
   }
 ]
diff --git a/tools/run_tests/run_csharp.bat b/tools/run_tests/run_csharp.bat
index 17c622cc2d..c86136767c 100644
--- a/tools/run_tests/run_csharp.bat
+++ b/tools/run_tests/run_csharp.bat
@@ -5,7 +5,10 @@ setlocal
 @rem enter this directory
 cd /d %~dp0\..\..\src\csharp
 
-packages\NUnit.Runners.2.6.4\tools\nunit-console-x86.exe -labels "%1/bin/Debug/%1.dll" || goto :error
+@rem set UUID variable to a random GUID, we will use it to put TestResults.xml to a dedicated directory, so that parallel test runs don't collide
+for /F %%i in ('powershell -Command "[guid]::NewGuid().ToString()"') do (set UUID=%%i)
+
+packages\NUnit.Runners.2.6.4\tools\nunit-console-x86.exe -labels "%1/bin/Debug/%1.dll" -work test-results/%UUID% || goto :error
 
 endlocal
 
diff --git a/tools/run_tests/run_python.sh b/tools/run_tests/run_python.sh
index cab08f9358..4959c0241c 100755
--- a/tools/run_tests/run_python.sh
+++ b/tools/run_tests/run_python.sh
@@ -36,5 +36,5 @@ cd $(dirname $0)/../..
 root=`pwd`
 export LD_LIBRARY_PATH=$root/libs/$CONFIG
 export DYLD_LIBRARY_PATH=$root/libs/$CONFIG
-source python2.7_virtual_environment/bin/activate
-python2.7 -B $*
+source "python"$PYVER"_virtual_environment"/bin/activate
+"python"$PYVER -B $*
diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py
index fd90613ad6..1f44fc34fa 100755
--- a/tools/run_tests/run_tests.py
+++ b/tools/run_tests/run_tests.py
@@ -189,27 +189,55 @@ class PythonLanguage(object):
   def __init__(self):
     with open('tools/run_tests/python_tests.json') as f:
       self._tests = json.load(f)
+    self._build_python_versions = set([
+        python_version
+        for test in self._tests
+        for python_version in test['pythonVersions']])
+    self._has_python_versions = []
 
   def test_specs(self, config, travis):
-    modules = [config.job_spec(['tools/run_tests/run_python.sh', '-m',
-                                test['module']],
-                               None,
-                               environ=_FORCE_ENVIRON_FOR_WRAPPERS,
-                               shortname=test['module'])
-               for test in self._tests if 'module' in test]
-    files = [config.job_spec(['tools/run_tests/run_python.sh',
-                              test['file']],
-                             None,
-                             environ=_FORCE_ENVIRON_FOR_WRAPPERS,
-                             shortname=test['file'])
-            for test in self._tests if 'file' in test]
-    return files + modules
+    job_specifications = []
+    for test in self._tests:
+      command = None
+      short_name = None
+      if 'module' in test:
+        command = ['tools/run_tests/run_python.sh', '-m', test['module']]
+        short_name = test['module']
+      elif 'file' in test:
+        command = ['tools/run_tests/run_python.sh', test['file']]
+        short_name = test['file']
+      else:
+        raise ValueError('expected input to be a module or file to run '
+                         'unittests from')
+      for python_version in test['pythonVersions']:
+        if python_version in self._has_python_versions:
+          environment = dict(_FORCE_ENVIRON_FOR_WRAPPERS)
+          environment['PYVER'] = python_version
+          job_specifications.append(config.job_spec(
+              command, None, environ=environment, shortname=short_name))
+        else:
+          jobset.message(
+              'WARNING',
+              'Could not find Python {}; skipping test'.format(python_version),
+              '{}\n'.format(command), do_newline=True)
+    return job_specifications
 
   def make_targets(self):
     return ['static_c', 'grpc_python_plugin', 'shared_c']
 
   def build_steps(self):
-    return [['tools/run_tests/build_python.sh']]
+    commands = []
+    for python_version in self._build_python_versions:
+      try:
+        with open(os.devnull, 'w') as output:
+          subprocess.check_call(['which', 'python' + python_version],
+                                stdout=output, stderr=output)
+        commands.append(['tools/run_tests/build_python.sh', python_version])
+        self._has_python_versions.append(python_version)
+      except:
+        jobset.message('WARNING', 'Missing Python ' + python_version,
+                       do_newline=True)
+    return commands
 
   def supports_multi_config(self):
     return False
diff --git a/tools/run_tests/sources_and_headers.json b/tools/run_tests/sources_and_headers.json
index b228aaa66c..1fc5c20fe4 100644
--- a/tools/run_tests/sources_and_headers.json
+++ b/tools/run_tests/sources_and_headers.json
@@ -538,6 +538,20 @@
     ], 
     "headers": [], 
     "language": "c", 
+    "name": "grpc_jwt_verifier_test", 
+    "src": [
+      "test/core/security/jwt_verifier_test.c"
+    ]
+  }, 
+  {
+    "deps": [
+      "gpr", 
+      "gpr_test_util", 
+      "grpc", 
+      "grpc_test_util"
+    ], 
+    "headers": [], 
+    "language": "c", 
     "name": "grpc_print_google_default_creds_token", 
     "src": [
       "test/core/security/print_google_default_creds_token.c"
@@ -580,6 +594,20 @@
     ], 
     "headers": [], 
     "language": "c", 
+    "name": "grpc_verify_jwt", 
+    "src": [
+      "test/core/security/verify_jwt.c"
+    ]
+  }, 
+  {
+    "deps": [
+      "gpr", 
+      "gpr_test_util", 
+      "grpc", 
+      "grpc_test_util"
+    ], 
+    "headers": [], 
+    "language": "c", 
     "name": "hpack_parser_test", 
     "src": [
       "test/core/transport/chttp2/hpack_parser_test.c"
@@ -1458,6 +1486,19 @@
   {
     "deps": [
       "gpr", 
+      "grpc", 
+      "grpc++"
+    ], 
+    "headers": [], 
+    "language": "c++", 
+    "name": "secure_auth_context_test", 
+    "src": [
+      "test/cpp/common/secure_auth_context_test.cc"
+    ]
+  }, 
+  {
+    "deps": [
+      "gpr", 
       "gpr_test_util", 
       "grpc", 
       "grpc++", 
@@ -8743,6 +8784,7 @@
       "src/core/security/base64.h", 
       "src/core/security/credentials.h", 
       "src/core/security/json_token.h", 
+      "src/core/security/jwt_verifier.h", 
       "src/core/security/secure_endpoint.h", 
       "src/core/security/secure_transport_setup.h", 
       "src/core/security/security_connector.h", 
@@ -8944,6 +8986,8 @@
       "src/core/security/google_default_credentials.c", 
       "src/core/security/json_token.c", 
       "src/core/security/json_token.h", 
+      "src/core/security/jwt_verifier.c", 
+      "src/core/security/jwt_verifier.h", 
       "src/core/security/secure_endpoint.c", 
       "src/core/security/secure_endpoint.h", 
       "src/core/security/secure_transport_setup.c", 
@@ -9444,6 +9488,7 @@
     "headers": [
       "include/grpc++/async_generic_service.h", 
       "include/grpc++/async_unary_call.h", 
+      "include/grpc++/auth_context.h", 
       "include/grpc++/byte_buffer.h", 
       "include/grpc++/channel_arguments.h", 
       "include/grpc++/channel_interface.h", 
@@ -9481,6 +9526,8 @@
       "include/grpc++/time.h", 
       "src/cpp/client/channel.h", 
       "src/cpp/client/secure_credentials.h", 
+      "src/cpp/common/create_auth_context.h", 
+      "src/cpp/common/secure_auth_context.h", 
       "src/cpp/server/secure_server_credentials.h", 
       "src/cpp/server/thread_pool.h"
     ], 
@@ -9489,6 +9536,7 @@
     "src": [
       "include/grpc++/async_generic_service.h", 
       "include/grpc++/async_unary_call.h", 
+      "include/grpc++/auth_context.h", 
       "include/grpc++/byte_buffer.h", 
       "include/grpc++/channel_arguments.h", 
       "include/grpc++/channel_interface.h", 
@@ -9538,7 +9586,11 @@
       "src/cpp/client/secure_credentials.h", 
       "src/cpp/common/call.cc", 
       "src/cpp/common/completion_queue.cc", 
+      "src/cpp/common/create_auth_context.h", 
       "src/cpp/common/rpc_method.cc", 
+      "src/cpp/common/secure_auth_context.cc", 
+      "src/cpp/common/secure_auth_context.h", 
+      "src/cpp/common/secure_create_auth_context.cc", 
       "src/cpp/proto/proto_utils.cc", 
       "src/cpp/server/async_generic_service.cc", 
       "src/cpp/server/create_default_thread_pool.cc", 
@@ -9607,6 +9659,7 @@
     "headers": [
       "include/grpc++/async_generic_service.h", 
       "include/grpc++/async_unary_call.h", 
+      "include/grpc++/auth_context.h", 
       "include/grpc++/byte_buffer.h", 
       "include/grpc++/channel_arguments.h", 
       "include/grpc++/channel_interface.h", 
@@ -9643,6 +9696,7 @@
       "include/grpc++/thread_pool_interface.h", 
       "include/grpc++/time.h", 
       "src/cpp/client/channel.h", 
+      "src/cpp/common/create_auth_context.h", 
       "src/cpp/server/thread_pool.h"
     ], 
     "language": "c++", 
@@ -9650,6 +9704,7 @@
     "src": [
       "include/grpc++/async_generic_service.h", 
       "include/grpc++/async_unary_call.h", 
+      "include/grpc++/auth_context.h", 
       "include/grpc++/byte_buffer.h", 
       "include/grpc++/channel_arguments.h", 
       "include/grpc++/channel_interface.h", 
@@ -9696,6 +9751,8 @@
       "src/cpp/client/internal_stub.cc", 
       "src/cpp/common/call.cc", 
       "src/cpp/common/completion_queue.cc", 
+      "src/cpp/common/create_auth_context.h", 
+      "src/cpp/common/insecure_create_auth_context.cc", 
       "src/cpp/common/rpc_method.cc", 
       "src/cpp/proto/proto_utils.cc", 
       "src/cpp/server/async_generic_service.cc", 
diff --git a/tools/run_tests/tests.json b/tools/run_tests/tests.json
index b46004d2b3..cc05870640 100644
--- a/tools/run_tests/tests.json
+++ b/tools/run_tests/tests.json
@@ -314,6 +314,15 @@
   {
     "flaky": false, 
     "language": "c", 
+    "name": "grpc_jwt_verifier_test", 
+    "platforms": [
+      "windows", 
+      "posix"
+    ]
+  }, 
+  {
+    "flaky": false, 
+    "language": "c", 
     "name": "grpc_security_connector_test", 
     "platforms": [
       "windows", 
@@ -714,6 +723,15 @@
   {
     "flaky": false, 
     "language": "c++", 
+    "name": "secure_auth_context_test", 
+    "platforms": [
+      "windows", 
+      "posix"
+    ]
+  }, 
+  {
+    "flaky": false, 
+    "language": "c++", 
     "name": "server_crash_test", 
     "platforms": [
       "windows",