Merge remote-tracking branch 'upstream/master' into c++_subchannel_list

8 files changed, 129 insertions, 112 deletions

diff --git a/tools/run_tests/README.md b/tools/run_tests/README.md
index 60f20747cd..ceddc21a98 100644
--- a/tools/run_tests/README.md
+++ b/tools/run_tests/README.md
@@ -14,6 +14,10 @@ Builds gRPC in given language and runs unit tests. Use `tools/run_tests/run_test
 - `--use_docker` Builds a docker container containing all the prerequisites for given language and runs the tests under that container.
 - `--build_only` Only build, do not run the tests.
 
+Note: some tests may be flaky. Check the "Issues" tab for known flakes and other issues.
+
+The full suite of unit tests will take many minutes to run.
+
 # Interop tests (run_interop_tests.py)
 
 Runs tests for cross-platform/cross-language interoperability. For more details, see [Interop tests descriptions](/doc/interop-test-descriptions.md)
diff --git a/tools/run_tests/artifacts/build_package_ruby.sh b/tools/run_tests/artifacts/build_package_ruby.sh
index 05eacd4d1e..4f74f08276 100755
--- a/tools/run_tests/artifacts/build_package_ruby.sh
+++ b/tools/run_tests/artifacts/build_package_ruby.sh
@@ -53,8 +53,11 @@ for arch in {x86,x64}; do
     output_dir="$base/src/ruby/tools/bin/${ruby_arch}-${plat}"
     mkdir -p "$output_dir"/google/protobuf
     mkdir -p "$output_dir"/google/protobuf/compiler  # needed for plugin.proto
-    cp "$input_dir"/protoc* "$output_dir"/
-    cp "$input_dir"/grpc_ruby_plugin* "$output_dir"/
+    cp "$input_dir"/protoc* "$input_dir"/grpc_ruby_plugin* "$output_dir/"
+    if [[ "$plat" != "windows" ]]
+    then
+      chmod +x "$output_dir/protoc" "$output_dir/grpc_ruby_plugin"
+    fi
     for proto in "${well_known_protos[@]}"; do
       cp "$base/third_party/protobuf/src/google/protobuf/$proto.proto" "$output_dir/google/protobuf/$proto.proto"
     done
diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json
index 9676fe379d..ebbfdbf230 100644
--- a/tools/run_tests/generated/sources_and_headers.json
+++ b/tools/run_tests/generated/sources_and_headers.json
@@ -10745,6 +10745,8 @@
     ], 
     "headers": [
       "include/grpc++/impl/codegen/proto_utils.h", 
+      "include/grpcpp/impl/codegen/proto_buffer_reader.h", 
+      "include/grpcpp/impl/codegen/proto_buffer_writer.h", 
       "include/grpcpp/impl/codegen/proto_utils.h"
     ], 
     "is_filegroup": true, 
@@ -10752,6 +10754,8 @@
     "name": "grpc++_codegen_proto", 
     "src": [
       "include/grpc++/impl/codegen/proto_utils.h", 
+      "include/grpcpp/impl/codegen/proto_buffer_reader.h", 
+      "include/grpcpp/impl/codegen/proto_buffer_writer.h", 
       "include/grpcpp/impl/codegen/proto_utils.h"
     ], 
     "third_party": false, 
@@ -10850,6 +10854,8 @@
       "include/grpcpp/support/byte_buffer.h", 
       "include/grpcpp/support/channel_arguments.h", 
       "include/grpcpp/support/config.h", 
+      "include/grpcpp/support/proto_buffer_reader.h", 
+      "include/grpcpp/support/proto_buffer_writer.h", 
       "include/grpcpp/support/slice.h", 
       "include/grpcpp/support/status.h", 
       "include/grpcpp/support/status_code_enum.h", 
@@ -10952,6 +10958,8 @@
       "include/grpcpp/support/byte_buffer.h", 
       "include/grpcpp/support/channel_arguments.h", 
       "include/grpcpp/support/config.h", 
+      "include/grpcpp/support/proto_buffer_reader.h", 
+      "include/grpcpp/support/proto_buffer_writer.h", 
       "include/grpcpp/support/slice.h", 
       "include/grpcpp/support/status.h", 
       "include/grpcpp/support/status_code_enum.h", 
@@ -10996,7 +11004,6 @@
       "src/cpp/thread_manager/thread_manager.cc", 
       "src/cpp/thread_manager/thread_manager.h", 
       "src/cpp/util/byte_buffer_cc.cc", 
-      "src/cpp/util/slice_cc.cc", 
       "src/cpp/util/status.cc", 
       "src/cpp/util/string_ref.cc", 
       "src/cpp/util/time_cc.cc"
diff --git a/tools/run_tests/helper_scripts/build_python.sh b/tools/run_tests/helper_scripts/build_python.sh
index dac29b91dd..0b5d3316ad 100755
--- a/tools/run_tests/helper_scripts/build_python.sh
+++ b/tools/run_tests/helper_scripts/build_python.sh
@@ -132,7 +132,7 @@ fi
 # Perform build operations #
 ############################
 
-# Instnatiate the virtualenv, preferring to do so from the relevant python
+# Instantiate the virtualenv, preferring to do so from the relevant python
 # version. Even if these commands fail (e.g. on Windows due to name conflicts)
 # it's possible that the virtualenv is still usable and we trust the tester to
 # be able to 'figure it out' instead of us e.g. doing potentially expensive and
@@ -158,7 +158,7 @@ case "$VENV" in
   ;;
 esac
 
-$VENV_PYTHON -m pip install --upgrade pip==9.0.1
+$VENV_PYTHON -m pip install --upgrade pip==9.0.2
 $VENV_PYTHON -m pip install setuptools
 $VENV_PYTHON -m pip install cython
 $VENV_PYTHON -m pip install six enum34 protobuf futures
diff --git a/tools/run_tests/interop/android/android_interop_helper.sh b/tools/run_tests/interop/android/android_interop_helper.sh
deleted file mode 100755
index 116549b83e..0000000000
--- a/tools/run_tests/interop/android/android_interop_helper.sh
+++ /dev/null
@@ -1,36 +0,0 @@
-#!/bin/bash
-# Copyright 2017 gRPC authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Helper that runs inside the docker container and builds the APKs and
-# invokes Firebase Test Lab via gcloud.
-
-SERVICE_KEY=$1
-
-gcloud auth activate-service-account --key-file="$SERVICE_KEY" || exit 1
-gcloud config set project grpc-testing || exit 1
-
-rm -rf grpc-java
-git clone https://github.com/grpc/grpc-java.git
-cd grpc-java
-./gradlew install || exit 1
-cd android-interop-testing
-../gradlew assembleDebug
-../gradlew assembleDebugAndroidTest
-
-gcloud firebase test android run \
-  --type instrumentation \
-  --app app/build/outputs/apk/app-debug.apk \
-  --test app/build/outputs/apk/app-debug-androidTest.apk \
-  --device model=Nexus6,version=21,locale=en,orientation=portrait
diff --git a/tools/run_tests/interop/android/run_android_tests_on_firebase.sh b/tools/run_tests/interop/android/run_android_tests_on_firebase.sh
deleted file mode 100755
index f6472eba55..0000000000
--- a/tools/run_tests/interop/android/run_android_tests_on_firebase.sh
+++ /dev/null
@@ -1,30 +0,0 @@
-#!/bin/bash
-# Copyright 2017 gRPC authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Builds the gRPC Android instrumented interop tests inside a docker container
-# and runs them on Firebase Test Lab
-
-DOCKERFILE=tools/dockerfile/interoptest/grpc_interop_android_java/Dockerfile
-DOCKER_TAG=android_interop_test
-SERVICE_KEY=~/android-interops-service-key.json
-HELPER=$(pwd)/tools/run_tests/interop/android/android_interop_helper.sh
-
-docker build -t "$DOCKER_TAG" -f "$DOCKERFILE" .
-
-docker run --interactive --rm \
-  --volume="$SERVICE_KEY":/service-key.json:ro \
-  --volume="$HELPER":/android_interop_helper.sh:ro \
-  $DOCKER_TAG \
-      /bin/bash -c "/android_interop_helper.sh /service-key.json"
diff --git a/tools/run_tests/python_utils/upload_test_results.py b/tools/run_tests/python_utils/upload_test_results.py
index a2dd1c66cf..09dcd57ad4 100644
--- a/tools/run_tests/python_utils/upload_test_results.py
+++ b/tools/run_tests/python_utils/upload_test_results.py
@@ -74,7 +74,7 @@ def _get_build_metadata(test_results):
     build_id = os.getenv('BUILD_ID') or os.getenv('KOKORO_BUILD_NUMBER')
     build_url = os.getenv('BUILD_URL')
     if os.getenv('KOKORO_BUILD_ID'):
-        build_url = 'https://sponge.corp.google.com/invocation?id=%s' % os.getenv(
+        build_url = 'https://source.cloud.google.com/results/invocations/%s' % os.getenv(
             'KOKORO_BUILD_ID')
     job_name = os.getenv('JOB_BASE_NAME') or os.getenv('KOKORO_JOB_NAME')
 
diff --git a/tools/run_tests/run_interop_tests.py b/tools/run_tests/run_interop_tests.py
index 970b0d7902..56aee6419e 100755
--- a/tools/run_tests/run_interop_tests.py
+++ b/tools/run_tests/run_interop_tests.py
@@ -637,6 +637,14 @@ _LANGUAGES_WITH_HTTP2_CLIENTS_FOR_HTTP2_SERVER_TEST_CASES = [
     'java', 'go', 'python', 'c++'
 ]
 
+#TODO: Add c++ when c++ ALTS interop client is ready.
+_LANGUAGES_FOR_ALTS_TEST_CASES = ['java', 'go']
+
+#TODO: Add c++ when c++ ALTS interop server is ready.
+_SERVERS_FOR_ALTS_TEST_CASES = ['java', 'go']
+
+_TRANSPORT_SECURITY_OPTIONS = ['tls', 'alts', 'insecure']
+
 DOCKER_WORKDIR_ROOT = '/var/local/git/grpc'
 
 
@@ -691,17 +699,29 @@ def bash_cmdline(cmdline):
     return ['bash', '-c', ' '.join(cmdline)]
 
 
-def auth_options(language, test_case):
+def compute_engine_creds_required(language, test_case):
+    """Returns True if given test requires access to compute engine creds."""
+    language = str(language)
+    if test_case == 'compute_engine_creds':
+        return True
+    if test_case == 'oauth2_auth_token' and language == 'c++':
+        # C++ oauth2 test uses GCE creds because C++ only supports JWT
+        return True
+    return False
+
+
+def auth_options(language, test_case, service_account_key_file=None):
     """Returns (cmdline, env) tuple with cloud_to_prod_auth test options."""
 
     language = str(language)
     cmdargs = []
     env = {}
 
-    # TODO(jtattermusch): this file path only works inside docker
-    key_filepath = '/root/service_account/GrpcTesting-726eb1347f15.json'
+    if not service_account_key_file:
+        # this file path only works inside docker
+        service_account_key_file = '/root/service_account/GrpcTesting-726eb1347f15.json'
     oauth_scope_arg = '--oauth_scope=https://www.googleapis.com/auth/xapi.zoo'
-    key_file_arg = '--service_account_key_file=%s' % key_filepath
+    key_file_arg = '--service_account_key_file=%s' % service_account_key_file
     default_account_arg = '--default_service_account=830293263384-compute@developer.gserviceaccount.com'
 
     if test_case in ['jwt_token_creds', 'per_rpc_creds', 'oauth2_auth_token']:
@@ -709,7 +729,7 @@ def auth_options(language, test_case):
                 'csharp', 'csharpcoreclr', 'node', 'php', 'php7', 'python',
                 'ruby', 'nodepurejs'
         ]:
-            env['GOOGLE_APPLICATION_CREDENTIALS'] = key_filepath
+            env['GOOGLE_APPLICATION_CREDENTIALS'] = service_account_key_file
         else:
             cmdargs += [key_file_arg]
 
@@ -742,7 +762,8 @@ def cloud_to_prod_jobspec(language,
                           server_host_detail,
                           docker_image=None,
                           auth=False,
-                          manual_cmd_log=None):
+                          manual_cmd_log=None,
+                          service_account_key_file=None):
     """Creates jobspec for cloud-to-prod interop test"""
     container_name = None
     cmdargs = [
@@ -753,7 +774,8 @@ def cloud_to_prod_jobspec(language,
     ]
     environ = dict(language.cloud_to_prod_env(), **language.global_env())
     if auth:
-        auth_cmdargs, auth_env = auth_options(language, test_case)
+        auth_cmdargs, auth_env = auth_options(language, test_case,
+                                              service_account_key_file)
         cmdargs += auth_cmdargs
         environ.update(auth_env)
     cmdline = bash_cmdline(language.client_cmd(cmdargs))
@@ -799,14 +821,22 @@ def cloud_to_cloud_jobspec(language,
                            server_host,
                            server_port,
                            docker_image=None,
-                           insecure=False,
+                           transport_security='tls',
                            manual_cmd_log=None):
     """Creates jobspec for cloud-to-cloud interop test"""
     interop_only_options = [
         '--server_host_override=foo.test.google.fr',
-        '--use_tls=%s' % ('false' if insecure else 'true'),
         '--use_test_ca=true',
     ]
+    if transport_security == 'tls':
+        interop_only_options += ['--use_tls=true']
+    elif transport_security == 'alts':
+        interop_only_options += ['--use_tls=false', '--use_alts=true']
+    elif transport_security == 'insecure':
+        interop_only_options += ['--use_tls=false']
+    else:
+        print('Invalid transport security option.')
+        sys.exit(1)
 
     client_test_case = test_case
     if test_case in _HTTP2_SERVER_TEST_CASES_THAT_USE_GRPC_CLIENTS:
@@ -871,15 +901,24 @@ def cloud_to_cloud_jobspec(language,
     return test_job
 
 
-def server_jobspec(language, docker_image, insecure=False, manual_cmd_log=None):
+def server_jobspec(language,
+                   docker_image,
+                   transport_security='tls',
+                   manual_cmd_log=None):
     """Create jobspec for running a server"""
     container_name = dockerjob.random_name(
         'interop_server_%s' % language.safename)
-    cmdline = bash_cmdline(
-        language.server_cmd([
-            '--port=%s' % _DEFAULT_SERVER_PORT,
-            '--use_tls=%s' % ('false' if insecure else 'true')
-        ]))
+    server_cmd = ['--port=%s' % _DEFAULT_SERVER_PORT]
+    if transport_security == 'tls':
+        server_cmd += ['--use_tls=true']
+    elif transport_security == 'alts':
+        server_cmd += ['--use_tls=false', '--use_alts=true']
+    elif transport_security == 'insecure':
+        server_cmd += ['--use_tls=false']
+    else:
+        print('Invalid transport security option.')
+        sys.exit(1)
+    cmdline = bash_cmdline(language.server_cmd(server_cmd))
     environ = language.global_env()
     docker_args = ['--name=%s' % container_name]
     if language.safename == 'http2':
@@ -1044,6 +1083,12 @@ argp.add_argument(
     'Use servername=HOST:PORT to explicitly specify a server. E.g. csharp=localhost:50000',
     default=[])
 argp.add_argument(
+    '--service_account_key_file',
+    type=str,
+    help=
+    'Override the default service account key file to use for auth interop tests.',
+    default=None)
+argp.add_argument(
     '-t', '--travis', default=False, action='store_const', const=True)
 argp.add_argument(
     '-v', '--verbose', default=False, action='store_const', const=True)
@@ -1086,11 +1131,19 @@ argp.add_argument(
     'Enable HTTP/2 server edge case testing. (Includes positive and negative tests'
 )
 argp.add_argument(
-    '--insecure',
+    '--transport_security',
+    choices=_TRANSPORT_SECURITY_OPTIONS,
+    default='tls',
+    type=str,
+    nargs='?',
+    const=True,
+    help='Which transport security mechanism to use.')
+argp.add_argument(
+    '--skip_compute_engine_creds',
     default=False,
     action='store_const',
     const=True,
-    help='Whether to use secure channel.')
+    help='Skip auth tests requiring access to compute engine credentials.')
 argp.add_argument(
     '--internal_ci',
     default=False,
@@ -1110,6 +1163,9 @@ servers = set(
     s
     for s in itertools.chain.from_iterable(
         _SERVERS if x == 'all' else [x] for x in args.server))
+# ALTS servers are only available for certain languages.
+if args.transport_security == 'alts':
+    servers = servers.intersection(_SERVERS_FOR_ALTS_TEST_CASES)
 
 if args.use_docker:
     if not args.travis:
@@ -1139,6 +1195,10 @@ all_but_objc = set(six.iterkeys(_LANGUAGES)) - set(['objc'])
 languages = set(_LANGUAGES[l]
                 for l in itertools.chain.from_iterable(
                     all_but_objc if x == 'all' else [x] for x in args.language))
+# ALTS interop clients are only available for certain languages.
+if args.transport_security == 'alts':
+    alts_languages = set(_LANGUAGES[l] for l in _LANGUAGES_FOR_ALTS_TEST_CASES)
+    languages = languages.intersection(alts_languages)
 
 languages_http2_clients_for_http2_server_interop = set()
 if args.http2_server_interop:
@@ -1207,7 +1267,7 @@ try:
         spec = server_jobspec(
             _LANGUAGES[lang],
             docker_images.get(lang),
-            args.insecure,
+            args.transport_security,
             manual_cmd_log=server_manual_cmd_log)
         if not args.manual_run:
             job = dockerjob.DockerJob(spec)
@@ -1235,7 +1295,7 @@ try:
 
     jobs = []
     if args.cloud_to_prod:
-        if args.insecure:
+        if args.transport_security != 'tls':
             print('TLS is always enabled for cloud_to_prod scenarios.')
         for server_host_name in args.prod_servers:
             for language in languages:
@@ -1248,7 +1308,9 @@ try:
                                 server_host_name,
                                 prod_servers[server_host_name],
                                 docker_image=docker_images.get(str(language)),
-                                manual_cmd_log=client_manual_cmd_log)
+                                manual_cmd_log=client_manual_cmd_log,
+                                service_account_key_file=args.
+                                service_account_key_file)
                             jobs.append(test_job)
 
             if args.http2_interop:
@@ -1259,25 +1321,31 @@ try:
                         server_host_name,
                         prod_servers[server_host_name],
                         docker_image=docker_images.get(str(http2Interop)),
-                        manual_cmd_log=client_manual_cmd_log)
+                        manual_cmd_log=client_manual_cmd_log,
+                        service_account_key_file=args.service_account_key_file)
                     jobs.append(test_job)
 
     if args.cloud_to_prod_auth:
-        if args.insecure:
+        if args.transport_security != 'tls':
             print('TLS is always enabled for cloud_to_prod scenarios.')
         for server_host_name in args.prod_servers:
             for language in languages:
                 for test_case in _AUTH_TEST_CASES:
-                    if not test_case in language.unimplemented_test_cases():
-                        test_job = cloud_to_prod_jobspec(
-                            language,
-                            test_case,
-                            server_host_name,
-                            prod_servers[server_host_name],
-                            docker_image=docker_images.get(str(language)),
-                            auth=True,
-                            manual_cmd_log=client_manual_cmd_log)
-                        jobs.append(test_job)
+                    if (not args.skip_compute_engine_creds or
+                            not compute_engine_creds_required(
+                                language, test_case)):
+                        if not test_case in language.unimplemented_test_cases():
+                            test_job = cloud_to_prod_jobspec(
+                                language,
+                                test_case,
+                                server_host_name,
+                                prod_servers[server_host_name],
+                                docker_image=docker_images.get(str(language)),
+                                auth=True,
+                                manual_cmd_log=client_manual_cmd_log,
+                                service_account_key_file=args.
+                                service_account_key_file)
+                            jobs.append(test_job)
 
     for server in args.override_server:
         server_name = server[0]
@@ -1301,7 +1369,7 @@ try:
                             server_host,
                             server_port,
                             docker_image=docker_images.get(str(language)),
-                            insecure=args.insecure,
+                            transport_security=args.transport_security,
                             manual_cmd_log=client_manual_cmd_log)
                         jobs.append(test_job)
 
@@ -1317,7 +1385,7 @@ try:
                     server_host,
                     server_port,
                     docker_image=docker_images.get(str(http2Interop)),
-                    insecure=args.insecure,
+                    transport_security=args.transport_security,
                     manual_cmd_log=client_manual_cmd_log)
                 jobs.append(test_job)
 
@@ -1353,11 +1421,12 @@ try:
                     server_port = _DEFAULT_SERVER_PORT + offset
                     if not args.manual_run:
                         server_port = http2_server_job.mapped_port(server_port)
-                    if not args.insecure:
-                        print((
-                            'Creating grpc cient to http2 server test case with insecure connection, even though'
-                            ' args.insecure is False. Http2 test server only supports insecure connections.'
-                        ))
+                    if args.transport_security != 'insecure':
+                        print(
+                            ('Creating grpc client to http2 server test case '
+                             'with insecure connection, even though '
+                             'args.transport_security is not insecure. Http2 '
+                             'test server only supports insecure connections.'))
                     test_job = cloud_to_cloud_jobspec(
                         language,
                         test_case,
@@ -1365,7 +1434,7 @@ try:
                         'localhost',
                         server_port,
                         docker_image=docker_images.get(str(language)),
-                        insecure=True,
+                        transport_security='insecure',
                         manual_cmd_log=client_manual_cmd_log)
                     jobs.append(test_job)