Merge pull request #12807 from adelez/backwards_comp

Exit 1 when there are failures and add gateway4 backend in test cases

6 files changed, 58 insertions, 11 deletions

diff --git a/tools/interop_matrix/README.md b/tools/interop_matrix/README.md
index c2f354399f..c0e9a33c5e 100644
--- a/tools/interop_matrix/README.md
+++ b/tools/interop_matrix/README.md
@@ -47,7 +47,7 @@ For more details on each step, refer to sections below.
 
 ## Instructions for running test cases against GCR images
 - Run `tools/interop_matrix/run_interop_matrix_tests.py`.  Useful options:
-  - `--release` specifies a git release tag.  Defaults to `--release=master`.  Make sure the GCR images with the tag have been created using `create_matrix_images.py` above.
+  - `--release` specifies a git release tag.  Defaults to `--release=all`.  Make sure the GCR images with the tag have been created using `create_matrix_images.py` above.
   - `--language` specifies a language.  Defaults to `--language=all`.
   For example, To test all languages for all gRPC releases across all runtimes, do `tools/interop_matrix/run_interop_matrix_test.py --release=all`.
 - The output for all the test cases is recorded in a junit style xml file (default to 'report.xml').
diff --git a/tools/interop_matrix/create_testcases.sh b/tools/interop_matrix/create_testcases.sh
index d06fb34ff9..3d34b2ef25 100755
--- a/tools/interop_matrix/create_testcases.sh
+++ b/tools/interop_matrix/create_testcases.sh
@@ -31,9 +31,12 @@ TESTCASES_DIR=${GRPC_ROOT}/tools/interop_matrix/testcases
 
 echo "Create '$LANG' test cases for gRPC release '${RELEASE:=master}'"
 
+echo $client_lang
 # Invoke run_interop_test in manual mode.
+# TODO(adelez): Add cloud_gateways when we figure out how to skip them if not 
+# running in GCE.
 ${GRPC_ROOT}/tools/run_tests/run_interop_tests.py -l $LANG --use_docker \
-  --cloud_to_prod --manual_run
+  --cloud_to_prod --prod_servers default gateway_v4 --manual_run
 
 # Clean up
 function cleanup {
@@ -52,11 +55,18 @@ function cleanup {
   [ -e "$CMDS_SH" ] && rm $CMDS_SH
 }
 trap cleanup EXIT
+# TODO(adelez): add test auth tests but do not run if not testing on GCE.
 # Running the testcases as sanity unless we are asked to skip.
 [ -z "$SKIP_TEST" ] && (echo "Running test cases: $CMDS_SH"; sh $CMDS_SH)
 
+# Convert c++ to cxx. 
+if [$LANG == "c++" ]; then
+client_lang="cxx"
+else
+client_lang=$LANG
+fi
 mkdir -p $TESTCASES_DIR
-testcase=$TESTCASES_DIR/${LANG}__$RELEASE
+testcase=$TESTCASES_DIR/${client_lang}__$RELEASE
 if [ -e $testcase ]; then
   echo "Updating: $testcase"
   diff $testcase $CMDS_SH || true
diff --git a/tools/interop_matrix/run_interop_matrix_tests.py b/tools/interop_matrix/run_interop_matrix_tests.py
index 4315c8277d..510bc7124d 100755
--- a/tools/interop_matrix/run_interop_matrix_tests.py
+++ b/tools/interop_matrix/run_interop_matrix_tests.py
@@ -48,9 +48,8 @@ argp.add_argument('-j', '--jobs', default=multiprocessing.cpu_count(), type=int)
 argp.add_argument('--gcr_path',
                   default='gcr.io/grpc-testing',
                   help='Path of docker images in Google Container Registry')
-
 argp.add_argument('--release',
-                  default='master',
+                  default='all',
                   choices=['all', 'master'] + _RELEASES,
                   help='Release tags to test.  When testing all '
                   'releases defined in client_matrix.py, use "all".')
@@ -94,14 +93,15 @@ def find_all_images_for_lang(lang):
   for runtime in client_matrix.LANG_RUNTIME_MATRIX[lang]:
     image_path = '%s/grpc_interop_%s' % (args.gcr_path, runtime)
     output = subprocess.check_output(['gcloud', 'beta', 'container', 'images',
-                                        'list-tags', '--format=json', image_path])
+                                      'list-tags', '--format=json', image_path])
     docker_image_list = json.loads(output)
     # All images should have a single tag or no tag.
+    # TODO(adelez): Remove tagless images.
     tags = [i['tags'][0] for i in docker_image_list if i['tags']]
     jobset.message('START', 'Found images for %s: %s' % (image_path, tags),
                    do_newline=True)
     skipped = len(docker_image_list) - len(tags)
-    jobset.message('START', 'Skipped images (no-tag/unknown-tag): %d' % skipped,
+    jobset.message('SKIPPED', 'Skipped images (no-tag/unknown-tag): %d' % skipped,
                    do_newline=True)
     # Filter tags based on the releases.
     images[runtime] = [(tag,'%s:%s' % (image_path,tag)) for tag in tags if
@@ -148,6 +148,7 @@ def run_tests_for_lang(lang, runtime, images):
 
   images is a list of (<release-tag>, <image-full-path>) tuple.
   """
+  total_num_failures = 0
   for image_tuple in images:
     release, image = image_tuple
     jobset.message('START', 'Testing %s' % image, do_newline=True)
@@ -161,6 +162,7 @@ def run_tests_for_lang(lang, runtime, images):
                                          maxjobs=args.jobs)
     if num_failures:
       jobset.message('FAILED', 'Some tests failed', do_newline=True)
+      total_num_failures += num_failures
     else:
       jobset.message('SUCCESS', 'All tests passed', do_newline=True)
 
@@ -170,6 +172,9 @@ def run_tests_for_lang(lang, runtime, images):
         'grpc_interop_matrix',
         '%s__%s %s'%(lang,runtime,release),
         str(uuid.uuid4()))
+  
+  return total_num_failures
+
 
 _docker_images_cleanup = []
 def cleanup():
@@ -180,9 +185,14 @@ def cleanup():
 atexit.register(cleanup)
 
 languages = args.language if args.language != ['all'] else _LANGUAGES
+total_num_failures = 0
 for lang in languages:
   docker_images = find_all_images_for_lang(lang)
   for runtime in sorted(docker_images.keys()):
-    run_tests_for_lang(lang, runtime, docker_images[runtime])
+    total_num_failures += run_tests_for_lang(lang, runtime, docker_images[runtime])
 
 report_utils.create_xml_report_file(_xml_report_tree, args.report_file)
+
+if total_num_failures:
+  sys.exit(1)
+sys.exit(0)
diff --git a/tools/interop_matrix/testcases/cxx__master b/tools/interop_matrix/testcases/cxx__master
index ccd2859530..e0fed53f08 100755
--- a/tools/interop_matrix/testcases/cxx__master
+++ b/tools/interop_matrix/testcases/cxx__master
@@ -1,5 +1,5 @@
 #!/bin/bash
-echo "Testing ${docker_image:=grpc_interop_cxx:1423f288-ac00-4f3a-9885-771258eecae3}"
+echo "Testing ${docker_image:=grpc_interop_cxx:78de6f80-524d-4bc9-bfb2-f00c24ceafed}"
 docker run -i --rm=true -w /var/local/git/grpc --net=host $docker_image bash -c "bins/opt/interop_client --server_host=216.239.32.254 --server_host_override=grpc-test.sandbox.googleapis.com --server_port=443 --use_tls=true --test_case=large_unary"
 docker run -i --rm=true -w /var/local/git/grpc --net=host $docker_image bash -c "bins/opt/interop_client --server_host=216.239.32.254 --server_host_override=grpc-test.sandbox.googleapis.com --server_port=443 --use_tls=true --test_case=empty_unary"
 docker run -i --rm=true -w /var/local/git/grpc --net=host $docker_image bash -c "bins/opt/interop_client --server_host=216.239.32.254 --server_host_override=grpc-test.sandbox.googleapis.com --server_port=443 --use_tls=true --test_case=ping_pong"
@@ -9,3 +9,12 @@ docker run -i --rm=true -w /var/local/git/grpc --net=host $docker_image bash -c
 docker run -i --rm=true -w /var/local/git/grpc --net=host $docker_image bash -c "bins/opt/interop_client --server_host=216.239.32.254 --server_host_override=grpc-test.sandbox.googleapis.com --server_port=443 --use_tls=true --test_case=cancel_after_begin"
 docker run -i --rm=true -w /var/local/git/grpc --net=host $docker_image bash -c "bins/opt/interop_client --server_host=216.239.32.254 --server_host_override=grpc-test.sandbox.googleapis.com --server_port=443 --use_tls=true --test_case=cancel_after_first_response"
 docker run -i --rm=true -w /var/local/git/grpc --net=host $docker_image bash -c "bins/opt/interop_client --server_host=216.239.32.254 --server_host_override=grpc-test.sandbox.googleapis.com --server_port=443 --use_tls=true --test_case=timeout_on_sleeping_server"
+docker run -i --rm=true -w /var/local/git/grpc --net=host $docker_image bash -c "bins/opt/interop_client --server_host=216.239.32.254 --server_host_override=grpc-test4.sandbox.googleapis.com --server_port=443 --use_tls=true --test_case=large_unary"
+docker run -i --rm=true -w /var/local/git/grpc --net=host $docker_image bash -c "bins/opt/interop_client --server_host=216.239.32.254 --server_host_override=grpc-test4.sandbox.googleapis.com --server_port=443 --use_tls=true --test_case=empty_unary"
+docker run -i --rm=true -w /var/local/git/grpc --net=host $docker_image bash -c "bins/opt/interop_client --server_host=216.239.32.254 --server_host_override=grpc-test4.sandbox.googleapis.com --server_port=443 --use_tls=true --test_case=ping_pong"
+docker run -i --rm=true -w /var/local/git/grpc --net=host $docker_image bash -c "bins/opt/interop_client --server_host=216.239.32.254 --server_host_override=grpc-test4.sandbox.googleapis.com --server_port=443 --use_tls=true --test_case=empty_stream"
+docker run -i --rm=true -w /var/local/git/grpc --net=host $docker_image bash -c "bins/opt/interop_client --server_host=216.239.32.254 --server_host_override=grpc-test4.sandbox.googleapis.com --server_port=443 --use_tls=true --test_case=client_streaming"
+docker run -i --rm=true -w /var/local/git/grpc --net=host $docker_image bash -c "bins/opt/interop_client --server_host=216.239.32.254 --server_host_override=grpc-test4.sandbox.googleapis.com --server_port=443 --use_tls=true --test_case=server_streaming"
+docker run -i --rm=true -w /var/local/git/grpc --net=host $docker_image bash -c "bins/opt/interop_client --server_host=216.239.32.254 --server_host_override=grpc-test4.sandbox.googleapis.com --server_port=443 --use_tls=true --test_case=cancel_after_begin"
+docker run -i --rm=true -w /var/local/git/grpc --net=host $docker_image bash -c "bins/opt/interop_client --server_host=216.239.32.254 --server_host_override=grpc-test4.sandbox.googleapis.com --server_port=443 --use_tls=true --test_case=cancel_after_first_response"
+docker run -i --rm=true -w /var/local/git/grpc --net=host $docker_image bash -c "bins/opt/interop_client --server_host=216.239.32.254 --server_host_override=grpc-test4.sandbox.googleapis.com --server_port=443 --use_tls=true --test_case=timeout_on_sleeping_server"
diff --git a/tools/interop_matrix/testcases/go__master b/tools/interop_matrix/testcases/go__master
index 2624c7f92c..33b25d6a16 100755
--- a/tools/interop_matrix/testcases/go__master
+++ b/tools/interop_matrix/testcases/go__master
@@ -1,5 +1,5 @@
 #!/bin/bash
-echo "Testing ${docker_image:=grpc_interop_go:41fffd01-a6c8-41b6-8136-c0aaa1ec2437}"
+echo "Testing ${docker_image:=grpc_interop_go:dd8fbf3a-4964-4387-9997-5dadeea09835}"
 docker run -i --rm=true -w /go/src/google.golang.org/grpc/interop/client --net=host $docker_image bash -c "go run client.go --server_host=216.239.32.254 --server_host_override=grpc-test.sandbox.googleapis.com --server_port=443 --use_tls=true --test_case=large_unary"
 docker run -i --rm=true -w /go/src/google.golang.org/grpc/interop/client --net=host $docker_image bash -c "go run client.go --server_host=216.239.32.254 --server_host_override=grpc-test.sandbox.googleapis.com --server_port=443 --use_tls=true --test_case=empty_unary"
 docker run -i --rm=true -w /go/src/google.golang.org/grpc/interop/client --net=host $docker_image bash -c "go run client.go --server_host=216.239.32.254 --server_host_override=grpc-test.sandbox.googleapis.com --server_port=443 --use_tls=true --test_case=ping_pong"
@@ -9,3 +9,12 @@ docker run -i --rm=true -w /go/src/google.golang.org/grpc/interop/client --net=h
 docker run -i --rm=true -w /go/src/google.golang.org/grpc/interop/client --net=host $docker_image bash -c "go run client.go --server_host=216.239.32.254 --server_host_override=grpc-test.sandbox.googleapis.com --server_port=443 --use_tls=true --test_case=cancel_after_begin"
 docker run -i --rm=true -w /go/src/google.golang.org/grpc/interop/client --net=host $docker_image bash -c "go run client.go --server_host=216.239.32.254 --server_host_override=grpc-test.sandbox.googleapis.com --server_port=443 --use_tls=true --test_case=cancel_after_first_response"
 docker run -i --rm=true -w /go/src/google.golang.org/grpc/interop/client --net=host $docker_image bash -c "go run client.go --server_host=216.239.32.254 --server_host_override=grpc-test.sandbox.googleapis.com --server_port=443 --use_tls=true --test_case=timeout_on_sleeping_server"
+docker run -i --rm=true -w /go/src/google.golang.org/grpc/interop/client --net=host $docker_image bash -c "go run client.go --server_host=216.239.32.254 --server_host_override=grpc-test4.sandbox.googleapis.com --server_port=443 --use_tls=true --test_case=large_unary"
+docker run -i --rm=true -w /go/src/google.golang.org/grpc/interop/client --net=host $docker_image bash -c "go run client.go --server_host=216.239.32.254 --server_host_override=grpc-test4.sandbox.googleapis.com --server_port=443 --use_tls=true --test_case=empty_unary"
+docker run -i --rm=true -w /go/src/google.golang.org/grpc/interop/client --net=host $docker_image bash -c "go run client.go --server_host=216.239.32.254 --server_host_override=grpc-test4.sandbox.googleapis.com --server_port=443 --use_tls=true --test_case=ping_pong"
+docker run -i --rm=true -w /go/src/google.golang.org/grpc/interop/client --net=host $docker_image bash -c "go run client.go --server_host=216.239.32.254 --server_host_override=grpc-test4.sandbox.googleapis.com --server_port=443 --use_tls=true --test_case=empty_stream"
+docker run -i --rm=true -w /go/src/google.golang.org/grpc/interop/client --net=host $docker_image bash -c "go run client.go --server_host=216.239.32.254 --server_host_override=grpc-test4.sandbox.googleapis.com --server_port=443 --use_tls=true --test_case=client_streaming"
+docker run -i --rm=true -w /go/src/google.golang.org/grpc/interop/client --net=host $docker_image bash -c "go run client.go --server_host=216.239.32.254 --server_host_override=grpc-test4.sandbox.googleapis.com --server_port=443 --use_tls=true --test_case=server_streaming"
+docker run -i --rm=true -w /go/src/google.golang.org/grpc/interop/client --net=host $docker_image bash -c "go run client.go --server_host=216.239.32.254 --server_host_override=grpc-test4.sandbox.googleapis.com --server_port=443 --use_tls=true --test_case=cancel_after_begin"
+docker run -i --rm=true -w /go/src/google.golang.org/grpc/interop/client --net=host $docker_image bash -c "go run client.go --server_host=216.239.32.254 --server_host_override=grpc-test4.sandbox.googleapis.com --server_port=443 --use_tls=true --test_case=cancel_after_first_response"
+docker run -i --rm=true -w /go/src/google.golang.org/grpc/interop/client --net=host $docker_image bash -c "go run client.go --server_host=216.239.32.254 --server_host_override=grpc-test4.sandbox.googleapis.com --server_port=443 --use_tls=true --test_case=timeout_on_sleeping_server"
diff --git a/tools/interop_matrix/testcases/java__master b/tools/interop_matrix/testcases/java__master
index cf431646e9..dbd87279a6 100755
--- a/tools/interop_matrix/testcases/java__master
+++ b/tools/interop_matrix/testcases/java__master
@@ -1,5 +1,5 @@
 #!/bin/bash
-echo "Testing ${docker_image:=grpc_interop_java:ea528843-be34-4ff3-a136-e4609424e061}"
+echo "Testing ${docker_image:=grpc_interop_java:a764b50c-1788-4387-9b9e-5cfa93927006}"
 docker run -i --rm=true -w /var/local/git/grpc/../grpc-java --net=host $docker_image bash -c "./run-test-client.sh --server_host=216.239.32.254 --server_host_override=grpc-test.sandbox.googleapis.com --server_port=443 --use_tls=true --test_case=large_unary"
 docker run -i --rm=true -w /var/local/git/grpc/../grpc-java --net=host $docker_image bash -c "./run-test-client.sh --server_host=216.239.32.254 --server_host_override=grpc-test.sandbox.googleapis.com --server_port=443 --use_tls=true --test_case=empty_unary"
 docker run -i --rm=true -w /var/local/git/grpc/../grpc-java --net=host $docker_image bash -c "./run-test-client.sh --server_host=216.239.32.254 --server_host_override=grpc-test.sandbox.googleapis.com --server_port=443 --use_tls=true --test_case=ping_pong"
@@ -9,3 +9,12 @@ docker run -i --rm=true -w /var/local/git/grpc/../grpc-java --net=host $docker_i
 docker run -i --rm=true -w /var/local/git/grpc/../grpc-java --net=host $docker_image bash -c "./run-test-client.sh --server_host=216.239.32.254 --server_host_override=grpc-test.sandbox.googleapis.com --server_port=443 --use_tls=true --test_case=cancel_after_begin"
 docker run -i --rm=true -w /var/local/git/grpc/../grpc-java --net=host $docker_image bash -c "./run-test-client.sh --server_host=216.239.32.254 --server_host_override=grpc-test.sandbox.googleapis.com --server_port=443 --use_tls=true --test_case=cancel_after_first_response"
 docker run -i --rm=true -w /var/local/git/grpc/../grpc-java --net=host $docker_image bash -c "./run-test-client.sh --server_host=216.239.32.254 --server_host_override=grpc-test.sandbox.googleapis.com --server_port=443 --use_tls=true --test_case=timeout_on_sleeping_server"
+docker run -i --rm=true -w /var/local/git/grpc/../grpc-java --net=host $docker_image bash -c "./run-test-client.sh --server_host=216.239.32.254 --server_host_override=grpc-test4.sandbox.googleapis.com --server_port=443 --use_tls=true --test_case=large_unary"
+docker run -i --rm=true -w /var/local/git/grpc/../grpc-java --net=host $docker_image bash -c "./run-test-client.sh --server_host=216.239.32.254 --server_host_override=grpc-test4.sandbox.googleapis.com --server_port=443 --use_tls=true --test_case=empty_unary"
+docker run -i --rm=true -w /var/local/git/grpc/../grpc-java --net=host $docker_image bash -c "./run-test-client.sh --server_host=216.239.32.254 --server_host_override=grpc-test4.sandbox.googleapis.com --server_port=443 --use_tls=true --test_case=ping_pong"
+docker run -i --rm=true -w /var/local/git/grpc/../grpc-java --net=host $docker_image bash -c "./run-test-client.sh --server_host=216.239.32.254 --server_host_override=grpc-test4.sandbox.googleapis.com --server_port=443 --use_tls=true --test_case=empty_stream"
+docker run -i --rm=true -w /var/local/git/grpc/../grpc-java --net=host $docker_image bash -c "./run-test-client.sh --server_host=216.239.32.254 --server_host_override=grpc-test4.sandbox.googleapis.com --server_port=443 --use_tls=true --test_case=client_streaming"
+docker run -i --rm=true -w /var/local/git/grpc/../grpc-java --net=host $docker_image bash -c "./run-test-client.sh --server_host=216.239.32.254 --server_host_override=grpc-test4.sandbox.googleapis.com --server_port=443 --use_tls=true --test_case=server_streaming"
+docker run -i --rm=true -w /var/local/git/grpc/../grpc-java --net=host $docker_image bash -c "./run-test-client.sh --server_host=216.239.32.254 --server_host_override=grpc-test4.sandbox.googleapis.com --server_port=443 --use_tls=true --test_case=cancel_after_begin"
+docker run -i --rm=true -w /var/local/git/grpc/../grpc-java --net=host $docker_image bash -c "./run-test-client.sh --server_host=216.239.32.254 --server_host_override=grpc-test4.sandbox.googleapis.com --server_port=443 --use_tls=true --test_case=cancel_after_first_response"
+docker run -i --rm=true -w /var/local/git/grpc/../grpc-java --net=host $docker_image bash -c "./run-test-client.sh --server_host=216.239.32.254 --server_host_override=grpc-test4.sandbox.googleapis.com --server_port=443 --use_tls=true --test_case=timeout_on_sleeping_server"