aboutsummaryrefslogtreecommitdiffhomepage
diff options
context:
space:
mode:
authorGravatar adelez <adelez@google.com>2017-10-09 14:34:59 -0700
committerGravatar GitHub <noreply@github.com>2017-10-09 14:34:59 -0700
commitadb75b9746e67fec5c2eab28533bf1d54cd2c657 (patch)
tree8e2f806ec30860a04133b65a2e07f49c74cba5c3
parent0669accb4c8eb4d9bacb3e9704c4f2c62dcbb628 (diff)
parente6b9c787c0314d31060c0c4547bf12e087fb4e93 (diff)
Merge pull request #12807 from adelez/backwards_comp
Exit 1 when there are failures and add gateway4 backend in test cases
-rw-r--r--tools/interop_matrix/README.md2
-rwxr-xr-xtools/interop_matrix/create_testcases.sh14
-rwxr-xr-xtools/interop_matrix/run_interop_matrix_tests.py20
-rwxr-xr-xtools/interop_matrix/testcases/cxx__master11
-rwxr-xr-xtools/interop_matrix/testcases/go__master11
-rwxr-xr-xtools/interop_matrix/testcases/java__master11
6 files changed, 58 insertions, 11 deletions
diff --git a/tools/interop_matrix/README.md b/tools/interop_matrix/README.md
index c2f354399f..c0e9a33c5e 100644
--- a/tools/interop_matrix/README.md
+++ b/tools/interop_matrix/README.md
@@ -47,7 +47,7 @@ For more details on each step, refer to sections below.
## Instructions for running test cases against GCR images
- Run `tools/interop_matrix/run_interop_matrix_tests.py`. Useful options:
- - `--release` specifies a git release tag. Defaults to `--release=master`. Make sure the GCR images with the tag have been created using `create_matrix_images.py` above.
+ - `--release` specifies a git release tag. Defaults to `--release=all`. Make sure the GCR images with the tag have been created using `create_matrix_images.py` above.
- `--language` specifies a language. Defaults to `--language=all`.
For example, To test all languages for all gRPC releases across all runtimes, do `tools/interop_matrix/run_interop_matrix_test.py --release=all`.
- The output for all the test cases is recorded in a junit style xml file (default to 'report.xml').
diff --git a/tools/interop_matrix/create_testcases.sh b/tools/interop_matrix/create_testcases.sh
index d06fb34ff9..3d34b2ef25 100755
--- a/tools/interop_matrix/create_testcases.sh
+++ b/tools/interop_matrix/create_testcases.sh
@@ -31,9 +31,12 @@ TESTCASES_DIR=${GRPC_ROOT}/tools/interop_matrix/testcases
echo "Create '$LANG' test cases for gRPC release '${RELEASE:=master}'"
+echo $client_lang
# Invoke run_interop_test in manual mode.
+# TODO(adelez): Add cloud_gateways when we figure out how to skip them if not
+# running in GCE.
${GRPC_ROOT}/tools/run_tests/run_interop_tests.py -l $LANG --use_docker \
- --cloud_to_prod --manual_run
+ --cloud_to_prod --prod_servers default gateway_v4 --manual_run
# Clean up
function cleanup {
@@ -52,11 +55,18 @@ function cleanup {
[ -e "$CMDS_SH" ] && rm $CMDS_SH
}
trap cleanup EXIT
+# TODO(adelez): add test auth tests but do not run if not testing on GCE.
# Running the testcases as sanity unless we are asked to skip.
[ -z "$SKIP_TEST" ] && (echo "Running test cases: $CMDS_SH"; sh $CMDS_SH)
+# Convert c++ to cxx.
+if [$LANG == "c++" ]; then
+client_lang="cxx"
+else
+client_lang=$LANG
+fi
mkdir -p $TESTCASES_DIR
-testcase=$TESTCASES_DIR/${LANG}__$RELEASE
+testcase=$TESTCASES_DIR/${client_lang}__$RELEASE
if [ -e $testcase ]; then
echo "Updating: $testcase"
diff $testcase $CMDS_SH || true
diff --git a/tools/interop_matrix/run_interop_matrix_tests.py b/tools/interop_matrix/run_interop_matrix_tests.py
index 4315c8277d..510bc7124d 100755
--- a/tools/interop_matrix/run_interop_matrix_tests.py
+++ b/tools/interop_matrix/run_interop_matrix_tests.py
@@ -48,9 +48,8 @@ argp.add_argument('-j', '--jobs', default=multiprocessing.cpu_count(), type=int)
argp.add_argument('--gcr_path',
default='gcr.io/grpc-testing',
help='Path of docker images in Google Container Registry')
-
argp.add_argument('--release',
- default='master',
+ default='all',
choices=['all', 'master'] + _RELEASES,
help='Release tags to test. When testing all '
'releases defined in client_matrix.py, use "all".')
@@ -94,14 +93,15 @@ def find_all_images_for_lang(lang):
for runtime in client_matrix.LANG_RUNTIME_MATRIX[lang]:
image_path = '%s/grpc_interop_%s' % (args.gcr_path, runtime)
output = subprocess.check_output(['gcloud', 'beta', 'container', 'images',
- 'list-tags', '--format=json', image_path])
+ 'list-tags', '--format=json', image_path])
docker_image_list = json.loads(output)
# All images should have a single tag or no tag.
+ # TODO(adelez): Remove tagless images.
tags = [i['tags'][0] for i in docker_image_list if i['tags']]
jobset.message('START', 'Found images for %s: %s' % (image_path, tags),
do_newline=True)
skipped = len(docker_image_list) - len(tags)
- jobset.message('START', 'Skipped images (no-tag/unknown-tag): %d' % skipped,
+ jobset.message('SKIPPED', 'Skipped images (no-tag/unknown-tag): %d' % skipped,
do_newline=True)
# Filter tags based on the releases.
images[runtime] = [(tag,'%s:%s' % (image_path,tag)) for tag in tags if
@@ -148,6 +148,7 @@ def run_tests_for_lang(lang, runtime, images):
images is a list of (<release-tag>, <image-full-path>) tuple.
"""
+ total_num_failures = 0
for image_tuple in images:
release, image = image_tuple
jobset.message('START', 'Testing %s' % image, do_newline=True)
@@ -161,6 +162,7 @@ def run_tests_for_lang(lang, runtime, images):
maxjobs=args.jobs)
if num_failures:
jobset.message('FAILED', 'Some tests failed', do_newline=True)
+ total_num_failures += num_failures
else:
jobset.message('SUCCESS', 'All tests passed', do_newline=True)
@@ -170,6 +172,9 @@ def run_tests_for_lang(lang, runtime, images):
'grpc_interop_matrix',
'%s__%s %s'%(lang,runtime,release),
str(uuid.uuid4()))
+
+ return total_num_failures
+
_docker_images_cleanup = []
def cleanup():
@@ -180,9 +185,14 @@ def cleanup():
atexit.register(cleanup)
languages = args.language if args.language != ['all'] else _LANGUAGES
+total_num_failures = 0
for lang in languages:
docker_images = find_all_images_for_lang(lang)
for runtime in sorted(docker_images.keys()):
- run_tests_for_lang(lang, runtime, docker_images[runtime])
+ total_num_failures += run_tests_for_lang(lang, runtime, docker_images[runtime])
report_utils.create_xml_report_file(_xml_report_tree, args.report_file)
+
+if total_num_failures:
+ sys.exit(1)
+sys.exit(0)
diff --git a/tools/interop_matrix/testcases/cxx__master b/tools/interop_matrix/testcases/cxx__master
index ccd2859530..e0fed53f08 100755
--- a/tools/interop_matrix/testcases/cxx__master
+++ b/tools/interop_matrix/testcases/cxx__master
@@ -1,5 +1,5 @@
#!/bin/bash
-echo "Testing ${docker_image:=grpc_interop_cxx:1423f288-ac00-4f3a-9885-771258eecae3}"
+echo "Testing ${docker_image:=grpc_interop_cxx:78de6f80-524d-4bc9-bfb2-f00c24ceafed}"
docker run -i --rm=true -w /var/local/git/grpc --net=host $docker_image bash -c "bins/opt/interop_client --server_host=216.239.32.254 --server_host_override=grpc-test.sandbox.googleapis.com --server_port=443 --use_tls=true --test_case=large_unary"
docker run -i --rm=true -w /var/local/git/grpc --net=host $docker_image bash -c "bins/opt/interop_client --server_host=216.239.32.254 --server_host_override=grpc-test.sandbox.googleapis.com --server_port=443 --use_tls=true --test_case=empty_unary"
docker run -i --rm=true -w /var/local/git/grpc --net=host $docker_image bash -c "bins/opt/interop_client --server_host=216.239.32.254 --server_host_override=grpc-test.sandbox.googleapis.com --server_port=443 --use_tls=true --test_case=ping_pong"
@@ -9,3 +9,12 @@ docker run -i --rm=true -w /var/local/git/grpc --net=host $docker_image bash -c
docker run -i --rm=true -w /var/local/git/grpc --net=host $docker_image bash -c "bins/opt/interop_client --server_host=216.239.32.254 --server_host_override=grpc-test.sandbox.googleapis.com --server_port=443 --use_tls=true --test_case=cancel_after_begin"
docker run -i --rm=true -w /var/local/git/grpc --net=host $docker_image bash -c "bins/opt/interop_client --server_host=216.239.32.254 --server_host_override=grpc-test.sandbox.googleapis.com --server_port=443 --use_tls=true --test_case=cancel_after_first_response"
docker run -i --rm=true -w /var/local/git/grpc --net=host $docker_image bash -c "bins/opt/interop_client --server_host=216.239.32.254 --server_host_override=grpc-test.sandbox.googleapis.com --server_port=443 --use_tls=true --test_case=timeout_on_sleeping_server"
+docker run -i --rm=true -w /var/local/git/grpc --net=host $docker_image bash -c "bins/opt/interop_client --server_host=216.239.32.254 --server_host_override=grpc-test4.sandbox.googleapis.com --server_port=443 --use_tls=true --test_case=large_unary"
+docker run -i --rm=true -w /var/local/git/grpc --net=host $docker_image bash -c "bins/opt/interop_client --server_host=216.239.32.254 --server_host_override=grpc-test4.sandbox.googleapis.com --server_port=443 --use_tls=true --test_case=empty_unary"
+docker run -i --rm=true -w /var/local/git/grpc --net=host $docker_image bash -c "bins/opt/interop_client --server_host=216.239.32.254 --server_host_override=grpc-test4.sandbox.googleapis.com --server_port=443 --use_tls=true --test_case=ping_pong"
+docker run -i --rm=true -w /var/local/git/grpc --net=host $docker_image bash -c "bins/opt/interop_client --server_host=216.239.32.254 --server_host_override=grpc-test4.sandbox.googleapis.com --server_port=443 --use_tls=true --test_case=empty_stream"
+docker run -i --rm=true -w /var/local/git/grpc --net=host $docker_image bash -c "bins/opt/interop_client --server_host=216.239.32.254 --server_host_override=grpc-test4.sandbox.googleapis.com --server_port=443 --use_tls=true --test_case=client_streaming"
+docker run -i --rm=true -w /var/local/git/grpc --net=host $docker_image bash -c "bins/opt/interop_client --server_host=216.239.32.254 --server_host_override=grpc-test4.sandbox.googleapis.com --server_port=443 --use_tls=true --test_case=server_streaming"
+docker run -i --rm=true -w /var/local/git/grpc --net=host $docker_image bash -c "bins/opt/interop_client --server_host=216.239.32.254 --server_host_override=grpc-test4.sandbox.googleapis.com --server_port=443 --use_tls=true --test_case=cancel_after_begin"
+docker run -i --rm=true -w /var/local/git/grpc --net=host $docker_image bash -c "bins/opt/interop_client --server_host=216.239.32.254 --server_host_override=grpc-test4.sandbox.googleapis.com --server_port=443 --use_tls=true --test_case=cancel_after_first_response"
+docker run -i --rm=true -w /var/local/git/grpc --net=host $docker_image bash -c "bins/opt/interop_client --server_host=216.239.32.254 --server_host_override=grpc-test4.sandbox.googleapis.com --server_port=443 --use_tls=true --test_case=timeout_on_sleeping_server"
diff --git a/tools/interop_matrix/testcases/go__master b/tools/interop_matrix/testcases/go__master
index 2624c7f92c..33b25d6a16 100755
--- a/tools/interop_matrix/testcases/go__master
+++ b/tools/interop_matrix/testcases/go__master
@@ -1,5 +1,5 @@
#!/bin/bash
-echo "Testing ${docker_image:=grpc_interop_go:41fffd01-a6c8-41b6-8136-c0aaa1ec2437}"
+echo "Testing ${docker_image:=grpc_interop_go:dd8fbf3a-4964-4387-9997-5dadeea09835}"
docker run -i --rm=true -w /go/src/google.golang.org/grpc/interop/client --net=host $docker_image bash -c "go run client.go --server_host=216.239.32.254 --server_host_override=grpc-test.sandbox.googleapis.com --server_port=443 --use_tls=true --test_case=large_unary"
docker run -i --rm=true -w /go/src/google.golang.org/grpc/interop/client --net=host $docker_image bash -c "go run client.go --server_host=216.239.32.254 --server_host_override=grpc-test.sandbox.googleapis.com --server_port=443 --use_tls=true --test_case=empty_unary"
docker run -i --rm=true -w /go/src/google.golang.org/grpc/interop/client --net=host $docker_image bash -c "go run client.go --server_host=216.239.32.254 --server_host_override=grpc-test.sandbox.googleapis.com --server_port=443 --use_tls=true --test_case=ping_pong"
@@ -9,3 +9,12 @@ docker run -i --rm=true -w /go/src/google.golang.org/grpc/interop/client --net=h
docker run -i --rm=true -w /go/src/google.golang.org/grpc/interop/client --net=host $docker_image bash -c "go run client.go --server_host=216.239.32.254 --server_host_override=grpc-test.sandbox.googleapis.com --server_port=443 --use_tls=true --test_case=cancel_after_begin"
docker run -i --rm=true -w /go/src/google.golang.org/grpc/interop/client --net=host $docker_image bash -c "go run client.go --server_host=216.239.32.254 --server_host_override=grpc-test.sandbox.googleapis.com --server_port=443 --use_tls=true --test_case=cancel_after_first_response"
docker run -i --rm=true -w /go/src/google.golang.org/grpc/interop/client --net=host $docker_image bash -c "go run client.go --server_host=216.239.32.254 --server_host_override=grpc-test.sandbox.googleapis.com --server_port=443 --use_tls=true --test_case=timeout_on_sleeping_server"
+docker run -i --rm=true -w /go/src/google.golang.org/grpc/interop/client --net=host $docker_image bash -c "go run client.go --server_host=216.239.32.254 --server_host_override=grpc-test4.sandbox.googleapis.com --server_port=443 --use_tls=true --test_case=large_unary"
+docker run -i --rm=true -w /go/src/google.golang.org/grpc/interop/client --net=host $docker_image bash -c "go run client.go --server_host=216.239.32.254 --server_host_override=grpc-test4.sandbox.googleapis.com --server_port=443 --use_tls=true --test_case=empty_unary"
+docker run -i --rm=true -w /go/src/google.golang.org/grpc/interop/client --net=host $docker_image bash -c "go run client.go --server_host=216.239.32.254 --server_host_override=grpc-test4.sandbox.googleapis.com --server_port=443 --use_tls=true --test_case=ping_pong"
+docker run -i --rm=true -w /go/src/google.golang.org/grpc/interop/client --net=host $docker_image bash -c "go run client.go --server_host=216.239.32.254 --server_host_override=grpc-test4.sandbox.googleapis.com --server_port=443 --use_tls=true --test_case=empty_stream"
+docker run -i --rm=true -w /go/src/google.golang.org/grpc/interop/client --net=host $docker_image bash -c "go run client.go --server_host=216.239.32.254 --server_host_override=grpc-test4.sandbox.googleapis.com --server_port=443 --use_tls=true --test_case=client_streaming"
+docker run -i --rm=true -w /go/src/google.golang.org/grpc/interop/client --net=host $docker_image bash -c "go run client.go --server_host=216.239.32.254 --server_host_override=grpc-test4.sandbox.googleapis.com --server_port=443 --use_tls=true --test_case=server_streaming"
+docker run -i --rm=true -w /go/src/google.golang.org/grpc/interop/client --net=host $docker_image bash -c "go run client.go --server_host=216.239.32.254 --server_host_override=grpc-test4.sandbox.googleapis.com --server_port=443 --use_tls=true --test_case=cancel_after_begin"
+docker run -i --rm=true -w /go/src/google.golang.org/grpc/interop/client --net=host $docker_image bash -c "go run client.go --server_host=216.239.32.254 --server_host_override=grpc-test4.sandbox.googleapis.com --server_port=443 --use_tls=true --test_case=cancel_after_first_response"
+docker run -i --rm=true -w /go/src/google.golang.org/grpc/interop/client --net=host $docker_image bash -c "go run client.go --server_host=216.239.32.254 --server_host_override=grpc-test4.sandbox.googleapis.com --server_port=443 --use_tls=true --test_case=timeout_on_sleeping_server"
diff --git a/tools/interop_matrix/testcases/java__master b/tools/interop_matrix/testcases/java__master
index cf431646e9..dbd87279a6 100755
--- a/tools/interop_matrix/testcases/java__master
+++ b/tools/interop_matrix/testcases/java__master
@@ -1,5 +1,5 @@
#!/bin/bash
-echo "Testing ${docker_image:=grpc_interop_java:ea528843-be34-4ff3-a136-e4609424e061}"
+echo "Testing ${docker_image:=grpc_interop_java:a764b50c-1788-4387-9b9e-5cfa93927006}"
docker run -i --rm=true -w /var/local/git/grpc/../grpc-java --net=host $docker_image bash -c "./run-test-client.sh --server_host=216.239.32.254 --server_host_override=grpc-test.sandbox.googleapis.com --server_port=443 --use_tls=true --test_case=large_unary"
docker run -i --rm=true -w /var/local/git/grpc/../grpc-java --net=host $docker_image bash -c "./run-test-client.sh --server_host=216.239.32.254 --server_host_override=grpc-test.sandbox.googleapis.com --server_port=443 --use_tls=true --test_case=empty_unary"
docker run -i --rm=true -w /var/local/git/grpc/../grpc-java --net=host $docker_image bash -c "./run-test-client.sh --server_host=216.239.32.254 --server_host_override=grpc-test.sandbox.googleapis.com --server_port=443 --use_tls=true --test_case=ping_pong"
@@ -9,3 +9,12 @@ docker run -i --rm=true -w /var/local/git/grpc/../grpc-java --net=host $docker_i
docker run -i --rm=true -w /var/local/git/grpc/../grpc-java --net=host $docker_image bash -c "./run-test-client.sh --server_host=216.239.32.254 --server_host_override=grpc-test.sandbox.googleapis.com --server_port=443 --use_tls=true --test_case=cancel_after_begin"
docker run -i --rm=true -w /var/local/git/grpc/../grpc-java --net=host $docker_image bash -c "./run-test-client.sh --server_host=216.239.32.254 --server_host_override=grpc-test.sandbox.googleapis.com --server_port=443 --use_tls=true --test_case=cancel_after_first_response"
docker run -i --rm=true -w /var/local/git/grpc/../grpc-java --net=host $docker_image bash -c "./run-test-client.sh --server_host=216.239.32.254 --server_host_override=grpc-test.sandbox.googleapis.com --server_port=443 --use_tls=true --test_case=timeout_on_sleeping_server"
+docker run -i --rm=true -w /var/local/git/grpc/../grpc-java --net=host $docker_image bash -c "./run-test-client.sh --server_host=216.239.32.254 --server_host_override=grpc-test4.sandbox.googleapis.com --server_port=443 --use_tls=true --test_case=large_unary"
+docker run -i --rm=true -w /var/local/git/grpc/../grpc-java --net=host $docker_image bash -c "./run-test-client.sh --server_host=216.239.32.254 --server_host_override=grpc-test4.sandbox.googleapis.com --server_port=443 --use_tls=true --test_case=empty_unary"
+docker run -i --rm=true -w /var/local/git/grpc/../grpc-java --net=host $docker_image bash -c "./run-test-client.sh --server_host=216.239.32.254 --server_host_override=grpc-test4.sandbox.googleapis.com --server_port=443 --use_tls=true --test_case=ping_pong"
+docker run -i --rm=true -w /var/local/git/grpc/../grpc-java --net=host $docker_image bash -c "./run-test-client.sh --server_host=216.239.32.254 --server_host_override=grpc-test4.sandbox.googleapis.com --server_port=443 --use_tls=true --test_case=empty_stream"
+docker run -i --rm=true -w /var/local/git/grpc/../grpc-java --net=host $docker_image bash -c "./run-test-client.sh --server_host=216.239.32.254 --server_host_override=grpc-test4.sandbox.googleapis.com --server_port=443 --use_tls=true --test_case=client_streaming"
+docker run -i --rm=true -w /var/local/git/grpc/../grpc-java --net=host $docker_image bash -c "./run-test-client.sh --server_host=216.239.32.254 --server_host_override=grpc-test4.sandbox.googleapis.com --server_port=443 --use_tls=true --test_case=server_streaming"
+docker run -i --rm=true -w /var/local/git/grpc/../grpc-java --net=host $docker_image bash -c "./run-test-client.sh --server_host=216.239.32.254 --server_host_override=grpc-test4.sandbox.googleapis.com --server_port=443 --use_tls=true --test_case=cancel_after_begin"
+docker run -i --rm=true -w /var/local/git/grpc/../grpc-java --net=host $docker_image bash -c "./run-test-client.sh --server_host=216.239.32.254 --server_host_override=grpc-test4.sandbox.googleapis.com --server_port=443 --use_tls=true --test_case=cancel_after_first_response"
+docker run -i --rm=true -w /var/local/git/grpc/../grpc-java --net=host $docker_image bash -c "./run-test-client.sh --server_host=216.239.32.254 --server_host_override=grpc-test4.sandbox.googleapis.com --server_port=443 --use_tls=true --test_case=timeout_on_sleeping_server"