From 4cbdf08e61b2f168a42a8984b58126aebdb9097e Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Fri, 4 Jan 2019 10:33:18 +0100 Subject: debug out of disk space in run_interop_matrix_tests.py --- tools/internal_ci/linux/grpc_interop_matrix.sh | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/tools/internal_ci/linux/grpc_interop_matrix.sh b/tools/internal_ci/linux/grpc_interop_matrix.sh index a5220ea087..5f1f69db95 100755 --- a/tools/internal_ci/linux/grpc_interop_matrix.sh +++ b/tools/internal_ci/linux/grpc_interop_matrix.sh @@ -22,4 +22,15 @@ cd $(dirname $0)/../../.. source tools/internal_ci/helper_scripts/prepare_build_linux_rc -tools/interop_matrix/run_interop_matrix_tests.py $RUN_TESTS_FLAGS +# TODO(jtattermusch): Diagnose out of disk space problems. Remove once not needed. +df -h + +tools/interop_matrix/run_interop_matrix_tests.py $RUN_TESTS_FLAGS || FAILED="true" + +# TODO(jtattermusch): Diagnose out of disk space problems. Remove once not needed. +df -h + +if [ "$FAILED" != "" ] +then + exit 1 +fi -- cgit v1.2.3 From b0663e7e1ebb164982fc3ca95c453c4b4dad5964 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Fri, 4 Jan 2019 12:31:34 +0100 Subject: interop_matrix: reliable cleanup of interop docker images --- tools/interop_matrix/run_interop_matrix_tests.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/tools/interop_matrix/run_interop_matrix_tests.py b/tools/interop_matrix/run_interop_matrix_tests.py index 6cf2a9b036..d47b0ee0f7 100755 --- a/tools/interop_matrix/run_interop_matrix_tests.py +++ b/tools/interop_matrix/run_interop_matrix_tests.py @@ -232,10 +232,13 @@ def _run_tests_for_lang(lang, runtime, images, xml_report_tree): images is a list of (, ) tuple. """ + skip_tests = False if not _pull_images_for_lang(lang, images): jobset.message( - 'FAILED', 'Image download failed. Exiting.', do_newline=True) - return 1 + 'FAILED', + 'Image download failed. Skipping tests for language "%s"' % lang, + do_newline=True) + skip_tests = True total_num_failures = 0 for release, image in images: @@ -246,17 +249,22 @@ def _run_tests_for_lang(lang, runtime, images, xml_report_tree): if not job_spec_list: jobset.message( 'FAILED', 'No test cases were found.', do_newline=True) - return 1 + total_num_failures += 1 + continue num_failures, resultset = jobset.run( job_spec_list, newline_on_success=True, add_env={'docker_image': image}, - maxjobs=args.jobs) + maxjobs=args.jobs, + skip_jobs=skip_tests) if args.bq_result_table and resultset: upload_test_results.upload_interop_results_to_bq( resultset, args.bq_result_table) - if num_failures: + if skip_tests: + jobset.message('FAILED', 'Tests were skipped', do_newline=True) + total_num_failures += 1 + elif num_failures: jobset.message('FAILED', 'Some tests failed', do_newline=True) total_num_failures += num_failures else: @@ -266,6 +274,8 @@ def _run_tests_for_lang(lang, runtime, images, xml_report_tree): 'grpc_interop_matrix', suite_name, str(uuid.uuid4())) + # cleanup all downloaded docker images + for _, image in images: if not args.keep: _cleanup_docker_image(image) -- cgit v1.2.3