diff options
| -rw-r--r-- | build.json | 3 | ||||
| -rwxr-xr-x | test/core/end2end/gen_build_json.py | 114 | ||||
| -rw-r--r-- | tools/run_tests/tests.json | 34 |
3 files changed, 77 insertions, 74 deletions
diff --git a/build.json b/build.json index ec48314d26..5144bb74b6 100644 --- a/build.json +++ b/build.json @@ -811,6 +811,7 @@ }, { "name": "census_statistics_multiple_writers_test", + "flaky": true, "build": "test", "language": "c", "src": [ @@ -825,6 +826,7 @@ }, { "name": "census_statistics_performance_test", + "flaky": true, "build": "test", "language": "c", "src": [ @@ -839,6 +841,7 @@ }, { "name": "census_statistics_quick_test", + "flaky": true, "build": "test", "language": "c", "src": [ diff --git a/test/core/end2end/gen_build_json.py b/test/core/end2end/gen_build_json.py index b4a45ea485..9b7db416b8 100755 --- a/test/core/end2end/gen_build_json.py +++ b/test/core/end2end/gen_build_json.py @@ -46,60 +46,60 @@ END2END_FIXTURES = { 'chttp2_socket_pair_one_byte_at_a_time': False, } +# maps tests names to whether they run fine or not (aka, not flaky) +END2END_TESTS = { + 'bad_hostname': True, + 'cancel_after_accept': False, + 'cancel_after_accept_and_writes_closed': True, + 'cancel_after_invoke': True, + 'cancel_before_invoke': True, + 'cancel_in_a_vacuum': True, + 'census_simple_request': True, + 'disappearing_server': True, + 'early_server_shutdown_finishes_inflight_calls': True, + 'early_server_shutdown_finishes_tags': True, + 'empty_batch': True, + 'graceful_server_shutdown': True, + 'invoke_large_request': False, + 'max_concurrent_streams': True, + 'no_op': True, + 'ping_pong_streaming': True, + 'request_response_with_binary_metadata_and_payload': True, + 'request_response_with_metadata_and_payload': True, + 'request_response_with_payload': True, + 'request_with_large_metadata': True, + 'request_with_payload': True, + 'simple_delayed_request': True, + 'simple_request': True, + 'registered_call': True, + 'thread_stress': True, + 'writes_done_hangs_with_pending_read': True, -END2END_TESTS = [ - 'bad_hostname', - 'cancel_after_accept', - 'cancel_after_accept_and_writes_closed', - 'cancel_after_invoke', - 'cancel_before_invoke', - 'cancel_in_a_vacuum', - 'census_simple_request', - 'disappearing_server', - 'early_server_shutdown_finishes_inflight_calls', - 'early_server_shutdown_finishes_tags', - 'empty_batch', - 'graceful_server_shutdown', - 'invoke_large_request', - 'max_concurrent_streams', - 'no_op', - 'ping_pong_streaming', - 'request_response_with_binary_metadata_and_payload', - 'request_response_with_metadata_and_payload', - 'request_response_with_payload', - 'request_with_large_metadata', - 'request_with_payload', - 'simple_delayed_request', - 'simple_request', - 'registered_call', - 'thread_stress', - 'writes_done_hangs_with_pending_read', - - 'cancel_after_accept_legacy', - 'cancel_after_accept_and_writes_closed_legacy', - 'cancel_after_invoke_legacy', - 'cancel_before_invoke_legacy', - 'cancel_in_a_vacuum_legacy', - 'census_simple_request_legacy', - 'disappearing_server_legacy', - 'early_server_shutdown_finishes_inflight_calls_legacy', - 'early_server_shutdown_finishes_tags_legacy', - 'graceful_server_shutdown_legacy', - 'invoke_large_request_legacy', - 'max_concurrent_streams_legacy', - 'no_op_legacy', - 'ping_pong_streaming_legacy', - 'request_response_with_binary_metadata_and_payload_legacy', - 'request_response_with_metadata_and_payload_legacy', - 'request_response_with_payload_legacy', - 'request_response_with_trailing_metadata_and_payload_legacy', - 'request_with_large_metadata_legacy', - 'request_with_payload_legacy', - 'simple_delayed_request_legacy', - 'simple_request_legacy', - 'thread_stress_legacy', - 'writes_done_hangs_with_pending_read_legacy', -] + 'cancel_after_accept_legacy': False, + 'cancel_after_accept_and_writes_closed_legacy': True, + 'cancel_after_invoke_legacy': True, + 'cancel_before_invoke_legacy': True, + 'cancel_in_a_vacuum_legacy': True, + 'census_simple_request_legacy': True, + 'disappearing_server_legacy': True, + 'early_server_shutdown_finishes_inflight_calls_legacy': True, + 'early_server_shutdown_finishes_tags_legacy': True, + 'graceful_server_shutdown_legacy': True, + 'invoke_large_request_legacy': False, + 'max_concurrent_streams_legacy': True, + 'no_op_legacy': True, + 'ping_pong_streaming_legacy': True, + 'request_response_with_binary_metadata_and_payload_legacy': True, + 'request_response_with_metadata_and_payload_legacy': True, + 'request_response_with_payload_legacy': True, + 'request_response_with_trailing_metadata_and_payload_legacy': True, + 'request_with_large_metadata_legacy': True, + 'request_with_payload_legacy': True, + 'simple_delayed_request_legacy': True, + 'simple_request_legacy': True, + 'thread_stress_legacy': True, + 'writes_done_hangs_with_pending_read_legacy': True, +} def main(): @@ -122,7 +122,7 @@ def main(): 'src': ['test/core/end2end/tests/%s.c' % t], 'headers': ['test/core/end2end/tests/cancel_test_helpers.h'] } - for t in sorted(END2END_TESTS)] + [ + for t in sorted(END2END_TESTS.keys())] + [ { 'name': 'end2end_certs', 'build': 'private', @@ -140,7 +140,7 @@ def main(): 'build': 'test', 'language': 'c', 'src': [], - 'flaky': 'invoke_large_request' in t, + 'flaky': not END2END_TESTS[t], 'deps': [ 'end2end_fixture_%s' % f, 'end2end_test_%s' % t, @@ -152,7 +152,7 @@ def main(): ] } for f in sorted(END2END_FIXTURES.keys()) - for t in sorted(END2END_TESTS)] + [ + for t in sorted(END2END_TESTS.keys())] + [ { 'name': '%s_%s_unsecure_test' % (f, t), 'build': 'test', @@ -170,7 +170,7 @@ def main(): ] } for f in sorted(END2END_FIXTURES.keys()) if not END2END_FIXTURES[f] - for t in sorted(END2END_TESTS)]} + for t in sorted(END2END_TESTS.keys())]} print simplejson.dumps(json, sort_keys=True, indent=2 * ' ') diff --git a/tools/run_tests/tests.json b/tools/run_tests/tests.json index ad994e99a3..68b22d7311 100644 --- a/tools/run_tests/tests.json +++ b/tools/run_tests/tests.json @@ -65,7 +65,7 @@ ] }, { - "flaky": false, + "flaky": true, "language": "c", "name": "census_statistics_multiple_writers_test", "platforms": [ @@ -74,7 +74,7 @@ ] }, { - "flaky": false, + "flaky": true, "language": "c", "name": "census_statistics_performance_test", "platforms": [ @@ -83,7 +83,7 @@ ] }, { - "flaky": false, + "flaky": true, "language": "c", "name": "census_statistics_quick_test", "platforms": [ @@ -703,7 +703,7 @@ ] }, { - "flaky": false, + "flaky": true, "language": "c", "name": "chttp2_fake_security_cancel_after_accept_test", "platforms": [ @@ -730,7 +730,7 @@ ] }, { - "flaky": false, + "flaky": true, "language": "c", "name": "chttp2_fake_security_cancel_after_accept_legacy_test", "platforms": [ @@ -1153,7 +1153,7 @@ ] }, { - "flaky": false, + "flaky": true, "language": "c", "name": "chttp2_fullstack_cancel_after_accept_test", "platforms": [ @@ -1180,7 +1180,7 @@ ] }, { - "flaky": false, + "flaky": true, "language": "c", "name": "chttp2_fullstack_cancel_after_accept_legacy_test", "platforms": [ @@ -1603,7 +1603,7 @@ ] }, { - "flaky": false, + "flaky": true, "language": "c", "name": "chttp2_fullstack_uds_cancel_after_accept_test", "platforms": [ @@ -1630,7 +1630,7 @@ ] }, { - "flaky": false, + "flaky": true, "language": "c", "name": "chttp2_fullstack_uds_cancel_after_accept_legacy_test", "platforms": [ @@ -2053,7 +2053,7 @@ ] }, { - "flaky": false, + "flaky": true, "language": "c", "name": "chttp2_simple_ssl_fullstack_cancel_after_accept_test", "platforms": [ @@ -2080,7 +2080,7 @@ ] }, { - "flaky": false, + "flaky": true, "language": "c", "name": "chttp2_simple_ssl_fullstack_cancel_after_accept_legacy_test", "platforms": [ @@ -2503,7 +2503,7 @@ ] }, { - "flaky": false, + "flaky": true, "language": "c", "name": "chttp2_simple_ssl_with_oauth2_fullstack_cancel_after_accept_test", "platforms": [ @@ -2530,7 +2530,7 @@ ] }, { - "flaky": false, + "flaky": true, "language": "c", "name": "chttp2_simple_ssl_with_oauth2_fullstack_cancel_after_accept_legacy_test", "platforms": [ @@ -2953,7 +2953,7 @@ ] }, { - "flaky": false, + "flaky": true, "language": "c", "name": "chttp2_socket_pair_cancel_after_accept_test", "platforms": [ @@ -2980,7 +2980,7 @@ ] }, { - "flaky": false, + "flaky": true, "language": "c", "name": "chttp2_socket_pair_cancel_after_accept_legacy_test", "platforms": [ @@ -3403,7 +3403,7 @@ ] }, { - "flaky": false, + "flaky": true, "language": "c", "name": "chttp2_socket_pair_one_byte_at_a_time_cancel_after_accept_test", "platforms": [ @@ -3430,7 +3430,7 @@ ] }, { - "flaky": false, + "flaky": true, "language": "c", "name": "chttp2_socket_pair_one_byte_at_a_time_cancel_after_accept_legacy_test", "platforms": [ |