Merge pull request #1360 from nicolasnoble/flakes

Flagging some tests as flaky.

3 files changed, 77 insertions, 74 deletions

diff --git a/build.json b/build.json
index ec48314d26..5144bb74b6 100644
--- a/build.json
+++ b/build.json
@@ -811,6 +811,7 @@
     },
     {
       "name": "census_statistics_multiple_writers_test",
+      "flaky": true,
       "build": "test",
       "language": "c",
       "src": [
@@ -825,6 +826,7 @@
     },
     {
       "name": "census_statistics_performance_test",
+      "flaky": true,
       "build": "test",
       "language": "c",
       "src": [
@@ -839,6 +841,7 @@
     },
     {
       "name": "census_statistics_quick_test",
+      "flaky": true,
       "build": "test",
       "language": "c",
       "src": [
diff --git a/test/core/end2end/gen_build_json.py b/test/core/end2end/gen_build_json.py
index b4a45ea485..9b7db416b8 100755
--- a/test/core/end2end/gen_build_json.py
+++ b/test/core/end2end/gen_build_json.py
@@ -46,60 +46,60 @@ END2END_FIXTURES = {
     'chttp2_socket_pair_one_byte_at_a_time': False,
 }
 
+# maps tests names to whether they run fine or not (aka, not flaky)
+END2END_TESTS = {
+    'bad_hostname': True,
+    'cancel_after_accept': False,
+    'cancel_after_accept_and_writes_closed': True,
+    'cancel_after_invoke': True,
+    'cancel_before_invoke': True,
+    'cancel_in_a_vacuum': True,
+    'census_simple_request': True,
+    'disappearing_server': True,
+    'early_server_shutdown_finishes_inflight_calls': True,
+    'early_server_shutdown_finishes_tags': True,
+    'empty_batch': True,
+    'graceful_server_shutdown': True,
+    'invoke_large_request': False,
+    'max_concurrent_streams': True,
+    'no_op': True,
+    'ping_pong_streaming': True,
+    'request_response_with_binary_metadata_and_payload': True,
+    'request_response_with_metadata_and_payload': True,
+    'request_response_with_payload': True,
+    'request_with_large_metadata': True,
+    'request_with_payload': True,
+    'simple_delayed_request': True,
+    'simple_request': True,
+    'registered_call': True,
+    'thread_stress': True,
+    'writes_done_hangs_with_pending_read': True,
 
-END2END_TESTS = [
-    'bad_hostname',
-    'cancel_after_accept',
-    'cancel_after_accept_and_writes_closed',
-    'cancel_after_invoke',
-    'cancel_before_invoke',
-    'cancel_in_a_vacuum',
-    'census_simple_request',
-    'disappearing_server',
-    'early_server_shutdown_finishes_inflight_calls',
-    'early_server_shutdown_finishes_tags',
-    'empty_batch',
-    'graceful_server_shutdown',
-    'invoke_large_request',
-    'max_concurrent_streams',
-    'no_op',
-    'ping_pong_streaming',
-    'request_response_with_binary_metadata_and_payload',
-    'request_response_with_metadata_and_payload',
-    'request_response_with_payload',
-    'request_with_large_metadata',
-    'request_with_payload',
-    'simple_delayed_request',
-    'simple_request',
-    'registered_call',
-    'thread_stress',
-    'writes_done_hangs_with_pending_read',
-
-    'cancel_after_accept_legacy',
-    'cancel_after_accept_and_writes_closed_legacy',
-    'cancel_after_invoke_legacy',
-    'cancel_before_invoke_legacy',
-    'cancel_in_a_vacuum_legacy',
-    'census_simple_request_legacy',
-    'disappearing_server_legacy',
-    'early_server_shutdown_finishes_inflight_calls_legacy',
-    'early_server_shutdown_finishes_tags_legacy',
-    'graceful_server_shutdown_legacy',
-    'invoke_large_request_legacy',
-    'max_concurrent_streams_legacy',
-    'no_op_legacy',
-    'ping_pong_streaming_legacy',
-    'request_response_with_binary_metadata_and_payload_legacy',
-    'request_response_with_metadata_and_payload_legacy',
-    'request_response_with_payload_legacy',
-    'request_response_with_trailing_metadata_and_payload_legacy',
-    'request_with_large_metadata_legacy',
-    'request_with_payload_legacy',
-    'simple_delayed_request_legacy',
-    'simple_request_legacy',
-    'thread_stress_legacy',
-    'writes_done_hangs_with_pending_read_legacy',
-]
+    'cancel_after_accept_legacy': False,
+    'cancel_after_accept_and_writes_closed_legacy': True,
+    'cancel_after_invoke_legacy': True,
+    'cancel_before_invoke_legacy': True,
+    'cancel_in_a_vacuum_legacy': True,
+    'census_simple_request_legacy': True,
+    'disappearing_server_legacy': True,
+    'early_server_shutdown_finishes_inflight_calls_legacy': True,
+    'early_server_shutdown_finishes_tags_legacy': True,
+    'graceful_server_shutdown_legacy': True,
+    'invoke_large_request_legacy': False,
+    'max_concurrent_streams_legacy': True,
+    'no_op_legacy': True,
+    'ping_pong_streaming_legacy': True,
+    'request_response_with_binary_metadata_and_payload_legacy': True,
+    'request_response_with_metadata_and_payload_legacy': True,
+    'request_response_with_payload_legacy': True,
+    'request_response_with_trailing_metadata_and_payload_legacy': True,
+    'request_with_large_metadata_legacy': True,
+    'request_with_payload_legacy': True,
+    'simple_delayed_request_legacy': True,
+    'simple_request_legacy': True,
+    'thread_stress_legacy': True,
+    'writes_done_hangs_with_pending_read_legacy': True,
+}
 
 
 def main():
@@ -122,7 +122,7 @@ def main():
               'src': ['test/core/end2end/tests/%s.c' % t],
               'headers': ['test/core/end2end/tests/cancel_test_helpers.h']
           }
-          for t in sorted(END2END_TESTS)] + [
+          for t in sorted(END2END_TESTS.keys())] + [
           {
               'name': 'end2end_certs',
               'build': 'private',
@@ -140,7 +140,7 @@ def main():
               'build': 'test',
               'language': 'c',
               'src': [],
-              'flaky': 'invoke_large_request' in t,
+              'flaky': not END2END_TESTS[t],
               'deps': [
                   'end2end_fixture_%s' % f,
                   'end2end_test_%s' % t,
@@ -152,7 +152,7 @@ def main():
               ]
           }
       for f in sorted(END2END_FIXTURES.keys())
-      for t in sorted(END2END_TESTS)] + [
+      for t in sorted(END2END_TESTS.keys())] + [
           {
               'name': '%s_%s_unsecure_test' % (f, t),
               'build': 'test',
@@ -170,7 +170,7 @@ def main():
               ]
           }
       for f in sorted(END2END_FIXTURES.keys()) if not END2END_FIXTURES[f]
-      for t in sorted(END2END_TESTS)]}
+      for t in sorted(END2END_TESTS.keys())]}
   print simplejson.dumps(json, sort_keys=True, indent=2 * ' ')
 
 
diff --git a/tools/run_tests/tests.json b/tools/run_tests/tests.json
index ad994e99a3..68b22d7311 100644
--- a/tools/run_tests/tests.json
+++ b/tools/run_tests/tests.json
@@ -65,7 +65,7 @@
     ]
   }, 
   {
-    "flaky": false, 
+    "flaky": true, 
     "language": "c", 
     "name": "census_statistics_multiple_writers_test", 
     "platforms": [
@@ -74,7 +74,7 @@
     ]
   }, 
   {
-    "flaky": false, 
+    "flaky": true, 
     "language": "c", 
     "name": "census_statistics_performance_test", 
     "platforms": [
@@ -83,7 +83,7 @@
     ]
   }, 
   {
-    "flaky": false, 
+    "flaky": true, 
     "language": "c", 
     "name": "census_statistics_quick_test", 
     "platforms": [
@@ -703,7 +703,7 @@
     ]
   }, 
   {
-    "flaky": false, 
+    "flaky": true, 
     "language": "c", 
     "name": "chttp2_fake_security_cancel_after_accept_test", 
     "platforms": [
@@ -730,7 +730,7 @@
     ]
   }, 
   {
-    "flaky": false, 
+    "flaky": true, 
     "language": "c", 
     "name": "chttp2_fake_security_cancel_after_accept_legacy_test", 
     "platforms": [
@@ -1153,7 +1153,7 @@
     ]
   }, 
   {
-    "flaky": false, 
+    "flaky": true, 
     "language": "c", 
     "name": "chttp2_fullstack_cancel_after_accept_test", 
     "platforms": [
@@ -1180,7 +1180,7 @@
     ]
   }, 
   {
-    "flaky": false, 
+    "flaky": true, 
     "language": "c", 
     "name": "chttp2_fullstack_cancel_after_accept_legacy_test", 
     "platforms": [
@@ -1603,7 +1603,7 @@
     ]
   }, 
   {
-    "flaky": false, 
+    "flaky": true, 
     "language": "c", 
     "name": "chttp2_fullstack_uds_cancel_after_accept_test", 
     "platforms": [
@@ -1630,7 +1630,7 @@
     ]
   }, 
   {
-    "flaky": false, 
+    "flaky": true, 
     "language": "c", 
     "name": "chttp2_fullstack_uds_cancel_after_accept_legacy_test", 
     "platforms": [
@@ -2053,7 +2053,7 @@
     ]
   }, 
   {
-    "flaky": false, 
+    "flaky": true, 
     "language": "c", 
     "name": "chttp2_simple_ssl_fullstack_cancel_after_accept_test", 
     "platforms": [
@@ -2080,7 +2080,7 @@
     ]
   }, 
   {
-    "flaky": false, 
+    "flaky": true, 
     "language": "c", 
     "name": "chttp2_simple_ssl_fullstack_cancel_after_accept_legacy_test", 
     "platforms": [
@@ -2503,7 +2503,7 @@
     ]
   }, 
   {
-    "flaky": false, 
+    "flaky": true, 
     "language": "c", 
     "name": "chttp2_simple_ssl_with_oauth2_fullstack_cancel_after_accept_test", 
     "platforms": [
@@ -2530,7 +2530,7 @@
     ]
   }, 
   {
-    "flaky": false, 
+    "flaky": true, 
     "language": "c", 
     "name": "chttp2_simple_ssl_with_oauth2_fullstack_cancel_after_accept_legacy_test", 
     "platforms": [
@@ -2953,7 +2953,7 @@
     ]
   }, 
   {
-    "flaky": false, 
+    "flaky": true, 
     "language": "c", 
     "name": "chttp2_socket_pair_cancel_after_accept_test", 
     "platforms": [
@@ -2980,7 +2980,7 @@
     ]
   }, 
   {
-    "flaky": false, 
+    "flaky": true, 
     "language": "c", 
     "name": "chttp2_socket_pair_cancel_after_accept_legacy_test", 
     "platforms": [
@@ -3403,7 +3403,7 @@
     ]
   }, 
   {
-    "flaky": false, 
+    "flaky": true, 
     "language": "c", 
     "name": "chttp2_socket_pair_one_byte_at_a_time_cancel_after_accept_test", 
     "platforms": [
@@ -3430,7 +3430,7 @@
     ]
   }, 
   {
-    "flaky": false, 
+    "flaky": true, 
     "language": "c", 
     "name": "chttp2_socket_pair_one_byte_at_a_time_cancel_after_accept_legacy_test", 
     "platforms": [