diff options
| -rw-r--r-- | tools/run_tests/generated/tests.json | 488 | ||||
| -rw-r--r-- | tools/run_tests/performance/scenario_config.py | 21 |
2 files changed, 12 insertions, 497 deletions
diff --git a/tools/run_tests/generated/tests.json b/tools/run_tests/generated/tests.json index f9e2a19d0e..ec55fb73b9 100644 --- a/tools/run_tests/generated/tests.json +++ b/tools/run_tests/generated/tests.json @@ -39639,31 +39639,6 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_unary_qps_unconstrained_secure_500kib_resource_quota\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"resource_quota_size\": 512000, \"async_server_threads\": 0, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"SYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 16, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" - ], - "boringssl": true, - "ci_platforms": [ - "linux" - ], - "cpu_cost": 1024, - "defaults": "boringssl", - "exclude_configs": [ - "tsan", - "asan" - ], - "excluded_poll_engines": [], - "flaky": false, - "language": "c++", - "name": "json_run_localhost", - "platforms": [ - "linux" - ], - "shortname": "json_run_localhost:cpp_protobuf_sync_unary_qps_unconstrained_secure_500kib_resource_quota", - "timeout_seconds": 360 - }, - { - "args": [ - "--scenarios_json", "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_unary_ping_pong_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 1, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, @@ -39714,31 +39689,6 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_unary_qps_unconstrained_secure_500kib_resource_quota\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"resource_quota_size\": 512000, \"async_server_threads\": 0, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" - ], - "boringssl": true, - "ci_platforms": [ - "linux" - ], - "cpu_cost": "capacity", - "defaults": "boringssl", - "exclude_configs": [ - "tsan", - "asan" - ], - "excluded_poll_engines": [], - "flaky": false, - "language": "c++", - "name": "json_run_localhost", - "platforms": [ - "linux" - ], - "shortname": "json_run_localhost:cpp_protobuf_async_unary_qps_unconstrained_secure_500kib_resource_quota", - "timeout_seconds": 360 - }, - { - "args": [ - "--scenarios_json", "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_ping_pong_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 1, \"client_config\": {\"client_type\": \"SYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, @@ -39789,31 +39739,6 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_qps_unconstrained_secure_500kib_resource_quota\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"resource_quota_size\": 512000, \"async_server_threads\": 0, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"SYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 16, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" - ], - "boringssl": true, - "ci_platforms": [ - "linux" - ], - "cpu_cost": 1024, - "defaults": "boringssl", - "exclude_configs": [ - "tsan", - "asan" - ], - "excluded_poll_engines": [], - "flaky": false, - "language": "c++", - "name": "json_run_localhost", - "platforms": [ - "linux" - ], - "shortname": "json_run_localhost:cpp_protobuf_sync_streaming_qps_unconstrained_secure_500kib_resource_quota", - "timeout_seconds": 360 - }, - { - "args": [ - "--scenarios_json", "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_ping_pong_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 1, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, @@ -39864,31 +39789,6 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_qps_unconstrained_secure_500kib_resource_quota\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"resource_quota_size\": 512000, \"async_server_threads\": 0, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" - ], - "boringssl": true, - "ci_platforms": [ - "linux" - ], - "cpu_cost": "capacity", - "defaults": "boringssl", - "exclude_configs": [ - "tsan", - "asan" - ], - "excluded_poll_engines": [], - "flaky": false, - "language": "c++", - "name": "json_run_localhost", - "platforms": [ - "linux" - ], - "shortname": "json_run_localhost:cpp_protobuf_async_streaming_qps_unconstrained_secure_500kib_resource_quota", - "timeout_seconds": 360 - }, - { - "args": [ - "--scenarios_json", "{\"scenarios\": [{\"name\": \"cpp_generic_async_streaming_ping_pong_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"num_clients\": 1, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, @@ -40068,31 +39968,6 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_unary_qps_unconstrained_insecure_500kib_resource_quota\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"resource_quota_size\": 512000, \"async_server_threads\": 0, \"security_params\": null, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"SYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 16, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" - ], - "boringssl": true, - "ci_platforms": [ - "linux" - ], - "cpu_cost": 1024, - "defaults": "boringssl", - "exclude_configs": [ - "tsan", - "asan" - ], - "excluded_poll_engines": [], - "flaky": false, - "language": "c++", - "name": "json_run_localhost", - "platforms": [ - "linux" - ], - "shortname": "json_run_localhost:cpp_protobuf_sync_unary_qps_unconstrained_insecure_500kib_resource_quota", - "timeout_seconds": 360 - }, - { - "args": [ - "--scenarios_json", "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_unary_ping_pong_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"security_params\": null, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 1, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, @@ -40143,31 +40018,6 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_unary_qps_unconstrained_insecure_500kib_resource_quota\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"resource_quota_size\": 512000, \"async_server_threads\": 0, \"security_params\": null, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" - ], - "boringssl": true, - "ci_platforms": [ - "linux" - ], - "cpu_cost": "capacity", - "defaults": "boringssl", - "exclude_configs": [ - "tsan", - "asan" - ], - "excluded_poll_engines": [], - "flaky": false, - "language": "c++", - "name": "json_run_localhost", - "platforms": [ - "linux" - ], - "shortname": "json_run_localhost:cpp_protobuf_async_unary_qps_unconstrained_insecure_500kib_resource_quota", - "timeout_seconds": 360 - }, - { - "args": [ - "--scenarios_json", "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_ping_pong_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"security_params\": null, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 1, \"client_config\": {\"client_type\": \"SYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, @@ -40218,31 +40068,6 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_qps_unconstrained_insecure_500kib_resource_quota\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"resource_quota_size\": 512000, \"async_server_threads\": 0, \"security_params\": null, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"SYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 16, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" - ], - "boringssl": true, - "ci_platforms": [ - "linux" - ], - "cpu_cost": 1024, - "defaults": "boringssl", - "exclude_configs": [ - "tsan", - "asan" - ], - "excluded_poll_engines": [], - "flaky": false, - "language": "c++", - "name": "json_run_localhost", - "platforms": [ - "linux" - ], - "shortname": "json_run_localhost:cpp_protobuf_sync_streaming_qps_unconstrained_insecure_500kib_resource_quota", - "timeout_seconds": 360 - }, - { - "args": [ - "--scenarios_json", "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_ping_pong_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"security_params\": null, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 1, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, @@ -40293,31 +40118,6 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_qps_unconstrained_insecure_500kib_resource_quota\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"resource_quota_size\": 512000, \"async_server_threads\": 0, \"security_params\": null, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" - ], - "boringssl": true, - "ci_platforms": [ - "linux" - ], - "cpu_cost": "capacity", - "defaults": "boringssl", - "exclude_configs": [ - "tsan", - "asan" - ], - "excluded_poll_engines": [], - "flaky": false, - "language": "c++", - "name": "json_run_localhost", - "platforms": [ - "linux" - ], - "shortname": "json_run_localhost:cpp_protobuf_async_streaming_qps_unconstrained_insecure_500kib_resource_quota", - "timeout_seconds": 360 - }, - { - "args": [ - "--scenarios_json", "{\"scenarios\": [{\"name\": \"cpp_generic_async_streaming_ping_pong_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"num_clients\": 1, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, @@ -40574,42 +40374,6 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_unary_qps_unconstrained_secure_500kib_resource_quota\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"resource_quota_size\": 512000, \"async_server_threads\": 0, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"SYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" - ], - "boringssl": true, - "ci_platforms": [ - "linux" - ], - "cpu_cost": 64, - "defaults": "boringssl", - "exclude_configs": [ - "asan-noleaks", - "asan-trace-cmp", - "basicprof", - "counters", - "dbg", - "gcov", - "helgrind", - "memcheck", - "msan", - "mutrace", - "opt", - "stapprof", - "ubsan" - ], - "excluded_poll_engines": [], - "flaky": false, - "language": "c++", - "name": "json_run_localhost", - "platforms": [ - "linux" - ], - "shortname": "json_run_localhost:cpp_protobuf_sync_unary_qps_unconstrained_secure_500kib_resource_quota_low_thread_count", - "timeout_seconds": 360 - }, - { - "args": [ - "--scenarios_json", "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_unary_ping_pong_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 1, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, @@ -40682,42 +40446,6 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_unary_qps_unconstrained_secure_500kib_resource_quota\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"resource_quota_size\": 512000, \"async_server_threads\": 0, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" - ], - "boringssl": true, - "ci_platforms": [ - "linux" - ], - "cpu_cost": "capacity", - "defaults": "boringssl", - "exclude_configs": [ - "asan-noleaks", - "asan-trace-cmp", - "basicprof", - "counters", - "dbg", - "gcov", - "helgrind", - "memcheck", - "msan", - "mutrace", - "opt", - "stapprof", - "ubsan" - ], - "excluded_poll_engines": [], - "flaky": false, - "language": "c++", - "name": "json_run_localhost", - "platforms": [ - "linux" - ], - "shortname": "json_run_localhost:cpp_protobuf_async_unary_qps_unconstrained_secure_500kib_resource_quota_low_thread_count", - "timeout_seconds": 360 - }, - { - "args": [ - "--scenarios_json", "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_ping_pong_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 1, \"client_config\": {\"client_type\": \"SYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, @@ -40790,42 +40518,6 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_qps_unconstrained_secure_500kib_resource_quota\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"resource_quota_size\": 512000, \"async_server_threads\": 0, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"SYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" - ], - "boringssl": true, - "ci_platforms": [ - "linux" - ], - "cpu_cost": 64, - "defaults": "boringssl", - "exclude_configs": [ - "asan-noleaks", - "asan-trace-cmp", - "basicprof", - "counters", - "dbg", - "gcov", - "helgrind", - "memcheck", - "msan", - "mutrace", - "opt", - "stapprof", - "ubsan" - ], - "excluded_poll_engines": [], - "flaky": false, - "language": "c++", - "name": "json_run_localhost", - "platforms": [ - "linux" - ], - "shortname": "json_run_localhost:cpp_protobuf_sync_streaming_qps_unconstrained_secure_500kib_resource_quota_low_thread_count", - "timeout_seconds": 360 - }, - { - "args": [ - "--scenarios_json", "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_ping_pong_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 1, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, @@ -40898,42 +40590,6 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_qps_unconstrained_secure_500kib_resource_quota\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"resource_quota_size\": 512000, \"async_server_threads\": 0, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" - ], - "boringssl": true, - "ci_platforms": [ - "linux" - ], - "cpu_cost": "capacity", - "defaults": "boringssl", - "exclude_configs": [ - "asan-noleaks", - "asan-trace-cmp", - "basicprof", - "counters", - "dbg", - "gcov", - "helgrind", - "memcheck", - "msan", - "mutrace", - "opt", - "stapprof", - "ubsan" - ], - "excluded_poll_engines": [], - "flaky": false, - "language": "c++", - "name": "json_run_localhost", - "platforms": [ - "linux" - ], - "shortname": "json_run_localhost:cpp_protobuf_async_streaming_qps_unconstrained_secure_500kib_resource_quota_low_thread_count", - "timeout_seconds": 360 - }, - { - "args": [ - "--scenarios_json", "{\"scenarios\": [{\"name\": \"cpp_generic_async_streaming_ping_pong_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"num_clients\": 1, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, @@ -41190,42 +40846,6 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_unary_qps_unconstrained_insecure_500kib_resource_quota\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"resource_quota_size\": 512000, \"async_server_threads\": 0, \"security_params\": null, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"SYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" - ], - "boringssl": true, - "ci_platforms": [ - "linux" - ], - "cpu_cost": 64, - "defaults": "boringssl", - "exclude_configs": [ - "asan-noleaks", - "asan-trace-cmp", - "basicprof", - "counters", - "dbg", - "gcov", - "helgrind", - "memcheck", - "msan", - "mutrace", - "opt", - "stapprof", - "ubsan" - ], - "excluded_poll_engines": [], - "flaky": false, - "language": "c++", - "name": "json_run_localhost", - "platforms": [ - "linux" - ], - "shortname": "json_run_localhost:cpp_protobuf_sync_unary_qps_unconstrained_insecure_500kib_resource_quota_low_thread_count", - "timeout_seconds": 360 - }, - { - "args": [ - "--scenarios_json", "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_unary_ping_pong_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"security_params\": null, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 1, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, @@ -41298,42 +40918,6 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_unary_qps_unconstrained_insecure_500kib_resource_quota\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"resource_quota_size\": 512000, \"async_server_threads\": 0, \"security_params\": null, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" - ], - "boringssl": true, - "ci_platforms": [ - "linux" - ], - "cpu_cost": "capacity", - "defaults": "boringssl", - "exclude_configs": [ - "asan-noleaks", - "asan-trace-cmp", - "basicprof", - "counters", - "dbg", - "gcov", - "helgrind", - "memcheck", - "msan", - "mutrace", - "opt", - "stapprof", - "ubsan" - ], - "excluded_poll_engines": [], - "flaky": false, - "language": "c++", - "name": "json_run_localhost", - "platforms": [ - "linux" - ], - "shortname": "json_run_localhost:cpp_protobuf_async_unary_qps_unconstrained_insecure_500kib_resource_quota_low_thread_count", - "timeout_seconds": 360 - }, - { - "args": [ - "--scenarios_json", "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_ping_pong_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"security_params\": null, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 1, \"client_config\": {\"client_type\": \"SYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, @@ -41406,42 +40990,6 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_qps_unconstrained_insecure_500kib_resource_quota\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"resource_quota_size\": 512000, \"async_server_threads\": 0, \"security_params\": null, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"SYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" - ], - "boringssl": true, - "ci_platforms": [ - "linux" - ], - "cpu_cost": 64, - "defaults": "boringssl", - "exclude_configs": [ - "asan-noleaks", - "asan-trace-cmp", - "basicprof", - "counters", - "dbg", - "gcov", - "helgrind", - "memcheck", - "msan", - "mutrace", - "opt", - "stapprof", - "ubsan" - ], - "excluded_poll_engines": [], - "flaky": false, - "language": "c++", - "name": "json_run_localhost", - "platforms": [ - "linux" - ], - "shortname": "json_run_localhost:cpp_protobuf_sync_streaming_qps_unconstrained_insecure_500kib_resource_quota_low_thread_count", - "timeout_seconds": 360 - }, - { - "args": [ - "--scenarios_json", "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_ping_pong_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"security_params\": null, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 1, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, @@ -41513,42 +41061,6 @@ }, { "args": [ - "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_qps_unconstrained_insecure_500kib_resource_quota\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"resource_quota_size\": 512000, \"async_server_threads\": 0, \"security_params\": null, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" - ], - "boringssl": true, - "ci_platforms": [ - "linux" - ], - "cpu_cost": "capacity", - "defaults": "boringssl", - "exclude_configs": [ - "asan-noleaks", - "asan-trace-cmp", - "basicprof", - "counters", - "dbg", - "gcov", - "helgrind", - "memcheck", - "msan", - "mutrace", - "opt", - "stapprof", - "ubsan" - ], - "excluded_poll_engines": [], - "flaky": false, - "language": "c++", - "name": "json_run_localhost", - "platforms": [ - "linux" - ], - "shortname": "json_run_localhost:cpp_protobuf_async_streaming_qps_unconstrained_insecure_500kib_resource_quota_low_thread_count", - "timeout_seconds": 360 - }, - { - "args": [ "test/core/end2end/fuzzers/api_fuzzer_corpus/00.bin" ], "ci_platforms": [ diff --git a/tools/run_tests/performance/scenario_config.py b/tools/run_tests/performance/scenario_config.py index 830b8c0849..1d91b61ba4 100644 --- a/tools/run_tests/performance/scenario_config.py +++ b/tools/run_tests/performance/scenario_config.py @@ -275,15 +275,18 @@ class CXXLanguage: secure=secure, categories=smoketest_categories+[SCALABLE]) - yield _ping_pong_scenario( - 'cpp_protobuf_%s_%s_qps_unconstrained_%s_500kib_resource_quota' % (synchronicity, rpc_type, secstr), - rpc_type=rpc_type.upper(), - client_type='%s_CLIENT' % synchronicity.upper(), - server_type='%s_SERVER' % synchronicity.upper(), - unconstrained_client=synchronicity, - secure=secure, - categories=smoketest_categories+[SCALABLE], - resource_quota_size=500*1024) + # TODO(vjpai): Re-enable this test. It has a lot of timeouts + # and hasn't yet been conclusively identified as a test failure + # or race in the library + # yield _ping_pong_scenario( + # 'cpp_protobuf_%s_%s_qps_unconstrained_%s_500kib_resource_quota' % (synchronicity, rpc_type, secstr), + # rpc_type=rpc_type.upper(), + # client_type='%s_CLIENT' % synchronicity.upper(), + # server_type='%s_SERVER' % synchronicity.upper(), + # unconstrained_client=synchronicity, + # secure=secure, + # categories=smoketest_categories+[SCALABLE], + # resource_quota_size=500*1024) for channels in geometric_progression(1, 20000, math.sqrt(10)): for outstanding in geometric_progression(1, 200000, math.sqrt(10)): |