diff options
| -rwxr-xr-x | test/cpp/qps/gen_build_yaml.py | 20 | ||||
| -rw-r--r-- | tools/run_tests/performance/scenario_config.py | 33 | ||||
| -rw-r--r-- | tools/run_tests/tests.json | 136 |
3 files changed, 37 insertions, 152 deletions
diff --git a/test/cpp/qps/gen_build_yaml.py b/test/cpp/qps/gen_build_yaml.py index 6b3329b046..e8ad74f0f6 100755 --- a/test/cpp/qps/gen_build_yaml.py +++ b/test/cpp/qps/gen_build_yaml.py @@ -46,6 +46,24 @@ import performance.scenario_config as scenario_config def _scenario_json_string(scenario_json): return json.dumps(scenario_config.remove_nonproto_fields(scenario_json)) +def threads_of_type(scenario_json, path): + d = scenario_json + for el in path.split('/'): + if el not in d: + return 0 + d = d[el] + return d + +def guess_cpu(scenario_json): + client = threads_of_type(scenario_json, 'client_config/async_client_threads') + server = threads_of_type(scenario_json, 'server_config/async_server_threads') + # make an arbitrary guess if set to auto-detect + # about the size of the jenkins instances we have for unit tests + if client == 0: client = 8 + if server == 0: server = 8 + return (scenario_json['num_clients'] * client + + scenario_json['num_servers'] * server) + print yaml.dump({ 'tests': [ { @@ -59,7 +77,7 @@ print yaml.dump({ 'language': 'c++', 'boringssl': True, 'defaults': 'boringssl', - 'cpu_cost': 1000.0, + 'cpu_cost': guess_cpu(scenario_json), 'exclude_configs': [] } for scenario_json in scenario_config.CXXLanguage().scenarios() diff --git a/tools/run_tests/performance/scenario_config.py b/tools/run_tests/performance/scenario_config.py index d50764f4d9..b55d728d84 100644 --- a/tools/run_tests/performance/scenario_config.py +++ b/tools/run_tests/performance/scenario_config.py @@ -94,7 +94,6 @@ def remove_nonproto_fields(scenario): def _ping_pong_scenario(name, rpc_type, client_type, server_type, secure=True, - use_big_generic_payload=False, use_generic_payload=False, unconstrained_client=None, client_language=None, @@ -129,12 +128,7 @@ def _ping_pong_scenario(name, rpc_type, 'warmup_seconds': warmup_seconds, 'benchmark_seconds': BENCHMARK_SECONDS } - if use_big_generic_payload: - if server_type != 'ASYNC_GENERIC_SERVER': - raise Exception('Use ASYNC_GENERIC_SERVER for big generic payload.') - scenario['client_config']['payload_config'] = BIG_GENERIC_PAYLOAD - scenario['server_config']['payload_config'] = BIG_GENERIC_PAYLOAD - elif use_generic_payload: + if use_generic_payload: if server_type != 'ASYNC_GENERIC_SERVER': raise Exception('Use ASYNC_GENERIC_SERVER for generic payload.') scenario['client_config']['payload_config'] = EMPTY_GENERIC_PAYLOAD @@ -147,23 +141,13 @@ def _ping_pong_scenario(name, rpc_type, if unconstrained_client == 'async': deep = DEEP wide = WIDE - num_clients = 0 # use as many clients as available elif unconstrained_client == 'sync': deep = SYNC_DEEP wide = SYNC_WIDE - num_clients = 0 # use as many clients as available - elif unconstrained_client == '1chan_bw': - deep = DEEP - wide = 1 - num_clients = 1 # limit to 1 for a single channel - elif unconstrained_client == 'Nchan_bw': - deep = DEEP - wide = WIDE - num_clients = 0 # use as many clients as available else: raise Exception('Illegal value of unconstrained_client option.') - scenario['num_clients'] = num_clients + scenario['num_clients'] = 0 # use as many client as available. scenario['client_config']['outstanding_rpcs_per_channel'] = deep scenario['client_config']['client_channels'] = wide scenario['client_config']['async_client_threads'] = 0 @@ -254,19 +238,6 @@ class CXXLanguage: server_core_limit=1, async_server_threads=1, secure=secure) - yield _ping_pong_scenario( - 'cpp_generic_async_streaming_single_channel_throughput_%s' % secstr, rpc_type='STREAMING', - client_type='ASYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER', - unconstrained_client='1chan_bw', use_big_generic_payload=True, - secure=secure, - categories=smoketest_categories) - - yield _ping_pong_scenario( - 'cpp_generic_async_streaming_multi_channel_throughput_%s' % secstr, rpc_type='STREAMING', - client_type='ASYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER', - unconstrained_client='Nchan_bw', use_big_generic_payload=True, - secure=secure) - def __str__(self): return 'c++' diff --git a/tools/run_tests/tests.json b/tools/run_tests/tests.json index a9fffd33b4..da597effa7 100644 --- a/tools/run_tests/tests.json +++ b/tools/run_tests/tests.json @@ -23031,7 +23031,7 @@ "posix", "windows" ], - "cpu_cost": 1000.0, + "cpu_cost": 2, "defaults": "boringssl", "exclude_configs": [], "flaky": false, @@ -23057,7 +23057,7 @@ "posix", "windows" ], - "cpu_cost": 1000.0, + "cpu_cost": 2, "defaults": "boringssl", "exclude_configs": [], "flaky": false, @@ -23083,7 +23083,7 @@ "posix", "windows" ], - "cpu_cost": 1000.0, + "cpu_cost": 2, "defaults": "boringssl", "exclude_configs": [], "flaky": false, @@ -23109,7 +23109,7 @@ "posix", "windows" ], - "cpu_cost": 1000.0, + "cpu_cost": 2, "defaults": "boringssl", "exclude_configs": [], "flaky": false, @@ -23135,7 +23135,7 @@ "posix", "windows" ], - "cpu_cost": 1000.0, + "cpu_cost": 8, "defaults": "boringssl", "exclude_configs": [], "flaky": false, @@ -23161,7 +23161,7 @@ "posix", "windows" ], - "cpu_cost": 1000.0, + "cpu_cost": 8, "defaults": "boringssl", "exclude_configs": [], "flaky": false, @@ -23187,7 +23187,7 @@ "posix", "windows" ], - "cpu_cost": 1000.0, + "cpu_cost": 8, "defaults": "boringssl", "exclude_configs": [], "flaky": false, @@ -23213,7 +23213,7 @@ "posix", "windows" ], - "cpu_cost": 1000.0, + "cpu_cost": 1, "defaults": "boringssl", "exclude_configs": [], "flaky": false, @@ -23230,58 +23230,6 @@ { "args": [ "--scenario_json", - "'{\"name\": \"cpp_generic_async_streaming_single_channel_throughput_secure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"core_limit\": 0, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 65536, \"req_size\": 65536}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 65536, \"req_size\": 65536}}, \"client_channels\": 1, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 1}'" - ], - "boringssl": true, - "ci_platforms": [ - "linux", - "mac", - "posix", - "windows" - ], - "cpu_cost": 1000.0, - "defaults": "boringssl", - "exclude_configs": [], - "flaky": false, - "language": "c++", - "name": "json_run_localhost", - "platforms": [ - "linux", - "mac", - "posix", - "windows" - ], - "shortname": "json_run_localhost:cpp_generic_async_streaming_single_channel_throughput_secure" - }, - { - "args": [ - "--scenario_json", - "'{\"name\": \"cpp_generic_async_streaming_multi_channel_throughput_secure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"core_limit\": 0, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 65536, \"req_size\": 65536}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 65536, \"req_size\": 65536}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}'" - ], - "boringssl": true, - "ci_platforms": [ - "linux", - "mac", - "posix", - "windows" - ], - "cpu_cost": 1000.0, - "defaults": "boringssl", - "exclude_configs": [], - "flaky": false, - "language": "c++", - "name": "json_run_localhost", - "platforms": [ - "linux", - "mac", - "posix", - "windows" - ], - "shortname": "json_run_localhost:cpp_generic_async_streaming_multi_channel_throughput_secure" - }, - { - "args": [ - "--scenario_json", "'{\"name\": \"cpp_generic_async_streaming_ping_pong_insecure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 1, \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 1}'" ], "boringssl": true, @@ -23291,7 +23239,7 @@ "posix", "windows" ], - "cpu_cost": 1000.0, + "cpu_cost": 2, "defaults": "boringssl", "exclude_configs": [], "flaky": false, @@ -23317,7 +23265,7 @@ "posix", "windows" ], - "cpu_cost": 1000.0, + "cpu_cost": 2, "defaults": "boringssl", "exclude_configs": [], "flaky": false, @@ -23343,7 +23291,7 @@ "posix", "windows" ], - "cpu_cost": 1000.0, + "cpu_cost": 2, "defaults": "boringssl", "exclude_configs": [], "flaky": false, @@ -23369,7 +23317,7 @@ "posix", "windows" ], - "cpu_cost": 1000.0, + "cpu_cost": 2, "defaults": "boringssl", "exclude_configs": [], "flaky": false, @@ -23395,7 +23343,7 @@ "posix", "windows" ], - "cpu_cost": 1000.0, + "cpu_cost": 8, "defaults": "boringssl", "exclude_configs": [], "flaky": false, @@ -23421,7 +23369,7 @@ "posix", "windows" ], - "cpu_cost": 1000.0, + "cpu_cost": 8, "defaults": "boringssl", "exclude_configs": [], "flaky": false, @@ -23447,7 +23395,7 @@ "posix", "windows" ], - "cpu_cost": 1000.0, + "cpu_cost": 8, "defaults": "boringssl", "exclude_configs": [], "flaky": false, @@ -23473,7 +23421,7 @@ "posix", "windows" ], - "cpu_cost": 1000.0, + "cpu_cost": 1, "defaults": "boringssl", "exclude_configs": [], "flaky": false, @@ -23489,58 +23437,6 @@ }, { "args": [ - "--scenario_json", - "'{\"name\": \"cpp_generic_async_streaming_single_channel_throughput_insecure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"core_limit\": 0, \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 65536, \"req_size\": 65536}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 65536, \"req_size\": 65536}}, \"client_channels\": 1, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 1}'" - ], - "boringssl": true, - "ci_platforms": [ - "linux", - "mac", - "posix", - "windows" - ], - "cpu_cost": 1000.0, - "defaults": "boringssl", - "exclude_configs": [], - "flaky": false, - "language": "c++", - "name": "json_run_localhost", - "platforms": [ - "linux", - "mac", - "posix", - "windows" - ], - "shortname": "json_run_localhost:cpp_generic_async_streaming_single_channel_throughput_insecure" - }, - { - "args": [ - "--scenario_json", - "'{\"name\": \"cpp_generic_async_streaming_multi_channel_throughput_insecure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"core_limit\": 0, \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 65536, \"req_size\": 65536}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 65536, \"req_size\": 65536}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}'" - ], - "boringssl": true, - "ci_platforms": [ - "linux", - "mac", - "posix", - "windows" - ], - "cpu_cost": 1000.0, - "defaults": "boringssl", - "exclude_configs": [], - "flaky": false, - "language": "c++", - "name": "json_run_localhost", - "platforms": [ - "linux", - "mac", - "posix", - "windows" - ], - "shortname": "json_run_localhost:cpp_generic_async_streaming_multi_channel_throughput_insecure" - }, - { - "args": [ "test/core/end2end/fuzzers/api_fuzzer_corpus/00.bin" ], "ci_platforms": [ |