1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
|
#!/usr/bin/env python2.7
# Copyright 2015 gRPC authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import print_function
import json
import pipes
import shutil
import sys
import os
import yaml
run_tests_root = os.path.abspath(os.path.join(
os.path.dirname(sys.argv[0]),
'../../../tools/run_tests'))
sys.path.append(run_tests_root)
import performance.scenario_config as scenario_config
configs_from_yaml = yaml.load(open(os.path.join(os.path.dirname(sys.argv[0]), '../../../build.yaml')))['configs'].keys()
def mutate_scenario(scenario_json, is_tsan):
# tweak parameters to get fast test times
scenario_json = dict(scenario_json)
scenario_json['warmup_seconds'] = 0
scenario_json['benchmark_seconds'] = 1
outstanding_rpcs_divisor = 1
if is_tsan and (
scenario_json['client_config']['client_type'] == 'SYNC_CLIENT' or
scenario_json['server_config']['server_type'] == 'SYNC_SERVER'):
outstanding_rpcs_divisor = 10
scenario_json['client_config']['outstanding_rpcs_per_channel'] = max(1,
int(scenario_json['client_config']['outstanding_rpcs_per_channel'] / outstanding_rpcs_divisor))
return scenario_json
def _scenario_json_string(scenario_json, is_tsan):
scenarios_json = {'scenarios': [scenario_config.remove_nonproto_fields(mutate_scenario(scenario_json, is_tsan))]}
return json.dumps(scenarios_json)
def threads_required(scenario_json, where, is_tsan):
scenario_json = mutate_scenario(scenario_json, is_tsan)
if scenario_json['%s_config' % where]['%s_type' % where] == 'ASYNC_%s' % where.upper():
return scenario_json['%s_config' % where].get('async_%s_threads' % where, 0)
return scenario_json['client_config']['outstanding_rpcs_per_channel'] * scenario_json['client_config']['client_channels']
def guess_cpu(scenario_json, is_tsan):
client = threads_required(scenario_json, 'client', is_tsan)
server = threads_required(scenario_json, 'server', is_tsan)
# make an arbitrary guess if set to auto-detect
# about the size of the jenkins instances we have for unit tests
if client == 0 or server == 0: return 'capacity'
return (scenario_json['num_clients'] * client +
scenario_json['num_servers'] * server)
def maybe_exclude_gcov(scenario_json):
if scenario_json['client_config']['client_channels'] > 100:
return ['gcov']
return []
def generate_yaml():
return {
'tests': [
{
'name': 'json_run_localhost',
'shortname': 'json_run_localhost:%s' % scenario_json['name'],
'args': ['--scenarios_json', _scenario_json_string(scenario_json, False)],
'ci_platforms': ['linux'],
'platforms': ['linux'],
'flaky': False,
'language': 'c++',
'boringssl': True,
'defaults': 'boringssl',
'cpu_cost': guess_cpu(scenario_json, False),
'exclude_configs': ['tsan', 'asan'] + maybe_exclude_gcov(scenario_json),
'timeout_seconds': 2*60,
'excluded_poll_engines': scenario_json.get('EXCLUDED_POLL_ENGINES', []),
'auto_timeout_scaling': False
}
for scenario_json in scenario_config.CXXLanguage().scenarios()
if 'scalable' in scenario_json.get('CATEGORIES', [])
] + [
{
'name': 'qps_json_driver',
'shortname': 'qps_json_driver:inproc_%s' % scenario_json['name'],
'args': ['--run_inproc', '--scenarios_json', _scenario_json_string(scenario_json, False)],
'ci_platforms': ['linux'],
'platforms': ['linux'],
'flaky': False,
'language': 'c++',
'boringssl': True,
'defaults': 'boringssl',
'cpu_cost': guess_cpu(scenario_json, False),
'exclude_configs': ['tsan', 'asan'],
'timeout_seconds': 6*60,
'excluded_poll_engines': scenario_json.get('EXCLUDED_POLL_ENGINES', [])
}
for scenario_json in scenario_config.CXXLanguage().scenarios()
if 'inproc' in scenario_json.get('CATEGORIES', [])
] + [
{
'name': 'json_run_localhost',
'shortname': 'json_run_localhost:%s_low_thread_count' % scenario_json['name'],
'args': ['--scenarios_json', _scenario_json_string(scenario_json, True)],
'ci_platforms': ['linux'],
'platforms': ['linux'],
'flaky': False,
'language': 'c++',
'boringssl': True,
'defaults': 'boringssl',
'cpu_cost': guess_cpu(scenario_json, True),
'exclude_configs': sorted(c for c in configs_from_yaml if c not in ('tsan', 'asan')),
'timeout_seconds': 10*60,
'excluded_poll_engines': scenario_json.get('EXCLUDED_POLL_ENGINES', []),
'auto_timeout_scaling': False
}
for scenario_json in scenario_config.CXXLanguage().scenarios()
if 'scalable' in scenario_json.get('CATEGORIES', [])
]
}
print(yaml.dump(generate_yaml()))
|
