Take into account all the configs

2 files changed, 479 insertions, 69 deletions

diff --git a/test/cpp/qps/gen_build_yaml.py b/test/cpp/qps/gen_build_yaml.py
index 67e5a29883..4aa58d2737 100755
--- a/test/cpp/qps/gen_build_yaml.py
+++ b/test/cpp/qps/gen_build_yaml.py
@@ -43,6 +43,8 @@ sys.path.append(run_tests_root)
 
 import performance.scenario_config as scenario_config
 
+configs_from_yaml = yaml.load(open(os.path.join(os.path.dirname(sys.argv[0]), '../../../build.yaml')))['configs'].keys()
+
 def mutate_scenario(scenario_json, is_tsan):
   # tweak parameters to get fast test times
   scenario_json = dict(scenario_json)
@@ -106,7 +108,7 @@ print yaml.dump({
       'boringssl': True,
       'defaults': 'boringssl',
       'cpu_cost': guess_cpu(scenario_json, True),
-      'exclude_configs': ['dbg', 'opt', 'asan', 'msan'],
+      'exclude_configs': sorted(c for c in configs_from_yaml if c != 'tsan'),
       'timeout_seconds': 6*60
     }
     for scenario_json in scenario_config.CXXLanguage().scenarios()
diff --git a/tools/run_tests/tests.json b/tools/run_tests/tests.json
index 9931640f01..c4bfd0a9a7 100644
--- a/tools/run_tests/tests.json
+++ b/tools/run_tests/tests.json
@@ -37547,10 +37547,22 @@
     "cpu_cost": 2, 
     "defaults": "boringssl", 
     "exclude_configs": [
+      "asan", 
+      "asan-noleaks", 
+      "asan-trace-cmp", 
+      "basicprof", 
       "dbg", 
+      "easan", 
+      "edbg", 
+      "etsan", 
+      "gcov", 
+      "helgrind", 
+      "memcheck", 
+      "msan", 
+      "mutrace", 
       "opt", 
-      "asan", 
-      "msan"
+      "stapprof", 
+      "ubsan"
     ], 
     "flaky": false, 
     "language": "c++", 
@@ -37573,10 +37585,22 @@
     "cpu_cost": "capacity", 
     "defaults": "boringssl", 
     "exclude_configs": [
+      "asan", 
+      "asan-noleaks", 
+      "asan-trace-cmp", 
+      "basicprof", 
       "dbg", 
+      "easan", 
+      "edbg", 
+      "etsan", 
+      "gcov", 
+      "helgrind", 
+      "memcheck", 
+      "msan", 
+      "mutrace", 
       "opt", 
-      "asan", 
-      "msan"
+      "stapprof", 
+      "ubsan"
     ], 
     "flaky": false, 
     "language": "c++", 
@@ -37599,10 +37623,22 @@
     "cpu_cost": "capacity", 
     "defaults": "boringssl", 
     "exclude_configs": [
+      "asan", 
+      "asan-noleaks", 
+      "asan-trace-cmp", 
+      "basicprof", 
       "dbg", 
+      "easan", 
+      "edbg", 
+      "etsan", 
+      "gcov", 
+      "helgrind", 
+      "memcheck", 
+      "msan", 
+      "mutrace", 
       "opt", 
-      "asan", 
-      "msan"
+      "stapprof", 
+      "ubsan"
     ], 
     "flaky": false, 
     "language": "c++", 
@@ -37625,10 +37661,22 @@
     "cpu_cost": "capacity", 
     "defaults": "boringssl", 
     "exclude_configs": [
+      "asan", 
+      "asan-noleaks", 
+      "asan-trace-cmp", 
+      "basicprof", 
       "dbg", 
+      "easan", 
+      "edbg", 
+      "etsan", 
+      "gcov", 
+      "helgrind", 
+      "memcheck", 
+      "msan", 
+      "mutrace", 
       "opt", 
-      "asan", 
-      "msan"
+      "stapprof", 
+      "ubsan"
     ], 
     "flaky": false, 
     "language": "c++", 
@@ -37651,10 +37699,22 @@
     "cpu_cost": "capacity", 
     "defaults": "boringssl", 
     "exclude_configs": [
+      "asan", 
+      "asan-noleaks", 
+      "asan-trace-cmp", 
+      "basicprof", 
       "dbg", 
+      "easan", 
+      "edbg", 
+      "etsan", 
+      "gcov", 
+      "helgrind", 
+      "memcheck", 
+      "msan", 
+      "mutrace", 
       "opt", 
-      "asan", 
-      "msan"
+      "stapprof", 
+      "ubsan"
     ], 
     "flaky": false, 
     "language": "c++", 
@@ -37677,10 +37737,22 @@
     "cpu_cost": 2, 
     "defaults": "boringssl", 
     "exclude_configs": [
+      "asan", 
+      "asan-noleaks", 
+      "asan-trace-cmp", 
+      "basicprof", 
       "dbg", 
+      "easan", 
+      "edbg", 
+      "etsan", 
+      "gcov", 
+      "helgrind", 
+      "memcheck", 
+      "msan", 
+      "mutrace", 
       "opt", 
-      "asan", 
-      "msan"
+      "stapprof", 
+      "ubsan"
     ], 
     "flaky": false, 
     "language": "c++", 
@@ -37703,10 +37775,22 @@
     "cpu_cost": 64, 
     "defaults": "boringssl", 
     "exclude_configs": [
+      "asan", 
+      "asan-noleaks", 
+      "asan-trace-cmp", 
+      "basicprof", 
       "dbg", 
+      "easan", 
+      "edbg", 
+      "etsan", 
+      "gcov", 
+      "helgrind", 
+      "memcheck", 
+      "msan", 
+      "mutrace", 
       "opt", 
-      "asan", 
-      "msan"
+      "stapprof", 
+      "ubsan"
     ], 
     "flaky": false, 
     "language": "c++", 
@@ -37729,10 +37813,22 @@
     "cpu_cost": 64, 
     "defaults": "boringssl", 
     "exclude_configs": [
+      "asan", 
+      "asan-noleaks", 
+      "asan-trace-cmp", 
+      "basicprof", 
       "dbg", 
+      "easan", 
+      "edbg", 
+      "etsan", 
+      "gcov", 
+      "helgrind", 
+      "memcheck", 
+      "msan", 
+      "mutrace", 
       "opt", 
-      "asan", 
-      "msan"
+      "stapprof", 
+      "ubsan"
     ], 
     "flaky": false, 
     "language": "c++", 
@@ -37755,10 +37851,22 @@
     "cpu_cost": 2, 
     "defaults": "boringssl", 
     "exclude_configs": [
+      "asan", 
+      "asan-noleaks", 
+      "asan-trace-cmp", 
+      "basicprof", 
       "dbg", 
+      "easan", 
+      "edbg", 
+      "etsan", 
+      "gcov", 
+      "helgrind", 
+      "memcheck", 
+      "msan", 
+      "mutrace", 
       "opt", 
-      "asan", 
-      "msan"
+      "stapprof", 
+      "ubsan"
     ], 
     "flaky": false, 
     "language": "c++", 
@@ -37781,10 +37889,22 @@
     "cpu_cost": "capacity", 
     "defaults": "boringssl", 
     "exclude_configs": [
+      "asan", 
+      "asan-noleaks", 
+      "asan-trace-cmp", 
+      "basicprof", 
       "dbg", 
+      "easan", 
+      "edbg", 
+      "etsan", 
+      "gcov", 
+      "helgrind", 
+      "memcheck", 
+      "msan", 
+      "mutrace", 
       "opt", 
-      "asan", 
-      "msan"
+      "stapprof", 
+      "ubsan"
     ], 
     "flaky": false, 
     "language": "c++", 
@@ -37807,10 +37927,22 @@
     "cpu_cost": "capacity", 
     "defaults": "boringssl", 
     "exclude_configs": [
+      "asan", 
+      "asan-noleaks", 
+      "asan-trace-cmp", 
+      "basicprof", 
       "dbg", 
+      "easan", 
+      "edbg", 
+      "etsan", 
+      "gcov", 
+      "helgrind", 
+      "memcheck", 
+      "msan", 
+      "mutrace", 
       "opt", 
-      "asan", 
-      "msan"
+      "stapprof", 
+      "ubsan"
     ], 
     "flaky": false, 
     "language": "c++", 
@@ -37833,10 +37965,22 @@
     "cpu_cost": 2, 
     "defaults": "boringssl", 
     "exclude_configs": [
+      "asan", 
+      "asan-noleaks", 
+      "asan-trace-cmp", 
+      "basicprof", 
       "dbg", 
+      "easan", 
+      "edbg", 
+      "etsan", 
+      "gcov", 
+      "helgrind", 
+      "memcheck", 
+      "msan", 
+      "mutrace", 
       "opt", 
-      "asan", 
-      "msan"
+      "stapprof", 
+      "ubsan"
     ], 
     "flaky": false, 
     "language": "c++", 
@@ -37859,10 +38003,22 @@
     "cpu_cost": 64, 
     "defaults": "boringssl", 
     "exclude_configs": [
+      "asan", 
+      "asan-noleaks", 
+      "asan-trace-cmp", 
+      "basicprof", 
       "dbg", 
+      "easan", 
+      "edbg", 
+      "etsan", 
+      "gcov", 
+      "helgrind", 
+      "memcheck", 
+      "msan", 
+      "mutrace", 
       "opt", 
-      "asan", 
-      "msan"
+      "stapprof", 
+      "ubsan"
     ], 
     "flaky": false, 
     "language": "c++", 
@@ -37885,10 +38041,22 @@
     "cpu_cost": 64, 
     "defaults": "boringssl", 
     "exclude_configs": [
+      "asan", 
+      "asan-noleaks", 
+      "asan-trace-cmp", 
+      "basicprof", 
       "dbg", 
+      "easan", 
+      "edbg", 
+      "etsan", 
+      "gcov", 
+      "helgrind", 
+      "memcheck", 
+      "msan", 
+      "mutrace", 
       "opt", 
-      "asan", 
-      "msan"
+      "stapprof", 
+      "ubsan"
     ], 
     "flaky": false, 
     "language": "c++", 
@@ -37911,10 +38079,22 @@
     "cpu_cost": 2, 
     "defaults": "boringssl", 
     "exclude_configs": [
+      "asan", 
+      "asan-noleaks", 
+      "asan-trace-cmp", 
+      "basicprof", 
       "dbg", 
+      "easan", 
+      "edbg", 
+      "etsan", 
+      "gcov", 
+      "helgrind", 
+      "memcheck", 
+      "msan", 
+      "mutrace", 
       "opt", 
-      "asan", 
-      "msan"
+      "stapprof", 
+      "ubsan"
     ], 
     "flaky": false, 
     "language": "c++", 
@@ -37937,10 +38117,22 @@
     "cpu_cost": "capacity", 
     "defaults": "boringssl", 
     "exclude_configs": [
+      "asan", 
+      "asan-noleaks", 
+      "asan-trace-cmp", 
+      "basicprof", 
       "dbg", 
+      "easan", 
+      "edbg", 
+      "etsan", 
+      "gcov", 
+      "helgrind", 
+      "memcheck", 
+      "msan", 
+      "mutrace", 
       "opt", 
-      "asan", 
-      "msan"
+      "stapprof", 
+      "ubsan"
     ], 
     "flaky": false, 
     "language": "c++", 
@@ -37963,10 +38155,22 @@
     "cpu_cost": "capacity", 
     "defaults": "boringssl", 
     "exclude_configs": [
+      "asan", 
+      "asan-noleaks", 
+      "asan-trace-cmp", 
+      "basicprof", 
       "dbg", 
+      "easan", 
+      "edbg", 
+      "etsan", 
+      "gcov", 
+      "helgrind", 
+      "memcheck", 
+      "msan", 
+      "mutrace", 
       "opt", 
-      "asan", 
-      "msan"
+      "stapprof", 
+      "ubsan"
     ], 
     "flaky": false, 
     "language": "c++", 
@@ -37989,10 +38193,22 @@
     "cpu_cost": 2, 
     "defaults": "boringssl", 
     "exclude_configs": [
+      "asan", 
+      "asan-noleaks", 
+      "asan-trace-cmp", 
+      "basicprof", 
       "dbg", 
+      "easan", 
+      "edbg", 
+      "etsan", 
+      "gcov", 
+      "helgrind", 
+      "memcheck", 
+      "msan", 
+      "mutrace", 
       "opt", 
-      "asan", 
-      "msan"
+      "stapprof", 
+      "ubsan"
     ], 
     "flaky": false, 
     "language": "c++", 
@@ -38015,10 +38231,22 @@
     "cpu_cost": "capacity", 
     "defaults": "boringssl", 
     "exclude_configs": [
+      "asan", 
+      "asan-noleaks", 
+      "asan-trace-cmp", 
+      "basicprof", 
       "dbg", 
+      "easan", 
+      "edbg", 
+      "etsan", 
+      "gcov", 
+      "helgrind", 
+      "memcheck", 
+      "msan", 
+      "mutrace", 
       "opt", 
-      "asan", 
-      "msan"
+      "stapprof", 
+      "ubsan"
     ], 
     "flaky": false, 
     "language": "c++", 
@@ -38041,10 +38269,22 @@
     "cpu_cost": "capacity", 
     "defaults": "boringssl", 
     "exclude_configs": [
+      "asan", 
+      "asan-noleaks", 
+      "asan-trace-cmp", 
+      "basicprof", 
       "dbg", 
+      "easan", 
+      "edbg", 
+      "etsan", 
+      "gcov", 
+      "helgrind", 
+      "memcheck", 
+      "msan", 
+      "mutrace", 
       "opt", 
-      "asan", 
-      "msan"
+      "stapprof", 
+      "ubsan"
     ], 
     "flaky": false, 
     "language": "c++", 
@@ -38067,10 +38307,22 @@
     "cpu_cost": "capacity", 
     "defaults": "boringssl", 
     "exclude_configs": [
+      "asan", 
+      "asan-noleaks", 
+      "asan-trace-cmp", 
+      "basicprof", 
       "dbg", 
+      "easan", 
+      "edbg", 
+      "etsan", 
+      "gcov", 
+      "helgrind", 
+      "memcheck", 
+      "msan", 
+      "mutrace", 
       "opt", 
-      "asan", 
-      "msan"
+      "stapprof", 
+      "ubsan"
     ], 
     "flaky": false, 
     "language": "c++", 
@@ -38093,10 +38345,22 @@
     "cpu_cost": "capacity", 
     "defaults": "boringssl", 
     "exclude_configs": [
+      "asan", 
+      "asan-noleaks", 
+      "asan-trace-cmp", 
+      "basicprof", 
       "dbg", 
+      "easan", 
+      "edbg", 
+      "etsan", 
+      "gcov", 
+      "helgrind", 
+      "memcheck", 
+      "msan", 
+      "mutrace", 
       "opt", 
-      "asan", 
-      "msan"
+      "stapprof", 
+      "ubsan"
     ], 
     "flaky": false, 
     "language": "c++", 
@@ -38119,10 +38383,22 @@
     "cpu_cost": 2, 
     "defaults": "boringssl", 
     "exclude_configs": [
+      "asan", 
+      "asan-noleaks", 
+      "asan-trace-cmp", 
+      "basicprof", 
       "dbg", 
+      "easan", 
+      "edbg", 
+      "etsan", 
+      "gcov", 
+      "helgrind", 
+      "memcheck", 
+      "msan", 
+      "mutrace", 
       "opt", 
-      "asan", 
-      "msan"
+      "stapprof", 
+      "ubsan"
     ], 
     "flaky": false, 
     "language": "c++", 
@@ -38145,10 +38421,22 @@
     "cpu_cost": 64, 
     "defaults": "boringssl", 
     "exclude_configs": [
+      "asan", 
+      "asan-noleaks", 
+      "asan-trace-cmp", 
+      "basicprof", 
       "dbg", 
+      "easan", 
+      "edbg", 
+      "etsan", 
+      "gcov", 
+      "helgrind", 
+      "memcheck", 
+      "msan", 
+      "mutrace", 
       "opt", 
-      "asan", 
-      "msan"
+      "stapprof", 
+      "ubsan"
     ], 
     "flaky": false, 
     "language": "c++", 
@@ -38171,10 +38459,22 @@
     "cpu_cost": 64, 
     "defaults": "boringssl", 
     "exclude_configs": [
+      "asan", 
+      "asan-noleaks", 
+      "asan-trace-cmp", 
+      "basicprof", 
       "dbg", 
+      "easan", 
+      "edbg", 
+      "etsan", 
+      "gcov", 
+      "helgrind", 
+      "memcheck", 
+      "msan", 
+      "mutrace", 
       "opt", 
-      "asan", 
-      "msan"
+      "stapprof", 
+      "ubsan"
     ], 
     "flaky": false, 
     "language": "c++", 
@@ -38197,10 +38497,22 @@
     "cpu_cost": 2, 
     "defaults": "boringssl", 
     "exclude_configs": [
+      "asan", 
+      "asan-noleaks", 
+      "asan-trace-cmp", 
+      "basicprof", 
       "dbg", 
+      "easan", 
+      "edbg", 
+      "etsan", 
+      "gcov", 
+      "helgrind", 
+      "memcheck", 
+      "msan", 
+      "mutrace", 
       "opt", 
-      "asan", 
-      "msan"
+      "stapprof", 
+      "ubsan"
     ], 
     "flaky": false, 
     "language": "c++", 
@@ -38223,10 +38535,22 @@
     "cpu_cost": "capacity", 
     "defaults": "boringssl", 
     "exclude_configs": [
+      "asan", 
+      "asan-noleaks", 
+      "asan-trace-cmp", 
+      "basicprof", 
       "dbg", 
+      "easan", 
+      "edbg", 
+      "etsan", 
+      "gcov", 
+      "helgrind", 
+      "memcheck", 
+      "msan", 
+      "mutrace", 
       "opt", 
-      "asan", 
-      "msan"
+      "stapprof", 
+      "ubsan"
     ], 
     "flaky": false, 
     "language": "c++", 
@@ -38249,10 +38573,22 @@
     "cpu_cost": "capacity", 
     "defaults": "boringssl", 
     "exclude_configs": [
+      "asan", 
+      "asan-noleaks", 
+      "asan-trace-cmp", 
+      "basicprof", 
       "dbg", 
+      "easan", 
+      "edbg", 
+      "etsan", 
+      "gcov", 
+      "helgrind", 
+      "memcheck", 
+      "msan", 
+      "mutrace", 
       "opt", 
-      "asan", 
-      "msan"
+      "stapprof", 
+      "ubsan"
     ], 
     "flaky": false, 
     "language": "c++", 
@@ -38275,10 +38611,22 @@
     "cpu_cost": 2, 
     "defaults": "boringssl", 
     "exclude_configs": [
+      "asan", 
+      "asan-noleaks", 
+      "asan-trace-cmp", 
+      "basicprof", 
       "dbg", 
+      "easan", 
+      "edbg", 
+      "etsan", 
+      "gcov", 
+      "helgrind", 
+      "memcheck", 
+      "msan", 
+      "mutrace", 
       "opt", 
-      "asan", 
-      "msan"
+      "stapprof", 
+      "ubsan"
     ], 
     "flaky": false, 
     "language": "c++", 
@@ -38301,10 +38649,22 @@
     "cpu_cost": 64, 
     "defaults": "boringssl", 
     "exclude_configs": [
+      "asan", 
+      "asan-noleaks", 
+      "asan-trace-cmp", 
+      "basicprof", 
       "dbg", 
+      "easan", 
+      "edbg", 
+      "etsan", 
+      "gcov", 
+      "helgrind", 
+      "memcheck", 
+      "msan", 
+      "mutrace", 
       "opt", 
-      "asan", 
-      "msan"
+      "stapprof", 
+      "ubsan"
     ], 
     "flaky": false, 
     "language": "c++", 
@@ -38327,10 +38687,22 @@
     "cpu_cost": 64, 
     "defaults": "boringssl", 
     "exclude_configs": [
+      "asan", 
+      "asan-noleaks", 
+      "asan-trace-cmp", 
+      "basicprof", 
       "dbg", 
+      "easan", 
+      "edbg", 
+      "etsan", 
+      "gcov", 
+      "helgrind", 
+      "memcheck", 
+      "msan", 
+      "mutrace", 
       "opt", 
-      "asan", 
-      "msan"
+      "stapprof", 
+      "ubsan"
     ], 
     "flaky": false, 
     "language": "c++", 
@@ -38353,10 +38725,22 @@
     "cpu_cost": 2, 
     "defaults": "boringssl", 
     "exclude_configs": [
+      "asan", 
+      "asan-noleaks", 
+      "asan-trace-cmp", 
+      "basicprof", 
       "dbg", 
+      "easan", 
+      "edbg", 
+      "etsan", 
+      "gcov", 
+      "helgrind", 
+      "memcheck", 
+      "msan", 
+      "mutrace", 
       "opt", 
-      "asan", 
-      "msan"
+      "stapprof", 
+      "ubsan"
     ], 
     "flaky": false, 
     "language": "c++", 
@@ -38379,10 +38763,22 @@
     "cpu_cost": "capacity", 
     "defaults": "boringssl", 
     "exclude_configs": [
+      "asan", 
+      "asan-noleaks", 
+      "asan-trace-cmp", 
+      "basicprof", 
       "dbg", 
+      "easan", 
+      "edbg", 
+      "etsan", 
+      "gcov", 
+      "helgrind", 
+      "memcheck", 
+      "msan", 
+      "mutrace", 
       "opt", 
-      "asan", 
-      "msan"
+      "stapprof", 
+      "ubsan"
     ], 
     "flaky": false, 
     "language": "c++", 
@@ -38405,10 +38801,22 @@
     "cpu_cost": "capacity", 
     "defaults": "boringssl", 
     "exclude_configs": [
+      "asan", 
+      "asan-noleaks", 
+      "asan-trace-cmp", 
+      "basicprof", 
       "dbg", 
+      "easan", 
+      "edbg", 
+      "etsan", 
+      "gcov", 
+      "helgrind", 
+      "memcheck", 
+      "msan", 
+      "mutrace", 
       "opt", 
-      "asan", 
-      "msan"
+      "stapprof", 
+      "ubsan"
     ], 
     "flaky": false, 
     "language": "c++",