From a0f85171d1efd1bd0c159c6940f34880c247596e Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 20 Jan 2016 15:56:06 -0800 Subject: Move build configs into build.yaml Importantly: - allows matching timeout values between makefile and run_tests without repeating ourselves - allows borrowing of compiler flags by other build systems There's still a little too much build configuration built into our Makefile, but we can start attacking that over time. --- tools/run_tests/configs.json | 65 ++++++++++++++++++++++++++++++++++++++++++++ tools/run_tests/run_tests.py | 47 +++++--------------------------- 2 files changed, 72 insertions(+), 40 deletions(-) create mode 100644 tools/run_tests/configs.json (limited to 'tools') diff --git a/tools/run_tests/configs.json b/tools/run_tests/configs.json new file mode 100644 index 0000000000..ef2ab61daa --- /dev/null +++ b/tools/run_tests/configs.json @@ -0,0 +1,65 @@ +[ + { + "config": "opt" + }, + { + "config": "basicprof" + }, + { + "config": "helgrind", + "timeout_multiplier": 20, + "tool_prefix": [ + "valgrind", + "--tool=helgrind" + ] + }, + { + "config": "asan-noleaks", + "environ": { + "ASAN_OPTIONS": "detect_leaks=0:color=always" + } + }, + { + "config": "ubsan", + "timeout_multiplier": 1.5 + }, + { + "config": "dbg" + }, + { + "config": "stapprof" + }, + { + "config": "gcov" + }, + { + "config": "memcheck", + "timeout_multiplier": 10, + "tool_prefix": [ + "valgrind", + "--tool=memcheck", + "--leak-check=full" + ] + }, + { + "config": "asan", + "environ": { + "ASAN_OPTIONS": "suppressions=tools/asan_suppressions.txt:detect_leaks=1:color=always", + "LSAN_OPTIONS": "suppressions=tools/asan_suppressions.txt:report_objects=1" + } + }, + { + "config": "tsan", + "environ": { + "TSAN_OPTIONS": "suppressions=tools/tsan_suppressions.txt:halt_on_error=1:second_deadlock_stack=1" + }, + "timeout_multiplier": 2 + }, + { + "config": "msan", + "timeout_multiplier": 1.5 + }, + { + "config": "mutrace" + } +] diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index c1f1576378..e31fc30980 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -66,15 +66,16 @@ def platform_string(): # SimpleConfig: just compile with CONFIG=config, and run the binary to test -class SimpleConfig(object): +class Config(object): - def __init__(self, config, environ=None, timeout_multiplier=1): + def __init__(self, config, environ=None, timeout_multiplier=1, tool_prefix=[]): if environ is None: environ = {} self.build_config = config self.allow_hashing = (config != 'gcov') self.environ = environ self.environ['CONFIG'] = config + self.tool_prefix = tool_prefix self.timeout_multiplier = timeout_multiplier def job_spec(self, cmdline, hash_targets, timeout_seconds=5*60, @@ -93,7 +94,7 @@ class SimpleConfig(object): actual_environ = self.environ.copy() for k, v in environ.iteritems(): actual_environ[k] = v - return jobset.JobSpec(cmdline=cmdline, + return jobset.JobSpec(cmdline=self.tool_prefix + cmdline, shortname=shortname, environ=actual_environ, cpu_cost=cpu_cost, @@ -104,27 +105,6 @@ class SimpleConfig(object): timeout_retries=3 if args.allow_flakes else 0) -# ValgrindConfig: compile with some CONFIG=config, but use valgrind to run -class ValgrindConfig(object): - - def __init__(self, config, tool, args=None): - if args is None: - args = [] - self.build_config = config - self.tool = tool - self.args = args - self.allow_hashing = False - - def job_spec(self, cmdline, hash_targets, cpu_cost=1.0): - return jobset.JobSpec(cmdline=['valgrind', '--tool=%s' % self.tool] + - self.args + cmdline, - shortname='valgrind %s' % cmdline[0], - hash_targets=None, - cpu_cost=cpu_cost, - flake_retries=5 if args.allow_flakes else 0, - timeout_retries=3 if args.allow_flakes else 0) - - def get_c_tests(travis, test_lang) : out = [] platforms_str = 'ci_platforms' if travis else 'platforms' @@ -499,22 +479,8 @@ class Build(object): # different configurations we can run under -_CONFIGS = { - 'dbg': SimpleConfig('dbg'), - 'opt': SimpleConfig('opt'), - 'tsan': SimpleConfig('tsan', timeout_multiplier=2, environ={ - 'TSAN_OPTIONS': 'suppressions=tools/tsan_suppressions.txt:halt_on_error=1:second_deadlock_stack=1'}), - 'msan': SimpleConfig('msan', timeout_multiplier=1.5), - 'ubsan': SimpleConfig('ubsan'), - 'asan': SimpleConfig('asan', timeout_multiplier=1.5, environ={ - 'ASAN_OPTIONS': 'suppressions=tools/asan_suppressions.txt:detect_leaks=1:color=always', - 'LSAN_OPTIONS': 'suppressions=tools/asan_suppressions.txt:report_objects=1'}), - 'asan-noleaks': SimpleConfig('asan', environ={ - 'ASAN_OPTIONS': 'detect_leaks=0:color=always'}), - 'gcov': SimpleConfig('gcov'), - 'memcheck': ValgrindConfig('valgrind', 'memcheck', ['--leak-check=full']), - 'helgrind': ValgrindConfig('dbg', 'helgrind') - } +with open('tools/run_tests/configs.json') as f: + _CONFIGS = dict((cfg['config'], Config(**cfg)) for cfg in json.loads(f.read())) _DEFAULT = ['opt'] @@ -1102,3 +1068,4 @@ else: if BuildAndRunError.POST_TEST in errors: exit_code |= 4 sys.exit(exit_code) + -- cgit v1.2.3 From 1dce906aeaebb3d1c5c7a14811c6b40faa68b09f Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 20 Jan 2016 17:01:56 -0800 Subject: Use ast.literal_eval instead of json.loads to avoid unicode problems on windows --- tools/run_tests/run_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'tools') diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index e31fc30980..6e4a1499e7 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -480,7 +480,7 @@ class Build(object): # different configurations we can run under with open('tools/run_tests/configs.json') as f: - _CONFIGS = dict((cfg['config'], Config(**cfg)) for cfg in json.loads(f.read())) + _CONFIGS = dict((cfg['config'], Config(**cfg)) for cfg in ast.literal_eval(f.read())) _DEFAULT = ['opt'] -- cgit v1.2.3 From 9279ac246e975f3967a1321f2d561a00cf91d0e6 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 20 Jan 2016 17:05:23 -0800 Subject: Fix imports --- tools/run_tests/run_tests.py | 1 + 1 file changed, 1 insertion(+) (limited to 'tools') diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index 6e4a1499e7..a4aa2068c2 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -31,6 +31,7 @@ """Run tests in parallel.""" import argparse +import ast import glob import hashlib import itertools -- cgit v1.2.3 From 81df68d6474ed2afb6cb95d8e180cf5d6ff33713 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 21 Jan 2016 13:59:50 -0800 Subject: Add some time dilation for ASAN builds --- Makefile | 3 +++ build.yaml | 4 +++- tools/run_tests/configs.json | 6 ++++-- 3 files changed, 10 insertions(+), 3 deletions(-) (limited to 'tools') diff --git a/Makefile b/Makefile index 5abd21c863..e4dc2a6127 100644 --- a/Makefile +++ b/Makefile @@ -117,6 +117,7 @@ DEFINES_helgrind = _DEBUG DEBUG DEFINES_helgrind += GRPC_TEST_SLOWDOWN_BUILD_FACTOR=20 VALID_CONFIG_asan-noleaks = 1 +REQUIRE_CUSTOM_LIBRARIES_asan-noleaks = 1 CC_asan-noleaks = clang CXX_asan-noleaks = clang++ LD_asan-noleaks = clang @@ -124,6 +125,7 @@ LDXX_asan-noleaks = clang++ CFLAGS_asan-noleaks = -O0 -fsanitize=address -fno-omit-frame-pointer -Wno-unused-command-line-argument CXXFLAGS_asan-noleaks = -O0 -fsanitize=address -fno-omit-frame-pointer -Wno-unused-command-line-argument LDFLAGS_asan-noleaks = -fsanitize=address +DEFINES_asan-noleaks += GRPC_TEST_SLOWDOWN_BUILD_FACTOR=1.5 VALID_CONFIG_ubsan = 1 REQUIRE_CUSTOM_LIBRARIES_ubsan = 1 @@ -183,6 +185,7 @@ LDXX_asan = clang++ CFLAGS_asan = -O0 -fsanitize=address -fno-omit-frame-pointer -Wno-unused-command-line-argument CXXFLAGS_asan = -O0 -fsanitize=address -fno-omit-frame-pointer -Wno-unused-command-line-argument LDFLAGS_asan = -fsanitize=address +DEFINES_asan += GRPC_TEST_SLOWDOWN_BUILD_FACTOR=1.5 VALID_CONFIG_tsan = 1 REQUIRE_CUSTOM_LIBRARIES_tsan = 1 diff --git a/build.yaml b/build.yaml index 6af1f7165a..cf53af57d8 100644 --- a/build.yaml +++ b/build.yaml @@ -2494,6 +2494,7 @@ configs: test_environ: ASAN_OPTIONS: suppressions=tools/asan_suppressions.txt:detect_leaks=1:color=always LSAN_OPTIONS: suppressions=tools/asan_suppressions.txt:report_objects=1 + timeout_multiplier: 1.5 asan-noleaks: CC: clang CFLAGS: -O0 -fsanitize=address -fno-omit-frame-pointer -Wno-unused-command-line-argument @@ -2502,9 +2503,10 @@ configs: LD: clang LDFLAGS: -fsanitize=address LDXX: clang++ - custom_libraries: true + compile_the_world: true test_environ: ASAN_OPTIONS: detect_leaks=0:color=always + timeout_multiplier: 1.5 basicprof: CPPFLAGS: -O2 -DGRPC_BASIC_PROFILER -DGRPC_TIMERS_RDTSC DEFINES: NDEBUG diff --git a/tools/run_tests/configs.json b/tools/run_tests/configs.json index ef2ab61daa..769942df99 100644 --- a/tools/run_tests/configs.json +++ b/tools/run_tests/configs.json @@ -17,7 +17,8 @@ "config": "asan-noleaks", "environ": { "ASAN_OPTIONS": "detect_leaks=0:color=always" - } + }, + "timeout_multiplier": 1.5 }, { "config": "ubsan", @@ -46,7 +47,8 @@ "environ": { "ASAN_OPTIONS": "suppressions=tools/asan_suppressions.txt:detect_leaks=1:color=always", "LSAN_OPTIONS": "suppressions=tools/asan_suppressions.txt:report_objects=1" - } + }, + "timeout_multiplier": 1.5 }, { "config": "tsan", -- cgit v1.2.3