simplify run_tests.py

1 files changed, 135 insertions, 122 deletions

diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py
index 41cf335a81..7f156f7b51 100755
--- a/tools/run_tests/run_tests.py
+++ b/tools/run_tests/run_tests.py
@@ -118,6 +118,11 @@ def get_c_tests(travis, test_lang) :
                 not (travis and tgt['flaky'])]
 
 
+def _check_compiler(compiler, supported_compilers):
+  if compiler not in supported_compilers:
+    raise Exception('Compiler %s not supported.' % compiler)
+
+
 class CLanguage(object):
 
   def __init__(self, make_target, test_lang):
@@ -125,38 +130,50 @@ class CLanguage(object):
     self.platform = platform_string()
     self.test_lang = test_lang
 
-  def test_specs(self, config, args):
+  def configure(self, config, args):
+    self.config = config
+    self.args = args
+    if self.platform == 'windows':
+      _check_compiler(self.args.compiler, ['default',
+                                           'vs2010',
+                                           'vs2013',
+                                           'vs2015'])
+    else:
+      _check_compiler(self.args.compiler, ['default'])
+
+  def test_specs(self):
     out = []
-    binaries = get_c_tests(args.travis, self.test_lang)
+    binaries = get_c_tests(self.args.travis, self.test_lang)
     for target in binaries:
-      if config.build_config in target['exclude_configs']:
+      if self.config.build_config in target['exclude_configs']:
         continue
       if self.platform == 'windows':
         binary = 'vsprojects/%s%s/%s.exe' % (
-            'x64/' if args.arch == 'x64' else '',
-            _WINDOWS_CONFIG[config.build_config],
+            'x64/' if self.args.arch == 'x64' else '',
+            _WINDOWS_CONFIG[self.config.build_config],
             target['name'])
       else:
-        binary = 'bins/%s/%s' % (config.build_config, target['name'])
+        binary = 'bins/%s/%s' % (self.config.build_config, target['name'])
       if os.path.isfile(binary):
         cmdline = [binary] + target['args']
-        out.append(config.job_spec(cmdline, [binary],
-                                   shortname=' '.join(cmdline),
-                                   cpu_cost=target['cpu_cost'],
-                                   environ={'GRPC_DEFAULT_SSL_ROOTS_FILE_PATH':
-                                            os.path.abspath(os.path.dirname(
-                                                sys.argv[0]) + '/../../src/core/tsi/test_creds/ca.pem')}))
-      elif args.regex == '.*' or platform_string() == 'windows':
+        out.append(self.config.job_spec(cmdline, [binary],
+                                        shortname=' '.join(cmdline),
+                                        cpu_cost=target['cpu_cost'],
+                                        environ={'GRPC_DEFAULT_SSL_ROOTS_FILE_PATH':
+                                                 os.path.abspath(os.path.dirname(
+                                                     sys.argv[0]) + '/../../src/core/tsi/test_creds/ca.pem')}))
+      elif self.args.regex == '.*' or self.platform == 'windows':
         print '\nWARNING: binary not found, skipping', binary
     return sorted(out)
 
-  def make_targets(self, test_regex):
-    if platform_string() != 'windows' and test_regex != '.*':
+  def make_targets(self):
+    test_regex = self.args.regex
+    if self.platform != 'windows' and self.args.regex != '.*':
       # use the regex to minimize the number of things to build
       return [os.path.basename(target['name'])
               for target in get_c_tests(False, self.test_lang)
               if re.search(test_regex, '/' + target['name'])]
-    if platform_string() == 'windows':
+    if self.platform == 'windows':
       # don't build tools on windows just yet
       return ['buildtests_%s' % self.make_target]
     return ['buildtests_%s' % self.make_target, 'tools_%s' % self.make_target]
@@ -182,11 +199,8 @@ class CLanguage(object):
   def makefile_name(self):
     return 'Makefile'
 
-  def supports_multi_config(self):
-    return True
-
-  def dockerfile_dir(self, config, arch):
-    return 'tools/dockerfile/test/cxx_jessie_%s' % _docker_arch_suffix(arch)
+  def dockerfile_dir(self):
+    return 'tools/dockerfile/test/cxx_jessie_%s' % _docker_arch_suffix(self.args.arch)
 
   def __str__(self):
     return self.make_target
@@ -198,13 +212,18 @@ class NodeLanguage(object):
     self.platform = platform_string()
     self.node_version = '0.12'
 
-  def test_specs(self, config, args):
+  def configure(self, config, args):
+    self.config = config
+    self.args = args
+    _check_compiler(self.args.compiler, ['default'])
+
+  def test_specs(self):
     if self.platform == 'windows':
-      return [config.job_spec(['tools\\run_tests\\run_node.bat'], None)]
+      return [self.config.job_spec(['tools\\run_tests\\run_node.bat'], None)]
     else:
-      return [config.job_spec(['tools/run_tests/run_node.sh', self.node_version],
-                              None,
-                              environ=_FORCE_ENVIRON_FOR_WRAPPERS)]
+      return [self.config.job_spec(['tools/run_tests/run_node.sh', self.node_version],
+                                   None,
+                                   environ=_FORCE_ENVIRON_FOR_WRAPPERS)]
 
   def pre_build_steps(self):
     if self.platform == 'windows':
@@ -212,7 +231,7 @@ class NodeLanguage(object):
     else:
       return [['tools/run_tests/pre_build_node.sh', self.node_version]]
 
-  def make_targets(self, test_regex):
+  def make_targets(self):
     return []
 
   def make_options(self):
@@ -230,11 +249,8 @@ class NodeLanguage(object):
   def makefile_name(self):
     return 'Makefile'
 
-  def supports_multi_config(self):
-    return False
-
-  def dockerfile_dir(self, config, arch):
-    return 'tools/dockerfile/test/node_jessie_%s' % _docker_arch_suffix(arch)
+  def dockerfile_dir(self):
+    return 'tools/dockerfile/test/node_jessie_%s' % _docker_arch_suffix(self.args.arch)
 
   def __str__(self):
     return 'node'
@@ -242,14 +258,19 @@ class NodeLanguage(object):
 
 class PhpLanguage(object):
 
-  def test_specs(self, config, args):
-    return [config.job_spec(['src/php/bin/run_tests.sh'], None,
-                            environ=_FORCE_ENVIRON_FOR_WRAPPERS)]
+  def configure(self, config, args):
+    self.config = config
+    self.args = args
+    _check_compiler(self.args.compiler, ['default'])
+
+  def test_specs(self):
+    return [self.config.job_spec(['src/php/bin/run_tests.sh'], None,
+                                  environ=_FORCE_ENVIRON_FOR_WRAPPERS)]
 
   def pre_build_steps(self):
     return []
 
-  def make_targets(self, test_regex):
+  def make_targets(self):
     return ['static_c', 'shared_c']
 
   def make_options(self):
@@ -264,11 +285,8 @@ class PhpLanguage(object):
   def makefile_name(self):
     return 'Makefile'
 
-  def supports_multi_config(self):
-    return False
-
-  def dockerfile_dir(self, config, arch):
-    return 'tools/dockerfile/test/php_jessie_%s' % _docker_arch_suffix(arch)
+  def dockerfile_dir(self):
+    return 'tools/dockerfile/test/php_jessie_%s' % _docker_arch_suffix(self.args.arch)
 
   def __str__(self):
     return 'php'
@@ -280,10 +298,15 @@ class PythonLanguage(object):
     self._build_python_versions = ['2.7']
     self._has_python_versions = []
 
-  def test_specs(self, config, args):
+  def configure(self, config, args):
+    self.config = config
+    self.args = args
+    _check_compiler(self.args.compiler, ['default'])
+
+  def test_specs(self):
     environment = dict(_FORCE_ENVIRON_FOR_WRAPPERS)
     environment['PYVER'] = '2.7'
-    return [config.job_spec(
+    return [self.config.job_spec(
         ['tools/run_tests/run_python.sh'],
         None,
         environ=environment,
@@ -294,7 +317,7 @@ class PythonLanguage(object):
   def pre_build_steps(self):
     return []
 
-  def make_targets(self, test_regex):
+  def make_targets(self):
     return ['static_c', 'grpc_python_plugin', 'shared_c']
 
   def make_options(self):
@@ -320,11 +343,8 @@ class PythonLanguage(object):
   def makefile_name(self):
     return 'Makefile'
 
-  def supports_multi_config(self):
-    return False
-
-  def dockerfile_dir(self, config, arch):
-    return 'tools/dockerfile/test/python_jessie_%s' % _docker_arch_suffix(arch)
+  def dockerfile_dir(self):
+    return 'tools/dockerfile/test/python_jessie_%s' % _docker_arch_suffix(self.args.arch)
 
   def __str__(self):
     return 'python'
@@ -332,10 +352,15 @@ class PythonLanguage(object):
 
 class RubyLanguage(object):
 
-  def test_specs(self, config, args):
-    return [config.job_spec(['tools/run_tests/run_ruby.sh'], None,
-                            timeout_seconds=10*60,
-                            environ=_FORCE_ENVIRON_FOR_WRAPPERS)]
+  def configure(self, config, args):
+    self.config = config
+    self.args = args
+    _check_compiler(self.args.compiler, ['default'])
+
+  def test_specs(self):
+    return [self.config.job_spec(['tools/run_tests/run_ruby.sh'], None,
+                                 timeout_seconds=10*60,
+                                 environ=_FORCE_ENVIRON_FOR_WRAPPERS)]
 
   def pre_build_steps(self):
     return [['tools/run_tests/pre_build_ruby.sh']]
@@ -355,27 +380,30 @@ class RubyLanguage(object):
   def makefile_name(self):
     return 'Makefile'
 
-  def supports_multi_config(self):
-    return False
-
-  def dockerfile_dir(self, config, arch):
-    return 'tools/dockerfile/test/ruby_jessie_%s' % _docker_arch_suffix(arch)
+  def dockerfile_dir(self):
+    return 'tools/dockerfile/test/ruby_jessie_%s' % _docker_arch_suffix(self.args.arch)
 
   def __str__(self):
     return 'ruby'
 
 
 class CSharpLanguage(object):
+
   def __init__(self):
     self.platform = platform_string()
 
-  def test_specs(self, config, args):
+  def configure(self, config, args):
+    self.config = config
+    self.args = args
+    _check_compiler(self.args.compiler, ['default'])
+
+  def test_specs(self):
     with open('src/csharp/tests.json') as f:
       tests_json = json.load(f)
     assemblies = tests_json['assemblies']
     tests = tests_json['tests']
 
-    msbuild_config = _WINDOWS_CONFIG[config.build_config]
+    msbuild_config = _WINDOWS_CONFIG[self.config.build_config]
     assembly_files = ['%s/bin/%s/%s.dll' % (a, msbuild_config, a)
                       for a in assemblies]
 
@@ -387,13 +415,13 @@ class CSharpLanguage(object):
     else:
       script_name = 'tools/run_tests/run_csharp.sh'
 
-    if config.build_config == 'gcov':
+    if self.config.build_config == 'gcov':
       # On Windows, we only collect C# code coverage.
       # On Linux, we only collect coverage for native extension.
       # For code coverage all tests need to run as one suite.
-      return [config.job_spec([script_name] + extra_args, None,
-                              shortname='csharp.coverage',
-                              environ=_FORCE_ENVIRON_FOR_WRAPPERS)]
+      return [self.config.job_spec([script_name] + extra_args, None,
+                                    shortname='csharp.coverage',
+                                    environ=_FORCE_ENVIRON_FOR_WRAPPERS)]
     else:
       specs = []
       for test in tests:
@@ -402,9 +430,9 @@ class CSharpLanguage(object):
           # use different output directory for each test to prevent
           # TestResult.xml clash between parallel test runs.
           cmdline += ['-work=test-result/%s' % uuid.uuid4()]
-        specs.append(config.job_spec(cmdline, None,
-                                     shortname='csharp.%s' % test,
-                                     environ=_FORCE_ENVIRON_FOR_WRAPPERS))
+        specs.append(self.config.job_spec(cmdline, None,
+                                          shortname='csharp.%s' % test,
+                                          environ=_FORCE_ENVIRON_FOR_WRAPPERS))
       return specs
 
   def pre_build_steps(self):
@@ -413,7 +441,7 @@ class CSharpLanguage(object):
     else:
       return [['tools/run_tests/pre_build_csharp.sh']]
 
-  def make_targets(self, test_regex):
+  def make_targets(self):
     # For Windows, this target doesn't really build anything,
     # everything is build by buildall script later.
     if self.platform == 'windows':
@@ -440,11 +468,8 @@ class CSharpLanguage(object):
   def makefile_name(self):
     return 'Makefile'
 
-  def supports_multi_config(self):
-    return False
-
-  def dockerfile_dir(self, config, arch):
-    return 'tools/dockerfile/test/csharp_jessie_%s' % _docker_arch_suffix(arch)
+  def dockerfile_dir(self):
+    return 'tools/dockerfile/test/csharp_jessie_%s' % _docker_arch_suffix(self.args.arch)
 
   def __str__(self):
     return 'csharp'
@@ -452,14 +477,19 @@ class CSharpLanguage(object):
 
 class ObjCLanguage(object):
 
-  def test_specs(self, config, args):
-    return [config.job_spec(['src/objective-c/tests/run_tests.sh'], None,
-                            environ=_FORCE_ENVIRON_FOR_WRAPPERS)]
+  def configure(self, config, args):
+    self.config = config
+    self.args = args
+    _check_compiler(self.args.compiler, ['default'])
+
+  def test_specs(self):
+    return [self.config.job_spec(['src/objective-c/tests/run_tests.sh'], None,
+                                  environ=_FORCE_ENVIRON_FOR_WRAPPERS)]
 
   def pre_build_steps(self):
     return []
 
-  def make_targets(self, test_regex):
+  def make_targets(self):
     return ['grpc_objective_c_plugin', 'interop_server']
 
   def make_options(self):
@@ -474,10 +504,7 @@ class ObjCLanguage(object):
   def makefile_name(self):
     return 'Makefile'
 
-  def supports_multi_config(self):
-    return False
-
-  def dockerfile_dir(self, config, arch):
+  def dockerfile_dir(self):
     return None
 
   def __str__(self):
@@ -486,18 +513,23 @@ class ObjCLanguage(object):
 
 class Sanity(object):
 
-  def test_specs(self, config, args):
+  def configure(self, config, args):
+    self.config = config
+    self.args = args
+    _check_compiler(self.args.compiler, ['default'])
+
+  def test_specs(self):
     import yaml
     with open('tools/run_tests/sanity/sanity_tests.yaml', 'r') as f:
-      return [config.job_spec(cmd['script'].split(), None,
-                              timeout_seconds=None, environ={'TEST': 'true'},
-                              cpu_cost=cmd.get('cpu_cost', 1))
+      return [self.config.job_spec(cmd['script'].split(), None,
+                                   timeout_seconds=None, environ={'TEST': 'true'},
+                                   cpu_cost=cmd.get('cpu_cost', 1))
               for cmd in yaml.load(f)]
 
   def pre_build_steps(self):
     return []
 
-  def make_targets(self, test_regex):
+  def make_targets(self):
     return ['run_dep_checks']
 
   def make_options(self):
@@ -512,10 +544,7 @@ class Sanity(object):
   def makefile_name(self):
     return 'Makefile'
 
-  def supports_multi_config(self):
-    return False
-
-  def dockerfile_dir(self, config, arch):
+  def dockerfile_dir(self):
     return 'tools/dockerfile/test/sanity'
 
   def __str__(self):
@@ -527,7 +556,6 @@ with open('tools/run_tests/configs.json') as f:
   _CONFIGS = dict((cfg['config'], Config(**cfg)) for cfg in ast.literal_eval(f.read()))
 
 
-_DEFAULT = ['opt']
 _LANGUAGES = {
     'c++': CLanguage('cxx', 'c++'),
     'c': CLanguage('c', 'c'),
@@ -540,6 +568,7 @@ _LANGUAGES = {
     'sanity': Sanity()
     }
 
+
 _WINDOWS_CONFIG = {
     'dbg': 'Debug',
     'opt': 'Release',
@@ -648,9 +677,8 @@ def runs_per_test_type(arg_str):
 # parse command line
 argp = argparse.ArgumentParser(description='Run grpc tests.')
 argp.add_argument('-c', '--config',
-                  choices=['all'] + sorted(_CONFIGS.keys()),
-                  nargs='+',
-                  default=_DEFAULT)
+                  choices=sorted(_CONFIGS.keys()),
+                  default='opt')
 argp.add_argument('-n', '--runs_per_test', default=1, type=runs_per_test_type,
         help='A positive integer or "inf". If "inf", all tests will run in an '
              'infinite loop. Especially useful in combination with "-f"')
@@ -696,7 +724,7 @@ argp.add_argument('--arch',
 argp.add_argument('--compiler',
                   choices=['default', 'vs2010', 'vs2013', 'vs2015'],
                   default='default',
-                  help='Selects compiler to use. For some platforms "default" is the only supported choice.')
+                  help='Selects compiler to use. Allowed values depend on the platform and language.')
 argp.add_argument('--build_only',
                   default=False,
                   action='store_const',
@@ -742,11 +770,8 @@ if need_to_regenerate_projects:
 
 
 # grab config
-run_configs = set(_CONFIGS[cfg]
-                  for cfg in itertools.chain.from_iterable(
-                      _CONFIGS.iterkeys() if x == 'all' else [x]
-                      for x in args.config))
-build_configs = set(cfg.build_config for cfg in run_configs)
+run_config = _CONFIGS[args.config]
+build_config = run_config.build_config
 
 if args.travis:
   _FORCE_ENVIRON_FOR_WRAPPERS = {'GRPC_TRACE': 'api'}
@@ -762,12 +787,8 @@ if 'gcov' in args.config:
       lang_list.remove(bad)
 
 languages = set(_LANGUAGES[l] for l in lang_list)
-
-if len(build_configs) > 1:
-  for language in languages:
-    if not language.supports_multi_config():
-      print language, 'does not support multiple build configurations'
-      sys.exit(1)
+for l in languages:
+  l.configure(run_config, args)
 
 language_make_options=[]
 if any(language.make_options() for language in languages):
@@ -777,8 +798,8 @@ if any(language.make_options() for language in languages):
   else:
     language_make_options = next(iter(languages)).make_options()
 
-if len(languages) != 1 or len(build_configs) != 1:
-  print 'Multi-language and multi-config testing is not supported.'
+if len(languages) != 1:
+  print 'Multi-language testing is not supported.'
   sys.exit(1)
 
 if args.use_docker:
@@ -809,10 +830,6 @@ if args.use_docker:
                         env=env)
   sys.exit(0)
 
-if platform_string() != 'windows' and args.compiler != 'default':
-    print 'Compiler %s not supported on current platform.' % args.compiler
-    sys.exit(1)
-
 _check_arch_option(args.arch)
 
 def make_jobspec(cfg, targets, makefile='Makefile'):
@@ -852,7 +869,7 @@ make_targets = {}
 for l in languages:
   makefile = l.makefile_name()
   make_targets[makefile] = make_targets.get(makefile, set()).union(
-      set(l.make_targets(args.regex)))
+      set(l.make_targets()))
 
 def build_step_environ(cfg):
   environ = {'CONFIG': cfg}
@@ -862,22 +879,19 @@ def build_step_environ(cfg):
   return environ
 
 build_steps = list(set(
-                   jobset.JobSpec(cmdline, environ=build_step_environ(cfg), flake_retries=5)
-                   for cfg in build_configs
+                   jobset.JobSpec(cmdline, environ=build_step_environ(build_config), flake_retries=5)
                    for l in languages
                    for cmdline in l.pre_build_steps()))
 if make_targets:
-  make_commands = itertools.chain.from_iterable(make_jobspec(cfg, list(targets), makefile) for cfg in build_configs for (makefile, targets) in make_targets.iteritems())
+  make_commands = itertools.chain.from_iterable(make_jobspec(build_config, list(targets), makefile) for (makefile, targets) in make_targets.iteritems())
   build_steps.extend(set(make_commands))
 build_steps.extend(set(
-                   jobset.JobSpec(cmdline, environ=build_step_environ(cfg), timeout_seconds=None)
-                   for cfg in build_configs
+                   jobset.JobSpec(cmdline, environ=build_step_environ(build_config), timeout_seconds=None)
                    for l in languages
                    for cmdline in l.build_steps()))
 
 post_tests_steps = list(set(
-                        jobset.JobSpec(cmdline, environ=build_step_environ(cfg))
-                        for cfg in build_configs
+                        jobset.JobSpec(cmdline, environ=build_step_environ(build_config))
                         for l in languages
                         for cmdline in l.post_tests_steps()))
 runs_per_test = args.runs_per_test
@@ -1068,9 +1082,8 @@ def _build_and_run(
     infinite_runs = runs_per_test == 0
     one_run = set(
       spec
-      for config in run_configs
       for language in languages
-      for spec in language.test_specs(config, args)
+      for spec in language.test_specs()
       if re.search(args.regex, spec.shortname))
     # When running on travis, we want out test runs to be as similar as possible
     # for reproducibility purposes.