aboutsummaryrefslogtreecommitdiffhomepage
path: root/tools/run_tests/run_tests.py
diff options
context:
space:
mode:
Diffstat (limited to 'tools/run_tests/run_tests.py')
-rwxr-xr-xtools/run_tests/run_tests.py250
1 files changed, 158 insertions, 92 deletions
diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py
index c1254275e6..57fff2ec9c 100755
--- a/tools/run_tests/run_tests.py
+++ b/tools/run_tests/run_tests.py
@@ -30,13 +30,18 @@
"""Run tests in parallel."""
+from __future__ import print_function
+
import argparse
import ast
+import collections
import glob
import itertools
import json
import multiprocessing
import os
+import os.path
+import pipes
import platform
import random
import re
@@ -46,7 +51,7 @@ import sys
import tempfile
import traceback
import time
-import urllib2
+from six.moves import urllib
import uuid
import jobset
@@ -70,6 +75,9 @@ def platform_string():
return jobset.platform_string()
+_DEFAULT_TIMEOUT_SECONDS = 5 * 60
+
+
# SimpleConfig: just compile with CONFIG=config, and run the binary to test
class Config(object):
@@ -82,7 +90,7 @@ class Config(object):
self.tool_prefix = tool_prefix
self.timeout_multiplier = timeout_multiplier
- def job_spec(self, cmdline, timeout_seconds=5*60,
+ def job_spec(self, cmdline, timeout_seconds=_DEFAULT_TIMEOUT_SECONDS,
shortname=None, environ={}, cpu_cost=1.0, flaky=False):
"""Construct a jobset.JobSpec for a test under this config
@@ -91,7 +99,7 @@ class Config(object):
would like to run
"""
actual_environ = self.environ.copy()
- for k, v in environ.iteritems():
+ for k, v in environ.items():
actual_environ[k] = v
return jobset.JobSpec(cmdline=self.tool_prefix + cmdline,
shortname=shortname,
@@ -157,7 +165,7 @@ class CLanguage(object):
env={'GRPC_DEFAULT_SSL_ROOTS_FILE_PATH':
_ROOT + '/src/core/lib/tsi/test_creds/ca.pem',
'GRPC_POLL_STRATEGY': polling_strategy}
- shortname_ext = '' if polling_strategy=='all' else ' polling=%s' % polling_strategy
+ shortname_ext = '' if polling_strategy=='all' else ' GRPC_POLL_STRATEGY=%s' % polling_strategy
if self.config.build_config in target['exclude_configs']:
continue
if self.platform == 'windows':
@@ -188,28 +196,26 @@ class CLanguage(object):
assert line[1] == ' '
test = base + line.strip()
cmdline = [binary] + ['--gtest_filter=%s' % test]
- out.append(self.config.job_spec(cmdline, [binary],
- shortname='%s:%s %s' % (binary, test, shortname_ext),
+ out.append(self.config.job_spec(cmdline,
+ shortname='%s --gtest_filter=%s %s' % (binary, test, shortname_ext),
cpu_cost=target['cpu_cost'],
environ=env))
else:
cmdline = [binary] + target['args']
- out.append(self.config.job_spec(cmdline, [binary],
- shortname=' '.join(cmdline) + shortname_ext,
+ out.append(self.config.job_spec(cmdline,
+ shortname=' '.join(
+ pipes.quote(arg)
+ for arg in cmdline) +
+ shortname_ext,
cpu_cost=target['cpu_cost'],
flaky=target.get('flaky', False),
+ timeout_seconds=target.get('timeout_seconds', _DEFAULT_TIMEOUT_SECONDS),
environ=env))
elif self.args.regex == '.*' or self.platform == 'windows':
- print '\nWARNING: binary not found, skipping', binary
+ print('\nWARNING: binary not found, skipping', binary)
return sorted(out)
def make_targets(self):
- test_regex = self.args.regex
- if self.platform != 'windows' and self.args.regex != '.*':
- # use the regex to minimize the number of things to build
- return [os.path.basename(target['name'])
- for target in get_c_tests(False, self.test_lang)
- if re.search(test_regex, '/' + target['name'])]
if self.platform == 'windows':
# don't build tools on windows just yet
return ['buildtests_%s' % self.make_target]
@@ -372,50 +378,42 @@ class PhpLanguage(object):
return 'php'
+class PythonConfig(collections.namedtuple('PythonConfig', [
+ 'name', 'build', 'run'])):
+ """Tuple of commands (named s.t. 'what it says on the tin' applies)"""
+
class PythonLanguage(object):
def configure(self, config, args):
self.config = config
self.args = args
- self._tox_envs = self._get_tox_envs(self.args.compiler)
+ self.pythons = self._get_pythons(self.args)
def test_specs(self):
# load list of known test suites
- with open('src/python/grpcio/tests/tests.json') as tests_json_file:
+ with open('src/python/grpcio_tests/tests/tests.json') as tests_json_file:
tests_json = json.load(tests_json_file)
environment = dict(_FORCE_ENVIRON_FOR_WRAPPERS)
- environment['PYTHONPATH'] = '{}:{}'.format(
- os.path.abspath('src/python/gens'),
- os.path.abspath('src/python/grpcio_health_checking'))
- if self.config.build_config != 'gcov':
- return [self.config.job_spec(
- ['tools/run_tests/run_python.sh', tox_env],
- environ=dict(environment.items() +
- [('GRPC_PYTHON_TESTRUNNER_FILTER', suite_name)]),
- shortname='%s.test.%s' % (tox_env, suite_name),
- timeout_seconds=5*60)
- for suite_name in tests_json
- for tox_env in self._tox_envs]
- else:
- return [self.config.job_spec(['tools/run_tests/run_python.sh', tox_env],
- environ=environment,
- shortname='%s.test.coverage' % tox_env,
- timeout_seconds=15*60)
- for tox_env in self._tox_envs]
-
+ return [self.config.job_spec(
+ config.run,
+ timeout_seconds=5*60,
+ environ=dict(list(environment.items()) +
+ [('GRPC_PYTHON_TESTRUNNER_FILTER', suite_name)]),
+ shortname='%s.test.%s' % (config.name, suite_name),)
+ for suite_name in tests_json
+ for config in self.pythons]
def pre_build_steps(self):
return []
def make_targets(self):
- return ['static_c', 'grpc_python_plugin', 'shared_c']
+ return []
def make_options(self):
return []
def build_steps(self):
- return [['tools/run_tests/build_python.sh', tox_env]
- for tox_env in self._tox_envs]
+ return [config.build for config in self.pythons]
def post_tests_steps(self):
return []
@@ -424,18 +422,61 @@ class PythonLanguage(object):
return 'Makefile'
def dockerfile_dir(self):
- return 'tools/dockerfile/test/python_jessie_%s' % _docker_arch_suffix(self.args.arch)
-
- def _get_tox_envs(self, compiler):
- """Returns name of tox environment based on selected compiler."""
- if compiler == 'default':
- return ('py27', 'py34')
- elif compiler == 'python2.7':
- return ('py27',)
- elif compiler == 'python3.4':
- return ('py34',)
+ return 'tools/dockerfile/test/python_%s_%s' % (self.python_manager_name(), _docker_arch_suffix(self.args.arch))
+
+ def python_manager_name(self):
+ return 'pyenv' if self.args.compiler in ['python3.5', 'python3.6'] else 'jessie'
+
+ def _get_pythons(self, args):
+ if args.arch == 'x86':
+ bits = '32'
else:
- raise Exception('Compiler %s not supported.' % compiler)
+ bits = '64'
+ if os.name == 'nt':
+ shell = ['bash']
+ builder = [os.path.abspath('tools/run_tests/build_python_msys2.sh')]
+ builder_prefix_arguments = ['MINGW{}'.format(bits)]
+ venv_relative_python = ['Scripts/python.exe']
+ toolchain = ['mingw32']
+ python_pattern_function = lambda major, minor, bits: (
+ '/c/Python{major}{minor}/python.exe'.format(major=major, minor=minor, bits=bits)
+ if bits == '64' else
+ '/c/Python{major}{minor}_{bits}bits/python.exe'.format(
+ major=major, minor=minor, bits=bits))
+ else:
+ shell = []
+ builder = [os.path.abspath('tools/run_tests/build_python.sh')]
+ builder_prefix_arguments = []
+ venv_relative_python = ['bin/python']
+ toolchain = ['unix']
+ # Bit-ness is handled by the test machine's environment
+ python_pattern_function = lambda major, minor, bits: 'python{major}.{minor}'.format(major=major, minor=minor)
+ runner = [os.path.abspath('tools/run_tests/run_python.sh')]
+ python_config_generator = lambda name, major, minor, bits: PythonConfig(
+ name,
+ shell + builder + builder_prefix_arguments
+ + [python_pattern_function(major=major, minor=minor, bits=bits)]
+ + [name] + venv_relative_python + toolchain,
+ shell + runner + [os.path.join(name, venv_relative_python[0])])
+ python27_config = python_config_generator(name='py27', major='2', minor='7', bits=bits)
+ python34_config = python_config_generator(name='py34', major='3', minor='4', bits=bits)
+ python35_config = python_config_generator(name='py35', major='3', minor='5', bits=bits)
+ python36_config = python_config_generator(name='py36', major='3', minor='6', bits=bits)
+ if args.compiler == 'default':
+ if os.name == 'nt':
+ return (python27_config,)
+ else:
+ return (python27_config, python34_config,)
+ elif args.compiler == 'python2.7':
+ return (python27_config,)
+ elif args.compiler == 'python3.4':
+ return (python34_config,)
+ elif args.compiler == 'python3.5':
+ return (python35_config,)
+ elif args.compiler == 'python3.6':
+ return (python36_config,)
+ else:
+ raise Exception('Compiler %s not supported.' % args.compiler)
def __str__(self):
return 'python'
@@ -621,17 +662,20 @@ class ObjCLanguage(object):
_check_compiler(self.args.compiler, ['default'])
def test_specs(self):
- return [self.config.job_spec(['src/objective-c/tests/run_tests.sh'], None,
- environ=_FORCE_ENVIRON_FOR_WRAPPERS),
+ return [self.config.job_spec(['src/objective-c/tests/run_tests.sh'],
+ timeout_seconds=None,
+ shortname='objc-tests',
+ environ=_FORCE_ENVIRON_FOR_WRAPPERS),
self.config.job_spec(['src/objective-c/tests/build_example_test.sh'],
- None, timeout_seconds=15*60,
+ timeout_seconds=15*60,
+ shortname='objc-examples-build',
environ=_FORCE_ENVIRON_FOR_WRAPPERS)]
def pre_build_steps(self):
return []
def make_targets(self):
- return ['grpc_objective_c_plugin', 'interop_server']
+ return ['interop_server']
def make_options(self):
return []
@@ -724,7 +768,7 @@ def _windows_arch_option(arch):
elif arch == 'x64':
return '/p:Platform=x64'
else:
- print 'Architecture %s not supported.' % arch
+ print('Architecture %s not supported.' % arch)
sys.exit(1)
@@ -742,11 +786,11 @@ def _check_arch_option(arch):
elif runtime_arch == '32bit' and arch == 'x86':
return
else:
- print 'Architecture %s does not match current runtime architecture.' % arch
+ print('Architecture %s does not match current runtime architecture.' % arch)
sys.exit(1)
else:
if args.arch != 'default':
- print 'Architecture %s not supported on current platform.' % args.arch
+ print('Architecture %s not supported on current platform.' % args.arch)
sys.exit(1)
@@ -760,7 +804,7 @@ def _windows_build_bat(compiler):
elif compiler == 'vs2010':
return 'vsprojects\\build_vs2010.bat'
else:
- print 'Compiler %s not supported.' % compiler
+ print('Compiler %s not supported.' % compiler)
sys.exit(1)
@@ -774,7 +818,7 @@ def _windows_toolset_option(compiler):
elif compiler == 'vs2010':
return '/p:PlatformToolset=v100'
else:
- print 'Compiler %s not supported.' % compiler
+ print('Compiler %s not supported.' % compiler)
sys.exit(1)
@@ -785,7 +829,7 @@ def _docker_arch_suffix(arch):
elif arch == 'x86':
return 'x86'
else:
- print 'Architecture %s not supported with current settings.' % arch
+ print('Architecture %s not supported with current settings.' % arch)
sys.exit(1)
@@ -862,7 +906,7 @@ argp.add_argument('--compiler',
'gcc4.4', 'gcc4.6', 'gcc4.9', 'gcc5.3',
'clang3.4', 'clang3.5', 'clang3.6', 'clang3.7',
'vs2010', 'vs2013', 'vs2015',
- 'python2.7', 'python3.4',
+ 'python2.7', 'python3.4', 'python3.5', 'python3.6',
'node0.12', 'node4', 'node5',
'coreclr'],
default='default',
@@ -901,7 +945,7 @@ for spec in args.update_submodules:
branch = spec[1]
cwd = 'third_party/%s' % submodule
def git(cmd, cwd=cwd):
- print 'in %s: git %s' % (cwd, cmd)
+ print('in %s: git %s' % (cwd, cmd))
subprocess.check_call('git %s' % cmd, cwd=cwd, shell=True)
git('fetch')
git('checkout %s' % branch)
@@ -912,8 +956,8 @@ if need_to_regenerate_projects:
if jobset.platform_string() == 'linux':
subprocess.check_call('tools/buildgen/generate_projects.sh', shell=True)
else:
- print 'WARNING: may need to regenerate projects, but since we are not on'
- print ' Linux this step is being skipped. Compilation MAY fail.'
+ print('WARNING: may need to regenerate projects, but since we are not on')
+ print(' Linux this step is being skipped. Compilation MAY fail.')
# grab config
@@ -940,18 +984,18 @@ for l in languages:
language_make_options=[]
if any(language.make_options() for language in languages):
if not 'gcov' in args.config and len(languages) != 1:
- print 'languages with custom make options cannot be built simultaneously with other languages'
+ print('languages with custom make options cannot be built simultaneously with other languages')
sys.exit(1)
else:
language_make_options = next(iter(languages)).make_options()
if args.use_docker:
if not args.travis:
- print 'Seen --use_docker flag, will run tests under docker.'
- print
- print 'IMPORTANT: The changes you are testing need to be locally committed'
- print 'because only the committed changes in the current branch will be'
- print 'copied to the docker environment.'
+ print('Seen --use_docker flag, will run tests under docker.')
+ print('')
+ print('IMPORTANT: The changes you are testing need to be locally committed')
+ print('because only the committed changes in the current branch will be')
+ print('copied to the docker environment.')
time.sleep(5)
dockerfile_dirs = set([l.dockerfile_dir() for l in languages])
@@ -1035,7 +1079,7 @@ build_steps = list(set(
for l in languages
for cmdline in l.pre_build_steps()))
if make_targets:
- make_commands = itertools.chain.from_iterable(make_jobspec(build_config, list(targets), makefile) for (makefile, targets) in make_targets.iteritems())
+ make_commands = itertools.chain.from_iterable(make_jobspec(build_config, list(targets), makefile) for (makefile, targets) in make_targets.items())
build_steps.extend(set(make_commands))
build_steps.extend(set(
jobset.JobSpec(cmdline, environ=build_step_environ(build_config), timeout_seconds=None)
@@ -1050,35 +1094,59 @@ runs_per_test = args.runs_per_test
forever = args.forever
+def _shut_down_legacy_server(legacy_server_port):
+ try:
+ version = int(urllib.request.urlopen(
+ 'http://localhost:%d/version_number' % legacy_server_port,
+ timeout=10).read())
+ except:
+ pass
+ else:
+ urllib.request.urlopen(
+ 'http://localhost:%d/quitquitquit' % legacy_server_port).read()
+
+
+def _shut_down_legacy_server(legacy_server_port):
+ try:
+ version = int(urllib2.urlopen(
+ 'http://localhost:%d/version_number' % legacy_server_port,
+ timeout=10).read())
+ except:
+ pass
+ else:
+ urllib2.urlopen(
+ 'http://localhost:%d/quitquitquit' % legacy_server_port).read()
+
+
def _start_port_server(port_server_port):
# check if a compatible port server is running
# if incompatible (version mismatch) ==> start a new one
# if not running ==> start a new one
# otherwise, leave it up
try:
- version = int(urllib2.urlopen(
+ version = int(urllib.request.urlopen(
'http://localhost:%d/version_number' % port_server_port,
timeout=10).read())
- print 'detected port server running version %d' % version
+ print('detected port server running version %d' % version)
running = True
except Exception as e:
- print 'failed to detect port server: %s' % sys.exc_info()[0]
- print e.strerror
+ print('failed to detect port server: %s' % sys.exc_info()[0])
+ print(e.strerror)
running = False
if running:
current_version = int(subprocess.check_output(
[sys.executable, os.path.abspath('tools/run_tests/port_server.py'),
'dump_version']))
- print 'my port server is version %d' % current_version
+ print('my port server is version %d' % current_version)
running = (version >= current_version)
if not running:
- print 'port_server version mismatch: killing the old one'
- urllib2.urlopen('http://localhost:%d/quitquitquit' % port_server_port).read()
+ print('port_server version mismatch: killing the old one')
+ urllib.request.urlopen('http://localhost:%d/quitquitquit' % port_server_port).read()
time.sleep(1)
if not running:
fd, logfile = tempfile.mkstemp()
os.close(fd)
- print 'starting port_server, with log file %s' % logfile
+ print('starting port_server, with log file %s' % logfile)
args = [sys.executable, os.path.abspath('tools/run_tests/port_server.py'),
'-p', '%d' % port_server_port, '-l', logfile]
env = dict(os.environ)
@@ -1104,34 +1172,34 @@ def _start_port_server(port_server_port):
waits = 0
while True:
if waits > 10:
- print 'killing port server due to excessive start up waits'
+ print('killing port server due to excessive start up waits')
port_server.kill()
if port_server.poll() is not None:
- print 'port_server failed to start'
+ print('port_server failed to start')
# try one final time: maybe another build managed to start one
time.sleep(1)
try:
- urllib2.urlopen('http://localhost:%d/get' % port_server_port,
+ urllib.request.urlopen('http://localhost:%d/get' % port_server_port,
timeout=1).read()
- print 'last ditch attempt to contact port server succeeded'
+ print('last ditch attempt to contact port server succeeded')
break
except:
traceback.print_exc()
port_log = open(logfile, 'r').read()
- print port_log
+ print(port_log)
sys.exit(1)
try:
- urllib2.urlopen('http://localhost:%d/get' % port_server_port,
+ urllib.request.urlopen('http://localhost:%d/get' % port_server_port,
timeout=1).read()
- print 'port server is up and ready'
+ print('port server is up and ready')
break
except socket.timeout:
- print 'waiting for port_server: timeout'
+ print('waiting for port_server: timeout')
traceback.print_exc();
time.sleep(1)
waits += 1
- except urllib2.URLError:
- print 'waiting for port_server: urlerror'
+ except urllib.error.URLError:
+ print('waiting for port_server: urlerror')
traceback.print_exc();
time.sleep(1)
waits += 1
@@ -1186,7 +1254,7 @@ def _build_and_run(
# start antagonists
antagonists = [subprocess.Popen(['tools/run_tests/antagonist.py'])
for _ in range(0, args.antagonists)]
- port_server_port = 32767
+ port_server_port = 32766
_start_port_server(port_server_port)
resultset = None
num_test_failures = 0
@@ -1228,8 +1296,6 @@ def _build_and_run(
jobset.message(
'FLAKE', '%s [%d/%d runs flaked]' % (k, num_failures, num_runs),
do_newline=True)
- else:
- jobset.message('PASSED', k, do_newline=True)
finally:
for antagonist in antagonists:
antagonist.kill()