diff options
| -rwxr-xr-x | tools/distrib/yapf_code.sh | 1 | ||||
| -rw-r--r-- | tools/run_tests/artifacts/artifact_targets.py | 611 | ||||
| -rw-r--r-- | tools/run_tests/artifacts/distribtest_targets.py | 525 | ||||
| -rw-r--r-- | tools/run_tests/artifacts/package_targets.py | 214 |
4 files changed, 709 insertions, 642 deletions
diff --git a/tools/distrib/yapf_code.sh b/tools/distrib/yapf_code.sh index 8f34e55245..6d2759ddd1 100755 --- a/tools/distrib/yapf_code.sh +++ b/tools/distrib/yapf_code.sh @@ -28,6 +28,7 @@ DIRS=( 'tools/run_tests/python_utils' 'tools/run_tests/sanity' 'tools/run_tests/performance' + 'tools/run_tests/artifacts' ) EXCLUSIONS=( 'grpcio/grpc_*.py' diff --git a/tools/run_tests/artifacts/artifact_targets.py b/tools/run_tests/artifacts/artifact_targets.py index ea202edb30..dc0803b3c7 100644 --- a/tools/run_tests/artifacts/artifact_targets.py +++ b/tools/run_tests/artifacts/artifact_targets.py @@ -12,7 +12,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - """Definition of targets to build artifacts.""" import os.path @@ -24,316 +23,350 @@ sys.path.insert(0, os.path.abspath('..')) import python_utils.jobset as jobset -def create_docker_jobspec(name, dockerfile_dir, shell_command, environ={}, - flake_retries=0, timeout_retries=0, timeout_seconds=30*60, - docker_base_image=None, extra_docker_args=None): - """Creates jobspec for a task running under docker.""" - environ = environ.copy() - environ['RUN_COMMAND'] = shell_command - environ['ARTIFACTS_OUT'] = 'artifacts/%s' % name - - docker_args=[] - for k,v in environ.items(): - docker_args += ['-e', '%s=%s' % (k, v)] - docker_env = {'DOCKERFILE_DIR': dockerfile_dir, - 'DOCKER_RUN_SCRIPT': 'tools/run_tests/dockerize/docker_run.sh', - 'OUTPUT_DIR': 'artifacts'} - - if docker_base_image is not None: - docker_env['DOCKER_BASE_IMAGE'] = docker_base_image - if extra_docker_args is not None: - docker_env['EXTRA_DOCKER_ARGS'] = extra_docker_args - jobspec = jobset.JobSpec( - cmdline=['tools/run_tests/dockerize/build_and_run_docker.sh'] + docker_args, - environ=docker_env, - shortname='build_artifact.%s' % (name), - timeout_seconds=timeout_seconds, - flake_retries=flake_retries, - timeout_retries=timeout_retries) - return jobspec - - -def create_jobspec(name, cmdline, environ={}, shell=False, - flake_retries=0, timeout_retries=0, timeout_seconds=30*60, +def create_docker_jobspec(name, + dockerfile_dir, + shell_command, + environ={}, + flake_retries=0, + timeout_retries=0, + timeout_seconds=30 * 60, + docker_base_image=None, + extra_docker_args=None): + """Creates jobspec for a task running under docker.""" + environ = environ.copy() + environ['RUN_COMMAND'] = shell_command + environ['ARTIFACTS_OUT'] = 'artifacts/%s' % name + + docker_args = [] + for k, v in environ.items(): + docker_args += ['-e', '%s=%s' % (k, v)] + docker_env = { + 'DOCKERFILE_DIR': dockerfile_dir, + 'DOCKER_RUN_SCRIPT': 'tools/run_tests/dockerize/docker_run.sh', + 'OUTPUT_DIR': 'artifacts' + } + + if docker_base_image is not None: + docker_env['DOCKER_BASE_IMAGE'] = docker_base_image + if extra_docker_args is not None: + docker_env['EXTRA_DOCKER_ARGS'] = extra_docker_args + jobspec = jobset.JobSpec( + cmdline=['tools/run_tests/dockerize/build_and_run_docker.sh'] + + docker_args, + environ=docker_env, + shortname='build_artifact.%s' % (name), + timeout_seconds=timeout_seconds, + flake_retries=flake_retries, + timeout_retries=timeout_retries) + return jobspec + + +def create_jobspec(name, + cmdline, + environ={}, + shell=False, + flake_retries=0, + timeout_retries=0, + timeout_seconds=30 * 60, use_workspace=False, cpu_cost=1.0): - """Creates jobspec.""" - environ = environ.copy() - if use_workspace: - environ['WORKSPACE_NAME'] = 'workspace_%s' % name - environ['ARTIFACTS_OUT'] = os.path.join('..', 'artifacts', name) - cmdline = ['bash', - 'tools/run_tests/artifacts/run_in_workspace.sh'] + cmdline - else: - environ['ARTIFACTS_OUT'] = os.path.join('artifacts', name) - - jobspec = jobset.JobSpec( - cmdline=cmdline, - environ=environ, - shortname='build_artifact.%s' % (name), - timeout_seconds=timeout_seconds, - flake_retries=flake_retries, - timeout_retries=timeout_retries, - shell=shell, - cpu_cost=cpu_cost) - return jobspec + """Creates jobspec.""" + environ = environ.copy() + if use_workspace: + environ['WORKSPACE_NAME'] = 'workspace_%s' % name + environ['ARTIFACTS_OUT'] = os.path.join('..', 'artifacts', name) + cmdline = ['bash', 'tools/run_tests/artifacts/run_in_workspace.sh' + ] + cmdline + else: + environ['ARTIFACTS_OUT'] = os.path.join('artifacts', name) + + jobspec = jobset.JobSpec( + cmdline=cmdline, + environ=environ, + shortname='build_artifact.%s' % (name), + timeout_seconds=timeout_seconds, + flake_retries=flake_retries, + timeout_retries=timeout_retries, + shell=shell, + cpu_cost=cpu_cost) + return jobspec _MACOS_COMPAT_FLAG = '-mmacosx-version-min=10.7' -_ARCH_FLAG_MAP = { - 'x86': '-m32', - 'x64': '-m64' -} +_ARCH_FLAG_MAP = {'x86': '-m32', 'x64': '-m64'} class PythonArtifact: - """Builds Python artifacts.""" - - def __init__(self, platform, arch, py_version): - self.name = 'python_%s_%s_%s' % (platform, arch, py_version) - self.platform = platform - self.arch = arch - self.labels = ['artifact', 'python', platform, arch, py_version] - self.py_version = py_version - - def pre_build_jobspecs(self): - return [] - - def build_jobspec(self): - environ = {} - if self.platform == 'linux_extra': - # Raspberry Pi build - environ['PYTHON'] = '/usr/local/bin/python{}'.format(self.py_version) - environ['PIP'] = '/usr/local/bin/pip{}'.format(self.py_version) - # https://github.com/resin-io-projects/armv7hf-debian-qemu/issues/9 - # A QEMU bug causes submodule update to hang, so we copy directly - environ['RELATIVE_COPY_PATH'] = '.' - extra_args = ' --entrypoint=/usr/bin/qemu-arm-static ' - return create_docker_jobspec(self.name, - 'tools/dockerfile/grpc_artifact_linux_{}'.format(self.arch), - 'tools/run_tests/artifacts/build_artifact_python.sh', - environ=environ, - timeout_seconds=60*60*5, - docker_base_image='quay.io/grpc/raspbian_{}'.format(self.arch), - extra_docker_args=extra_args) - elif self.platform == 'linux': - if self.arch == 'x86': - environ['SETARCH_CMD'] = 'linux32' - # Inside the manylinux container, the python installations are located in - # special places... - environ['PYTHON'] = '/opt/python/{}/bin/python'.format(self.py_version) - environ['PIP'] = '/opt/python/{}/bin/pip'.format(self.py_version) - # Platform autodetection for the manylinux1 image breaks so we set the - # defines ourselves. - # TODO(atash) get better platform-detection support in core so we don't - # need to do this manually... - environ['CFLAGS'] = '-DGPR_MANYLINUX1=1' - environ['GRPC_BUILD_GRPCIO_TOOLS_DEPENDENTS'] = 'TRUE' - environ['GRPC_BUILD_MANYLINUX_WHEEL'] = 'TRUE' - return create_docker_jobspec(self.name, - 'tools/dockerfile/grpc_artifact_python_manylinux_%s' % self.arch, - 'tools/run_tests/artifacts/build_artifact_python.sh', - environ=environ, - timeout_seconds=60*60, - docker_base_image='quay.io/pypa/manylinux1_i686' if self.arch == 'x86' else 'quay.io/pypa/manylinux1_x86_64') - elif self.platform == 'windows': - if 'Python27' in self.py_version or 'Python34' in self.py_version: - environ['EXT_COMPILER'] = 'mingw32' - else: - environ['EXT_COMPILER'] = 'msvc' - # For some reason, the batch script %random% always runs with the same - # seed. We create a random temp-dir here - dir = ''.join(random.choice(string.ascii_uppercase) for _ in range(10)) - return create_jobspec(self.name, - ['tools\\run_tests\\artifacts\\build_artifact_python.bat', - self.py_version, - '32' if self.arch == 'x86' else '64'], - environ=environ, - timeout_seconds=45*60, - use_workspace=True) - else: - environ['PYTHON'] = self.py_version - environ['SKIP_PIP_INSTALL'] = 'TRUE' - return create_jobspec(self.name, - ['tools/run_tests/artifacts/build_artifact_python.sh'], - environ=environ, - timeout_seconds=60*60, - use_workspace=True) - - def __str__(self): - return self.name + """Builds Python artifacts.""" + + def __init__(self, platform, arch, py_version): + self.name = 'python_%s_%s_%s' % (platform, arch, py_version) + self.platform = platform + self.arch = arch + self.labels = ['artifact', 'python', platform, arch, py_version] + self.py_version = py_version + + def pre_build_jobspecs(self): + return [] + + def build_jobspec(self): + environ = {} + if self.platform == 'linux_extra': + # Raspberry Pi build + environ['PYTHON'] = '/usr/local/bin/python{}'.format( + self.py_version) + environ['PIP'] = '/usr/local/bin/pip{}'.format(self.py_version) + # https://github.com/resin-io-projects/armv7hf-debian-qemu/issues/9 + # A QEMU bug causes submodule update to hang, so we copy directly + environ['RELATIVE_COPY_PATH'] = '.' + extra_args = ' --entrypoint=/usr/bin/qemu-arm-static ' + return create_docker_jobspec( + self.name, + 'tools/dockerfile/grpc_artifact_linux_{}'.format(self.arch), + 'tools/run_tests/artifacts/build_artifact_python.sh', + environ=environ, + timeout_seconds=60 * 60 * 5, + docker_base_image='quay.io/grpc/raspbian_{}'.format(self.arch), + extra_docker_args=extra_args) + elif self.platform == 'linux': + if self.arch == 'x86': + environ['SETARCH_CMD'] = 'linux32' + # Inside the manylinux container, the python installations are located in + # special places... + environ['PYTHON'] = '/opt/python/{}/bin/python'.format( + self.py_version) + environ['PIP'] = '/opt/python/{}/bin/pip'.format(self.py_version) + # Platform autodetection for the manylinux1 image breaks so we set the + # defines ourselves. + # TODO(atash) get better platform-detection support in core so we don't + # need to do this manually... + environ['CFLAGS'] = '-DGPR_MANYLINUX1=1' + environ['GRPC_BUILD_GRPCIO_TOOLS_DEPENDENTS'] = 'TRUE' + environ['GRPC_BUILD_MANYLINUX_WHEEL'] = 'TRUE' + return create_docker_jobspec( + self.name, + 'tools/dockerfile/grpc_artifact_python_manylinux_%s' % + self.arch, + 'tools/run_tests/artifacts/build_artifact_python.sh', + environ=environ, + timeout_seconds=60 * 60, + docker_base_image='quay.io/pypa/manylinux1_i686' + if self.arch == 'x86' else 'quay.io/pypa/manylinux1_x86_64') + elif self.platform == 'windows': + if 'Python27' in self.py_version or 'Python34' in self.py_version: + environ['EXT_COMPILER'] = 'mingw32' + else: + environ['EXT_COMPILER'] = 'msvc' + # For some reason, the batch script %random% always runs with the same + # seed. We create a random temp-dir here + dir = ''.join( + random.choice(string.ascii_uppercase) for _ in range(10)) + return create_jobspec( + self.name, [ + 'tools\\run_tests\\artifacts\\build_artifact_python.bat', + self.py_version, '32' if self.arch == 'x86' else '64' + ], + environ=environ, + timeout_seconds=45 * 60, + use_workspace=True) + else: + environ['PYTHON'] = self.py_version + environ['SKIP_PIP_INSTALL'] = 'TRUE' + return create_jobspec( + self.name, + ['tools/run_tests/artifacts/build_artifact_python.sh'], + environ=environ, + timeout_seconds=60 * 60, + use_workspace=True) + + def __str__(self): + return self.name class RubyArtifact: - """Builds ruby native gem.""" + """Builds ruby native gem.""" - def __init__(self, platform, arch): - self.name = 'ruby_native_gem_%s_%s' % (platform, arch) - self.platform = platform - self.arch = arch - self.labels = ['artifact', 'ruby', platform, arch] + def __init__(self, platform, arch): + self.name = 'ruby_native_gem_%s_%s' % (platform, arch) + self.platform = platform + self.arch = arch + self.labels = ['artifact', 'ruby', platform, arch] - def pre_build_jobspecs(self): - return [] + def pre_build_jobspecs(self): + return [] - def build_jobspec(self): - # Ruby build uses docker internally and docker cannot be nested. - # We are using a custom workspace instead. - return create_jobspec(self.name, - ['tools/run_tests/artifacts/build_artifact_ruby.sh'], - use_workspace=True, - timeout_seconds=45*60) + def build_jobspec(self): + # Ruby build uses docker internally and docker cannot be nested. + # We are using a custom workspace instead. + return create_jobspec( + self.name, ['tools/run_tests/artifacts/build_artifact_ruby.sh'], + use_workspace=True, + timeout_seconds=45 * 60) class CSharpExtArtifact: - """Builds C# native extension library""" - - def __init__(self, platform, arch): - self.name = 'csharp_ext_%s_%s' % (platform, arch) - self.platform = platform - self.arch = arch - self.labels = ['artifact', 'csharp', platform, arch] - - def pre_build_jobspecs(self): - return [] - - def build_jobspec(self): - if self.platform == 'windows': - cmake_arch_option = 'Win32' if self.arch == 'x86' else self.arch - return create_jobspec(self.name, - ['tools\\run_tests\\artifacts\\build_artifact_csharp.bat', - cmake_arch_option], - use_workspace=True) - else: - environ = {'CONFIG': 'opt', - 'EMBED_OPENSSL': 'true', - 'EMBED_ZLIB': 'true', - 'CFLAGS': '-DGPR_BACKWARDS_COMPATIBILITY_MODE', - 'CXXFLAGS': '-DGPR_BACKWARDS_COMPATIBILITY_MODE', - 'LDFLAGS': ''} - if self.platform == 'linux': - return create_docker_jobspec(self.name, - 'tools/dockerfile/grpc_artifact_linux_%s' % self.arch, - 'tools/run_tests/artifacts/build_artifact_csharp.sh', - environ=environ) - else: - archflag = _ARCH_FLAG_MAP[self.arch] - environ['CFLAGS'] += ' %s %s' % (archflag, _MACOS_COMPAT_FLAG) - environ['CXXFLAGS'] += ' %s %s' % (archflag, _MACOS_COMPAT_FLAG) - environ['LDFLAGS'] += ' %s' % archflag - return create_jobspec(self.name, - ['tools/run_tests/artifacts/build_artifact_csharp.sh'], - environ=environ, - use_workspace=True) - - def __str__(self): - return self.name + """Builds C# native extension library""" + + def __init__(self, platform, arch): + self.name = 'csharp_ext_%s_%s' % (platform, arch) + self.platform = platform + self.arch = arch + self.labels = ['artifact', 'csharp', platform, arch] + + def pre_build_jobspecs(self): + return [] + + def build_jobspec(self): + if self.platform == 'windows': + cmake_arch_option = 'Win32' if self.arch == 'x86' else self.arch + return create_jobspec( + self.name, [ + 'tools\\run_tests\\artifacts\\build_artifact_csharp.bat', + cmake_arch_option + ], + use_workspace=True) + else: + environ = { + 'CONFIG': 'opt', + 'EMBED_OPENSSL': 'true', + 'EMBED_ZLIB': 'true', + 'CFLAGS': '-DGPR_BACKWARDS_COMPATIBILITY_MODE', + 'CXXFLAGS': '-DGPR_BACKWARDS_COMPATIBILITY_MODE', + 'LDFLAGS': '' + } + if self.platform == 'linux': + return create_docker_jobspec( + self.name, + 'tools/dockerfile/grpc_artifact_linux_%s' % self.arch, + 'tools/run_tests/artifacts/build_artifact_csharp.sh', + environ=environ) + else: + archflag = _ARCH_FLAG_MAP[self.arch] + environ['CFLAGS'] += ' %s %s' % (archflag, _MACOS_COMPAT_FLAG) + environ['CXXFLAGS'] += ' %s %s' % (archflag, _MACOS_COMPAT_FLAG) + environ['LDFLAGS'] += ' %s' % archflag + return create_jobspec( + self.name, + ['tools/run_tests/artifacts/build_artifact_csharp.sh'], + environ=environ, + use_workspace=True) + + def __str__(self): + return self.name + class PHPArtifact: - """Builds PHP PECL package""" - - def __init__(self, platform, arch): - self.name = 'php_pecl_package_{0}_{1}'.format(platform, arch) - self.platform = platform - self.arch = arch - self.labels = ['artifact', 'php', platform, arch] - - def pre_build_jobspecs(self): - return [] - - def build_jobspec(self): - if self.platform == 'linux': - return create_docker_jobspec( - self.name, - 'tools/dockerfile/grpc_artifact_linux_{}'.format(self.arch), - 'tools/run_tests/artifacts/build_artifact_php.sh') - else: - return create_jobspec(self.name, - ['tools/run_tests/artifacts/build_artifact_php.sh'], - use_workspace=True) + """Builds PHP PECL package""" -class ProtocArtifact: - """Builds protoc and protoc-plugin artifacts""" - - def __init__(self, platform, arch): - self.name = 'protoc_%s_%s' % (platform, arch) - self.platform = platform - self.arch = arch - self.labels = ['artifact', 'protoc', platform, arch] - - def pre_build_jobspecs(self): - return [] - - def build_jobspec(self): - if self.platform != 'windows': - cxxflags = '-DNDEBUG %s' % _ARCH_FLAG_MAP[self.arch] - ldflags = '%s' % _ARCH_FLAG_MAP[self.arch] - if self.platform != 'macos': - ldflags += ' -static-libgcc -static-libstdc++ -s' - environ={'CONFIG': 'opt', - 'CXXFLAGS': cxxflags, - 'LDFLAGS': ldflags, - 'PROTOBUF_LDFLAGS_EXTRA': ldflags} - if self.platform == 'linux': - return create_docker_jobspec(self.name, - 'tools/dockerfile/grpc_artifact_protoc', - 'tools/run_tests/artifacts/build_artifact_protoc.sh', - environ=environ) - else: - environ['CXXFLAGS'] += ' -std=c++11 -stdlib=libc++ %s' % _MACOS_COMPAT_FLAG - return create_jobspec(self.name, - ['tools/run_tests/artifacts/build_artifact_protoc.sh'], - environ=environ, - use_workspace=True) - else: - generator = 'Visual Studio 14 2015 Win64' if self.arch == 'x64' else 'Visual Studio 14 2015' - return create_jobspec(self.name, - ['tools\\run_tests\\artifacts\\build_artifact_protoc.bat'], - environ={'generator': generator}, - use_workspace=True) + def __init__(self, platform, arch): + self.name = 'php_pecl_package_{0}_{1}'.format(platform, arch) + self.platform = platform + self.arch = arch + self.labels = ['artifact', 'php', platform, arch] + + def pre_build_jobspecs(self): + return [] - def __str__(self): - return self.name + def build_jobspec(self): + if self.platform == 'linux': + return create_docker_jobspec( + self.name, + 'tools/dockerfile/grpc_artifact_linux_{}'.format(self.arch), + 'tools/run_tests/artifacts/build_artifact_php.sh') + else: + return create_jobspec( + self.name, ['tools/run_tests/artifacts/build_artifact_php.sh'], + use_workspace=True) + + +class ProtocArtifact: + """Builds protoc and protoc-plugin artifacts""" + + def __init__(self, platform, arch): + self.name = 'protoc_%s_%s' % (platform, arch) + self.platform = platform + self.arch = arch + self.labels = ['artifact', 'protoc', platform, arch] + + def pre_build_jobspecs(self): + return [] + + def build_jobspec(self): + if self.platform != 'windows': + cxxflags = '-DNDEBUG %s' % _ARCH_FLAG_MAP[self.arch] + ldflags = '%s' % _ARCH_FLAG_MAP[self.arch] + if self.platform != 'macos': + ldflags += ' -static-libgcc -static-libstdc++ -s' + environ = { + 'CONFIG': 'opt', + 'CXXFLAGS': cxxflags, + 'LDFLAGS': ldflags, + 'PROTOBUF_LDFLAGS_EXTRA': ldflags + } + if self.platform == 'linux': + return create_docker_jobspec( + self.name, + 'tools/dockerfile/grpc_artifact_protoc', + 'tools/run_tests/artifacts/build_artifact_protoc.sh', + environ=environ) + else: + environ[ + 'CXXFLAGS'] += ' -std=c++11 -stdlib=libc++ %s' % _MACOS_COMPAT_FLAG + return create_jobspec( + self.name, + ['tools/run_tests/artifacts/build_artifact_protoc.sh'], + environ=environ, + use_workspace=True) + else: + generator = 'Visual Studio 14 2015 Win64' if self.arch == 'x64' else 'Visual Studio 14 2015' + return create_jobspec( + self.name, + ['tools\\run_tests\\artifacts\\build_artifact_protoc.bat'], + environ={'generator': generator}, + use_workspace=True) + + def __str__(self): + return self.name def targets(): - """Gets list of supported targets""" - return ([Cls(platform, arch) - for Cls in (CSharpExtArtifact, ProtocArtifact) - for platform in ('linux', 'macos', 'windows') - for arch in ('x86', 'x64')] + - [PythonArtifact('linux', 'x86', 'cp27-cp27m'), - PythonArtifact('linux', 'x86', 'cp27-cp27mu'), - PythonArtifact('linux', 'x86', 'cp34-cp34m'), - PythonArtifact('linux', 'x86', 'cp35-cp35m'), - PythonArtifact('linux', 'x86', 'cp36-cp36m'), - PythonArtifact('linux_extra', 'armv7', '2.7'), - PythonArtifact('linux_extra', 'armv7', '3.4'), - PythonArtifact('linux_extra', 'armv7', '3.5'), - PythonArtifact('linux_extra', 'armv7', '3.6'), - PythonArtifact('linux_extra', 'armv6', '2.7'), - PythonArtifact('linux_extra', 'armv6', '3.4'), - PythonArtifact('linux_extra', 'armv6', '3.5'), - PythonArtifact('linux_extra', 'armv6', '3.6'), - PythonArtifact('linux', 'x64', 'cp27-cp27m'), - PythonArtifact('linux', 'x64', 'cp27-cp27mu'), - PythonArtifact('linux', 'x64', 'cp34-cp34m'), - PythonArtifact('linux', 'x64', 'cp35-cp35m'), - PythonArtifact('linux', 'x64', 'cp36-cp36m'), - PythonArtifact('macos', 'x64', 'python2.7'), - PythonArtifact('macos', 'x64', 'python3.4'), - PythonArtifact('macos', 'x64', 'python3.5'), - PythonArtifact('macos', 'x64', 'python3.6'), - PythonArtifact('windows', 'x86', 'Python27_32bits'), - PythonArtifact('windows', 'x86', 'Python34_32bits'), - PythonArtifact('windows', 'x86', 'Python35_32bits'), - PythonArtifact('windows', 'x86', 'Python36_32bits'), - PythonArtifact('windows', 'x64', 'Python27'), - PythonArtifact('windows', 'x64', 'Python34'), - PythonArtifact('windows', 'x64', 'Python35'), - PythonArtifact('windows', 'x64', 'Python36'), - RubyArtifact('linux', 'x64'), - RubyArtifact('macos', 'x64'), - PHPArtifact('linux', 'x64'), - PHPArtifact('macos', 'x64')]) + """Gets list of supported targets""" + return ([ + Cls(platform, arch) + for Cls in (CSharpExtArtifact, ProtocArtifact) + for platform in ('linux', 'macos', 'windows') for arch in ('x86', 'x64') + ] + [ + PythonArtifact('linux', 'x86', 'cp27-cp27m'), PythonArtifact( + 'linux', 'x86', 'cp27-cp27mu'), PythonArtifact( + 'linux', 'x86', 'cp34-cp34m'), PythonArtifact( + 'linux', 'x86', 'cp35-cp35m'), PythonArtifact( + 'linux', 'x86', 'cp36-cp36m'), PythonArtifact( + 'linux_extra', 'armv7', '2.7'), PythonArtifact( + 'linux_extra', 'armv7', '3.4'), PythonArtifact( + 'linux_extra', 'armv7', '3.5'), + PythonArtifact('linux_extra', 'armv7', '3.6'), PythonArtifact( + 'linux_extra', 'armv6', '2.7'), PythonArtifact( + 'linux_extra', 'armv6', '3.4'), PythonArtifact( + 'linux_extra', 'armv6', '3.5'), PythonArtifact( + 'linux_extra', 'armv6', '3.6'), PythonArtifact( + 'linux', 'x64', 'cp27-cp27m'), PythonArtifact( + 'linux', 'x64', 'cp27-cp27mu'), PythonArtifact( + 'linux', 'x64', 'cp34-cp34m'), + PythonArtifact('linux', 'x64', 'cp35-cp35m'), PythonArtifact( + 'linux', 'x64', 'cp36-cp36m'), PythonArtifact( + 'macos', 'x64', 'python2.7'), PythonArtifact( + 'macos', 'x64', 'python3.4'), PythonArtifact('macos', 'x64', + 'python3.5'), + PythonArtifact('macos', 'x64', 'python3.6'), PythonArtifact( + 'windows', 'x86', 'Python27_32bits'), PythonArtifact( + 'windows', 'x86', 'Python34_32bits'), PythonArtifact( + 'windows', 'x86', 'Python35_32bits'), PythonArtifact( + 'windows', 'x86', 'Python36_32bits'), PythonArtifact( + 'windows', 'x64', 'Python27'), + PythonArtifact('windows', 'x64', 'Python34'), PythonArtifact( + 'windows', 'x64', 'Python35'), PythonArtifact( + 'windows', 'x64', 'Python36'), RubyArtifact( + 'linux', 'x64'), RubyArtifact('macos', 'x64'), PHPArtifact( + 'linux', 'x64'), PHPArtifact('macos', 'x64') + ]) diff --git a/tools/run_tests/artifacts/distribtest_targets.py b/tools/run_tests/artifacts/distribtest_targets.py index 7ba0e0ebc9..83f656b433 100644 --- a/tools/run_tests/artifacts/distribtest_targets.py +++ b/tools/run_tests/artifacts/distribtest_targets.py @@ -12,7 +12,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - """Definition of targets run distribution package tests.""" import os.path @@ -22,280 +21,306 @@ sys.path.insert(0, os.path.abspath('..')) import python_utils.jobset as jobset -def create_docker_jobspec(name, dockerfile_dir, shell_command, environ={}, - flake_retries=0, timeout_retries=0, - copy_rel_path=None): - """Creates jobspec for a task running under docker.""" - environ = environ.copy() - environ['RUN_COMMAND'] = shell_command - # the entire repo will be cloned if copy_rel_path is not set. - if copy_rel_path: - environ['RELATIVE_COPY_PATH'] = copy_rel_path - - docker_args=[] - for k,v in environ.items(): - docker_args += ['-e', '%s=%s' % (k, v)] - docker_env = {'DOCKERFILE_DIR': dockerfile_dir, - 'DOCKER_RUN_SCRIPT': 'tools/run_tests/dockerize/docker_run.sh'} - jobspec = jobset.JobSpec( - cmdline=['tools/run_tests/dockerize/build_and_run_docker.sh'] + docker_args, - environ=docker_env, - shortname='distribtest.%s' % (name), - timeout_seconds=30*60, - flake_retries=flake_retries, - timeout_retries=timeout_retries) - return jobspec - - -def create_jobspec(name, cmdline, environ=None, shell=False, - flake_retries=0, timeout_retries=0, +def create_docker_jobspec(name, + dockerfile_dir, + shell_command, + environ={}, + flake_retries=0, + timeout_retries=0, + copy_rel_path=None): + """Creates jobspec for a task running under docker.""" + environ = environ.copy() + environ['RUN_COMMAND'] = shell_command + # the entire repo will be cloned if copy_rel_path is not set. + if copy_rel_path: + environ['RELATIVE_COPY_PATH'] = copy_rel_path + + docker_args = [] + for k, v in environ.items(): + docker_args += ['-e', '%s=%s' % (k, v)] + docker_env = { + 'DOCKERFILE_DIR': dockerfile_dir, + 'DOCKER_RUN_SCRIPT': 'tools/run_tests/dockerize/docker_run.sh' + } + jobspec = jobset.JobSpec( + cmdline=['tools/run_tests/dockerize/build_and_run_docker.sh'] + + docker_args, + environ=docker_env, + shortname='distribtest.%s' % (name), + timeout_seconds=30 * 60, + flake_retries=flake_retries, + timeout_retries=timeout_retries) + return jobspec + + +def create_jobspec(name, + cmdline, + environ=None, + shell=False, + flake_retries=0, + timeout_retries=0, use_workspace=False, - timeout_seconds=10*60): - """Creates jobspec.""" - environ = environ.copy() - if use_workspace: - environ['WORKSPACE_NAME'] = 'workspace_%s' % name - cmdline = ['bash', - 'tools/run_tests/artifacts/run_in_workspace.sh'] + cmdline - jobspec = jobset.JobSpec( - cmdline=cmdline, - environ=environ, - shortname='distribtest.%s' % (name), - timeout_seconds=timeout_seconds, - flake_retries=flake_retries, - timeout_retries=timeout_retries, - shell=shell) - return jobspec + timeout_seconds=10 * 60): + """Creates jobspec.""" + environ = environ.copy() + if use_workspace: + environ['WORKSPACE_NAME'] = 'workspace_%s' % name + cmdline = ['bash', 'tools/run_tests/artifacts/run_in_workspace.sh' + ] + cmdline + jobspec = jobset.JobSpec( + cmdline=cmdline, + environ=environ, + shortname='distribtest.%s' % (name), + timeout_seconds=timeout_seconds, + flake_retries=flake_retries, + timeout_retries=timeout_retries, + shell=shell) + return jobspec class CSharpDistribTest(object): - """Tests C# NuGet package""" - - def __init__(self, platform, arch, docker_suffix=None, use_dotnet_cli=False): - self.name = 'csharp_%s_%s' % (platform, arch) - self.platform = platform - self.arch = arch - self.docker_suffix = docker_suffix - self.labels = ['distribtest', 'csharp', platform, arch] - self.script_suffix = '' - if docker_suffix: - self.name += '_%s' % docker_suffix - self.labels.append(docker_suffix) - if use_dotnet_cli: - self.name += '_dotnetcli' - self.script_suffix = '_dotnetcli' - self.labels.append('dotnetcli') - else: - self.labels.append('olddotnet') - - def pre_build_jobspecs(self): - return [] - - def build_jobspec(self): - if self.platform == 'linux': - return create_docker_jobspec(self.name, - 'tools/dockerfile/distribtest/csharp_%s_%s' % ( - self.docker_suffix, - self.arch), - 'test/distrib/csharp/run_distrib_test%s.sh' % self.script_suffix, - copy_rel_path='test/distrib') - elif self.platform == 'macos': - return create_jobspec(self.name, - ['test/distrib/csharp/run_distrib_test%s.sh' % self.script_suffix], - environ={'EXTERNAL_GIT_ROOT': '../../../..'}, - use_workspace=True) - elif self.platform == 'windows': - if self.arch == 'x64': - # Use double leading / as the first occurence gets removed by msys bash - # when invoking the .bat file (side-effect of posix path conversion) - environ={'MSBUILD_EXTRA_ARGS': '//p:Platform=x64', - 'DISTRIBTEST_OUTPATH': 'DistribTest\\bin\\x64\\Debug'} - else: - environ={'DISTRIBTEST_OUTPATH': 'DistribTest\\bin\\Debug'} - return create_jobspec(self.name, - ['test\\distrib\\csharp\\run_distrib_test%s.bat' % self.script_suffix], - environ=environ, - use_workspace=True) - else: - raise Exception("Not supported yet.") - - def __str__(self): - return self.name + """Tests C# NuGet package""" + + def __init__(self, platform, arch, docker_suffix=None, + use_dotnet_cli=False): + self.name = 'csharp_%s_%s' % (platform, arch) + self.platform = platform + self.arch = arch + self.docker_suffix = docker_suffix + self.labels = ['distribtest', 'csharp', platform, arch] + self.script_suffix = '' + if docker_suffix: + self.name += '_%s' % docker_suffix + self.labels.append(docker_suffix) + if use_dotnet_cli: + self.name += '_dotnetcli' + self.script_suffix = '_dotnetcli' + self.labels.append('dotnetcli') + else: + self.labels.append('olddotnet') + + def pre_build_jobspecs(self): + return [] + + def build_jobspec(self): + if self.platform == 'linux': + return create_docker_jobspec( + self.name, + 'tools/dockerfile/distribtest/csharp_%s_%s' % ( + self.docker_suffix, self.arch), + 'test/distrib/csharp/run_distrib_test%s.sh' % + self.script_suffix, + copy_rel_path='test/distrib') + elif self.platform == 'macos': + return create_jobspec( + self.name, [ + 'test/distrib/csharp/run_distrib_test%s.sh' % + self.script_suffix + ], + environ={'EXTERNAL_GIT_ROOT': '../../../..'}, + use_workspace=True) + elif self.platform == 'windows': + if self.arch == 'x64': + # Use double leading / as the first occurence gets removed by msys bash + # when invoking the .bat file (side-effect of posix path conversion) + environ = { + 'MSBUILD_EXTRA_ARGS': '//p:Platform=x64', + 'DISTRIBTEST_OUTPATH': 'DistribTest\\bin\\x64\\Debug' + } + else: + environ = {'DISTRIBTEST_OUTPATH': 'DistribTest\\bin\\Debug'} + return create_jobspec( + self.name, [ + 'test\\distrib\\csharp\\run_distrib_test%s.bat' % + self.script_suffix + ], + environ=environ, + use_workspace=True) + else: + raise Exception("Not supported yet.") + + def __str__(self): + return self.name + class PythonDistribTest(object): - """Tests Python package""" + """Tests Python package""" - def __init__(self, platform, arch, docker_suffix): - self.name = 'python_%s_%s_%s' % (platform, arch, docker_suffix) - self.platform = platform - self.arch = arch - self.docker_suffix = docker_suffix - self.labels = ['distribtest', 'python', platform, arch, docker_suffix] + def __init__(self, platform, arch, docker_suffix): + self.name = 'python_%s_%s_%s' % (platform, arch, docker_suffix) + self.platform = platform + self.arch = arch + self.docker_suffix = docker_suffix + self.labels = ['distribtest', 'python', platform, arch, docker_suffix] - def pre_build_jobspecs(self): - return [] + def pre_build_jobspecs(self): + return [] - def build_jobspec(self): - if not self.platform == 'linux': - raise Exception("Not supported yet.") + def build_jobspec(self): + if not self.platform == 'linux': + raise Exception("Not supported yet.") - return create_docker_jobspec(self.name, - 'tools/dockerfile/distribtest/python_%s_%s' % ( - self.docker_suffix, - self.arch), - 'test/distrib/python/run_distrib_test.sh', - copy_rel_path='test/distrib') + return create_docker_jobspec( + self.name, + 'tools/dockerfile/distribtest/python_%s_%s' % (self.docker_suffix, + self.arch), + 'test/distrib/python/run_distrib_test.sh', + copy_rel_path='test/distrib') - def __str__(self): - return self.name + def __str__(self): + return self.name class RubyDistribTest(object): - """Tests Ruby package""" + """Tests Ruby package""" - def __init__(self, platform, arch, docker_suffix): - self.name = 'ruby_%s_%s_%s' % (platform, arch, docker_suffix) - self.platform = platform - self.arch = arch - self.docker_suffix = docker_suffix - self.labels = ['distribtest', 'ruby', platform, arch, docker_suffix] + def __init__(self, platform, arch, docker_suffix): + self.name = 'ruby_%s_%s_%s' % (platform, arch, docker_suffix) + self.platform = platform + self.arch = arch + self.docker_suffix = docker_suffix + self.labels = ['distribtest', 'ruby', platform, arch, docker_suffix] - def pre_build_jobspecs(self): - return [] + def pre_build_jobspecs(self): + return [] - def build_jobspec(self): - if not self.platform == 'linux': - raise Exception("Not supported yet.") + def build_jobspec(self): + if not self.platform == 'linux': + raise Exception("Not supported yet.") - return create_docker_jobspec(self.name, - 'tools/dockerfile/distribtest/ruby_%s_%s' % ( - self.docker_suffix, - self.arch), - 'test/distrib/ruby/run_distrib_test.sh', - copy_rel_path='test/distrib') + return create_docker_jobspec( + self.name, + 'tools/dockerfile/distribtest/ruby_%s_%s' % (self.docker_suffix, + self.arch), + 'test/distrib/ruby/run_distrib_test.sh', + copy_rel_path='test/distrib') - def __str__(self): - return self.name + def __str__(self): + return self.name class PHPDistribTest(object): - """Tests PHP package""" - - def __init__(self, platform, arch, docker_suffix=None): - self.name = 'php_%s_%s_%s' % (platform, arch, docker_suffix) - self.platform = platform - self.arch = arch - self.docker_suffix = docker_suffix - self.labels = ['distribtest', 'php', platform, arch, docker_suffix] - - def pre_build_jobspecs(self): - return [] - - def build_jobspec(self): - if self.platform == 'linux': - return create_docker_jobspec(self.name, - 'tools/dockerfile/distribtest/php_%s_%s' % ( - self.docker_suffix, - self.arch), - 'test/distrib/php/run_distrib_test.sh', - copy_rel_path='test/distrib') - elif self.platform == 'macos': - return create_jobspec(self.name, - ['test/distrib/php/run_distrib_test.sh'], - environ={'EXTERNAL_GIT_ROOT': '../../../..'}, - use_workspace=True) - else: - raise Exception("Not supported yet.") - - def __str__(self): - return self.name + """Tests PHP package""" + + def __init__(self, platform, arch, docker_suffix=None): + self.name = 'php_%s_%s_%s' % (platform, arch, docker_suffix) + self.platform = platform + self.arch = arch + self.docker_suffix = docker_suffix + self.labels = ['distribtest', 'php', platform, arch, docker_suffix] + + def pre_build_jobspecs(self): + return [] + + def build_jobspec(self): + if self.platform == 'linux': + return create_docker_jobspec( + self.name, + 'tools/dockerfile/distribtest/php_%s_%s' % (self.docker_suffix, + self.arch), + 'test/distrib/php/run_distrib_test.sh', + copy_rel_path='test/distrib') + elif self.platform == 'macos': + return create_jobspec( + self.name, ['test/distrib/php/run_distrib_test.sh'], + environ={'EXTERNAL_GIT_ROOT': '../../../..'}, + use_workspace=True) + else: + raise Exception("Not supported yet.") + + def __str__(self): + return self.name class CppDistribTest(object): - """Tests Cpp make intall by building examples.""" - - def __init__(self, platform, arch, docker_suffix=None, testcase=None): - if platform == 'linux': - self.name = 'cpp_%s_%s_%s_%s' % (platform, arch, docker_suffix, testcase) - else: - self.name = 'cpp_%s_%s_%s' % (platform, arch, testcase) - self.platform = platform - self.arch = arch - self.docker_suffix = docker_suffix - self.testcase = testcase - self.labels = ['distribtest', 'cpp', platform, arch, docker_suffix, testcase] - - def pre_build_jobspecs(self): - return [] - - def build_jobspec(self): - if self.platform == 'linux': - return create_docker_jobspec(self.name, - 'tools/dockerfile/distribtest/cpp_%s_%s' % ( - self.docker_suffix, - self.arch), - 'test/distrib/cpp/run_distrib_test_%s.sh' % self.testcase) - elif self.platform == 'windows': - return create_jobspec(self.name, - ['test\\distrib\\cpp\\run_distrib_test_%s.bat' % self.testcase], - environ={}, - timeout_seconds=30*60, - use_workspace=True) - else: - raise Exception("Not supported yet.") - - def __str__(self): - return self.name + """Tests Cpp make intall by building examples.""" + + def __init__(self, platform, arch, docker_suffix=None, testcase=None): + if platform == 'linux': + self.name = 'cpp_%s_%s_%s_%s' % (platform, arch, docker_suffix, + testcase) + else: + self.name = 'cpp_%s_%s_%s' % (platform, arch, testcase) + self.platform = platform + self.arch = arch + self.docker_suffix = docker_suffix + self.testcase = testcase + self.labels = [ + 'distribtest', 'cpp', platform, arch, docker_suffix, testcase + ] + + def pre_build_jobspecs(self): + return [] + + def build_jobspec(self): + if self.platform == 'linux': + return create_docker_jobspec( + self.name, 'tools/dockerfile/distribtest/cpp_%s_%s' % ( + self.docker_suffix, self.arch), + 'test/distrib/cpp/run_distrib_test_%s.sh' % self.testcase) + elif self.platform == 'windows': + return create_jobspec( + self.name, + ['test\\distrib\\cpp\\run_distrib_test_%s.bat' % self.testcase], + environ={}, + timeout_seconds=30 * 60, + use_workspace=True) + else: + raise Exception("Not supported yet.") + + def __str__(self): + return self.name def targets(): - """Gets list of supported targets""" - return [CppDistribTest('linux', 'x64', 'jessie', 'routeguide'), - CppDistribTest('linux', 'x64', 'jessie', 'cmake'), - CppDistribTest('windows', 'x86', testcase='cmake'), - CSharpDistribTest('linux', 'x64', 'wheezy'), - CSharpDistribTest('linux', 'x64', 'jessie'), - CSharpDistribTest('linux', 'x86', 'jessie'), - CSharpDistribTest('linux', 'x64', 'centos7'), - CSharpDistribTest('linux', 'x64', 'ubuntu1404'), - CSharpDistribTest('linux', 'x64', 'ubuntu1504'), - CSharpDistribTest('linux', 'x64', 'ubuntu1510'), - CSharpDistribTest('linux', 'x64', 'ubuntu1604'), - CSharpDistribTest('linux', 'x64', 'ubuntu1404', use_dotnet_cli=True), - CSharpDistribTest('macos', 'x86'), - CSharpDistribTest('windows', 'x86'), - CSharpDistribTest('windows', 'x64'), - PythonDistribTest('linux', 'x64', 'wheezy'), - PythonDistribTest('linux', 'x64', 'jessie'), - PythonDistribTest('linux', 'x86', 'jessie'), - PythonDistribTest('linux', 'x64', 'centos6'), - PythonDistribTest('linux', 'x64', 'centos7'), - PythonDistribTest('linux', 'x64', 'fedora20'), - PythonDistribTest('linux', 'x64', 'fedora21'), - PythonDistribTest('linux', 'x64', 'fedora22'), - PythonDistribTest('linux', 'x64', 'fedora23'), - PythonDistribTest('linux', 'x64', 'opensuse'), - PythonDistribTest('linux', 'x64', 'arch'), - PythonDistribTest('linux', 'x64', 'ubuntu1204'), - PythonDistribTest('linux', 'x64', 'ubuntu1404'), - PythonDistribTest('linux', 'x64', 'ubuntu1504'), - PythonDistribTest('linux', 'x64', 'ubuntu1510'), - PythonDistribTest('linux', 'x64', 'ubuntu1604'), - RubyDistribTest('linux', 'x64', 'wheezy'), - RubyDistribTest('linux', 'x64', 'jessie'), - RubyDistribTest('linux', 'x86', 'jessie'), - RubyDistribTest('linux', 'x64', 'centos6'), - RubyDistribTest('linux', 'x64', 'centos7'), - RubyDistribTest('linux', 'x64', 'fedora20'), - RubyDistribTest('linux', 'x64', 'fedora21'), - RubyDistribTest('linux', 'x64', 'fedora22'), - RubyDistribTest('linux', 'x64', 'fedora23'), - RubyDistribTest('linux', 'x64', 'opensuse'), - RubyDistribTest('linux', 'x64', 'ubuntu1204'), - RubyDistribTest('linux', 'x64', 'ubuntu1404'), - RubyDistribTest('linux', 'x64', 'ubuntu1504'), - RubyDistribTest('linux', 'x64', 'ubuntu1510'), - RubyDistribTest('linux', 'x64', 'ubuntu1604'), - PHPDistribTest('linux', 'x64', 'jessie'), - PHPDistribTest('macos', 'x64'), - ] + """Gets list of supported targets""" + return [ + CppDistribTest('linux', 'x64', 'jessie', 'routeguide'), + CppDistribTest('linux', 'x64', 'jessie', 'cmake'), + CppDistribTest('windows', 'x86', testcase='cmake'), + CSharpDistribTest('linux', 'x64', 'wheezy'), + CSharpDistribTest('linux', 'x64', 'jessie'), + CSharpDistribTest('linux', 'x86', 'jessie'), + CSharpDistribTest('linux', 'x64', 'centos7'), + CSharpDistribTest('linux', 'x64', 'ubuntu1404'), + CSharpDistribTest('linux', 'x64', 'ubuntu1504'), + CSharpDistribTest('linux', 'x64', 'ubuntu1510'), + CSharpDistribTest('linux', 'x64', 'ubuntu1604'), + CSharpDistribTest('linux', 'x64', 'ubuntu1404', use_dotnet_cli=True), + CSharpDistribTest('macos', 'x86'), + CSharpDistribTest('windows', 'x86'), + CSharpDistribTest('windows', 'x64'), + PythonDistribTest('linux', 'x64', 'wheezy'), + PythonDistribTest('linux', 'x64', 'jessie'), + PythonDistribTest('linux', 'x86', 'jessie'), + PythonDistribTest('linux', 'x64', 'centos6'), + PythonDistribTest('linux', 'x64', 'centos7'), + PythonDistribTest('linux', 'x64', 'fedora20'), + PythonDistribTest('linux', 'x64', 'fedora21'), + PythonDistribTest('linux', 'x64', 'fedora22'), + PythonDistribTest('linux', 'x64', 'fedora23'), + PythonDistribTest('linux', 'x64', 'opensuse'), + PythonDistribTest('linux', 'x64', 'arch'), + PythonDistribTest('linux', 'x64', 'ubuntu1204'), + PythonDistribTest('linux', 'x64', 'ubuntu1404'), + PythonDistribTest('linux', 'x64', 'ubuntu1504'), + PythonDistribTest('linux', 'x64', 'ubuntu1510'), + PythonDistribTest('linux', 'x64', 'ubuntu1604'), + RubyDistribTest('linux', 'x64', 'wheezy'), + RubyDistribTest('linux', 'x64', 'jessie'), + RubyDistribTest('linux', 'x86', 'jessie'), + RubyDistribTest('linux', 'x64', 'centos6'), + RubyDistribTest('linux', 'x64', 'centos7'), + RubyDistribTest('linux', 'x64', 'fedora20'), + RubyDistribTest('linux', 'x64', 'fedora21'), + RubyDistribTest('linux', 'x64', 'fedora22'), + RubyDistribTest('linux', 'x64', 'fedora23'), + RubyDistribTest('linux', 'x64', 'opensuse'), + RubyDistribTest('linux', 'x64', 'ubuntu1204'), + RubyDistribTest('linux', 'x64', 'ubuntu1404'), + RubyDistribTest('linux', 'x64', 'ubuntu1504'), + RubyDistribTest('linux', 'x64', 'ubuntu1510'), + RubyDistribTest('linux', 'x64', 'ubuntu1604'), + PHPDistribTest('linux', 'x64', 'jessie'), + PHPDistribTest('macos', 'x64'), + ] diff --git a/tools/run_tests/artifacts/package_targets.py b/tools/run_tests/artifacts/package_targets.py index ff93bb30e8..52908454f2 100644 --- a/tools/run_tests/artifacts/package_targets.py +++ b/tools/run_tests/artifacts/package_targets.py @@ -12,7 +12,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - """Definition of targets to build distribution packages.""" import os.path @@ -22,128 +21,137 @@ sys.path.insert(0, os.path.abspath('..')) import python_utils.jobset as jobset -def create_docker_jobspec(name, dockerfile_dir, shell_command, environ={}, - flake_retries=0, timeout_retries=0): - """Creates jobspec for a task running under docker.""" - environ = environ.copy() - environ['RUN_COMMAND'] = shell_command - - docker_args=[] - for k,v in environ.items(): - docker_args += ['-e', '%s=%s' % (k, v)] - docker_env = {'DOCKERFILE_DIR': dockerfile_dir, - 'DOCKER_RUN_SCRIPT': 'tools/run_tests/dockerize/docker_run.sh', - 'OUTPUT_DIR': 'artifacts'} - jobspec = jobset.JobSpec( - cmdline=['tools/run_tests/dockerize/build_and_run_docker.sh'] + docker_args, - environ=docker_env, - shortname='build_package.%s' % (name), - timeout_seconds=30*60, - flake_retries=flake_retries, - timeout_retries=timeout_retries) - return jobspec - -def create_jobspec(name, cmdline, environ=None, cwd=None, shell=False, - flake_retries=0, timeout_retries=0): - """Creates jobspec.""" - jobspec = jobset.JobSpec( - cmdline=cmdline, - environ=environ, - cwd=cwd, - shortname='build_package.%s' % (name), - timeout_seconds=10*60, - flake_retries=flake_retries, - timeout_retries=timeout_retries, - shell=shell) - return jobspec +def create_docker_jobspec(name, + dockerfile_dir, + shell_command, + environ={}, + flake_retries=0, + timeout_retries=0): + """Creates jobspec for a task running under docker.""" + environ = environ.copy() + environ['RUN_COMMAND'] = shell_command + + docker_args = [] + for k, v in environ.items(): + docker_args += ['-e', '%s=%s' % (k, v)] + docker_env = { + 'DOCKERFILE_DIR': dockerfile_dir, + 'DOCKER_RUN_SCRIPT': 'tools/run_tests/dockerize/docker_run.sh', + 'OUTPUT_DIR': 'artifacts' + } + jobspec = jobset.JobSpec( + cmdline=['tools/run_tests/dockerize/build_and_run_docker.sh'] + + docker_args, + environ=docker_env, + shortname='build_package.%s' % (name), + timeout_seconds=30 * 60, + flake_retries=flake_retries, + timeout_retries=timeout_retries) + return jobspec + + +def create_jobspec(name, + cmdline, + environ=None, + cwd=None, + shell=False, + flake_retries=0, + timeout_retries=0): + """Creates jobspec.""" + jobspec = jobset.JobSpec( + cmdline=cmdline, + environ=environ, + cwd=cwd, + shortname='build_package.%s' % (name), + timeout_seconds=10 * 60, + flake_retries=flake_retries, + timeout_retries=timeout_retries, + shell=shell) + return jobspec class CSharpPackage: - """Builds C# nuget packages.""" - - def __init__(self, linux=False): - self.linux = linux - self.labels = ['package', 'csharp'] - if linux: - self.name = 'csharp_package_dotnetcli_linux' - self.labels += ['linux'] - else: - self.name = 'csharp_package_dotnetcli_windows' - self.labels += ['windows'] - - def pre_build_jobspecs(self): - return [] - - def build_jobspec(self): - if self.linux: - return create_docker_jobspec( - self.name, - 'tools/dockerfile/test/csharp_jessie_x64', - 'src/csharp/build_packages_dotnetcli.sh') - else: - return create_jobspec(self.name, - ['build_packages_dotnetcli.bat'], - cwd='src\\csharp', - shell=True) - - def __str__(self): - return self.name + """Builds C# nuget packages.""" + + def __init__(self, linux=False): + self.linux = linux + self.labels = ['package', 'csharp'] + if linux: + self.name = 'csharp_package_dotnetcli_linux' + self.labels += ['linux'] + else: + self.name = 'csharp_package_dotnetcli_windows' + self.labels += ['windows'] + + def pre_build_jobspecs(self): + return [] + + def build_jobspec(self): + if self.linux: + return create_docker_jobspec( + self.name, 'tools/dockerfile/test/csharp_jessie_x64', + 'src/csharp/build_packages_dotnetcli.sh') + else: + return create_jobspec( + self.name, ['build_packages_dotnetcli.bat'], + cwd='src\\csharp', + shell=True) + + def __str__(self): + return self.name + class RubyPackage: - """Collects ruby gems created in the artifact phase""" + """Collects ruby gems created in the artifact phase""" - def __init__(self): - self.name = 'ruby_package' - self.labels = ['package', 'ruby', 'linux'] + def __init__(self): + self.name = 'ruby_package' + self.labels = ['package', 'ruby', 'linux'] - def pre_build_jobspecs(self): - return [] + def pre_build_jobspecs(self): + return [] - def build_jobspec(self): - return create_docker_jobspec( - self.name, - 'tools/dockerfile/grpc_artifact_linux_x64', - 'tools/run_tests/artifacts/build_package_ruby.sh') + def build_jobspec(self): + return create_docker_jobspec( + self.name, 'tools/dockerfile/grpc_artifact_linux_x64', + 'tools/run_tests/artifacts/build_package_ruby.sh') class PythonPackage: - """Collects python eggs and wheels created in the artifact phase""" + """Collects python eggs and wheels created in the artifact phase""" - def __init__(self): - self.name = 'python_package' - self.labels = ['package', 'python', 'linux'] + def __init__(self): + self.name = 'python_package' + self.labels = ['package', 'python', 'linux'] - def pre_build_jobspecs(self): - return [] + def pre_build_jobspecs(self): + return [] - def build_jobspec(self): - return create_docker_jobspec( - self.name, - 'tools/dockerfile/grpc_artifact_linux_x64', - 'tools/run_tests/artifacts/build_package_python.sh') + def build_jobspec(self): + return create_docker_jobspec( + self.name, 'tools/dockerfile/grpc_artifact_linux_x64', + 'tools/run_tests/artifacts/build_package_python.sh') class PHPPackage: - """Copy PHP PECL package artifact""" + """Copy PHP PECL package artifact""" - def __init__(self): - self.name = 'php_package' - self.labels = ['package', 'php', 'linux'] + def __init__(self): + self.name = 'php_package' + self.labels = ['package', 'php', 'linux'] - def pre_build_jobspecs(self): - return [] + def pre_build_jobspecs(self): + return [] - def build_jobspec(self): - return create_docker_jobspec( - self.name, - 'tools/dockerfile/grpc_artifact_linux_x64', - 'tools/run_tests/artifacts/build_package_php.sh') + def build_jobspec(self): + return create_docker_jobspec( + self.name, 'tools/dockerfile/grpc_artifact_linux_x64', + 'tools/run_tests/artifacts/build_package_php.sh') def targets(): - """Gets list of supported targets""" - return [CSharpPackage(), - CSharpPackage(linux=True), - RubyPackage(), - PythonPackage(), - PHPPackage()] + """Gets list of supported targets""" + return [ + CSharpPackage(), CSharpPackage(linux=True), RubyPackage(), + PythonPackage(), PHPPackage() + ] |