Merge branch 'slice_with_exec_ctx' into metadata_filter

1 files changed, 356 insertions, 0 deletions

diff --git a/tools/run_tests/artifacts/artifact_targets.py b/tools/run_tests/artifacts/artifact_targets.py
new file mode 100644
index 0000000000..005d99790a
--- /dev/null
+++ b/tools/run_tests/artifacts/artifact_targets.py
@@ -0,0 +1,356 @@
+#!/usr/bin/env python2.7
+# Copyright 2016, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Definition of targets to build artifacts."""
+
+import os.path
+import random
+import string
+import sys
+
+sys.path.insert(0, os.path.abspath('..'))
+import python_utils.jobset as jobset
+
+
+def create_docker_jobspec(name, dockerfile_dir, shell_command, environ={},
+                   flake_retries=0, timeout_retries=0, timeout_seconds=30*60):
+  """Creates jobspec for a task running under docker."""
+  environ = environ.copy()
+  environ['RUN_COMMAND'] = shell_command
+
+  docker_args=[]
+  for k,v in environ.items():
+    docker_args += ['-e', '%s=%s' % (k, v)]
+  docker_env = {'DOCKERFILE_DIR': dockerfile_dir,
+                'DOCKER_RUN_SCRIPT': 'tools/run_tests/dockerize/docker_run.sh',
+                'OUTPUT_DIR': 'artifacts'}
+  jobspec = jobset.JobSpec(
+          cmdline=['tools/run_tests/dockerize/build_and_run_docker.sh'] + docker_args,
+          environ=docker_env,
+          shortname='build_artifact.%s' % (name),
+          timeout_seconds=timeout_seconds,
+          flake_retries=flake_retries,
+          timeout_retries=timeout_retries)
+  return jobspec
+
+
+def create_jobspec(name, cmdline, environ=None, shell=False,
+                   flake_retries=0, timeout_retries=0, timeout_seconds=30*60):
+  """Creates jobspec."""
+  jobspec = jobset.JobSpec(
+          cmdline=cmdline,
+          environ=environ,
+          shortname='build_artifact.%s' % (name),
+          timeout_seconds=timeout_seconds,
+          flake_retries=flake_retries,
+          timeout_retries=timeout_retries,
+          shell=shell)
+  return jobspec
+
+
+_MACOS_COMPAT_FLAG = '-mmacosx-version-min=10.7'
+
+_ARCH_FLAG_MAP = {
+  'x86': '-m32',
+  'x64': '-m64'
+}
+
+
+class PythonArtifact:
+  """Builds Python artifacts."""
+
+  def __init__(self, platform, arch, py_version):
+    self.name = 'python_%s_%s_%s' % (platform, arch, py_version)
+    self.platform = platform
+    self.arch = arch
+    self.labels = ['artifact', 'python', platform, arch, py_version]
+    self.py_version = py_version
+
+  def pre_build_jobspecs(self):
+    return []
+
+  def build_jobspec(self):
+    environ = {}
+    if self.platform == 'linux':
+      if self.arch == 'x86':
+        environ['SETARCH_CMD'] = 'linux32'
+      # Inside the manylinux container, the python installations are located in
+      # special places...
+      environ['PYTHON'] = '/opt/python/{}/bin/python'.format(self.py_version)
+      environ['PIP'] = '/opt/python/{}/bin/pip'.format(self.py_version)
+      # Platform autodetection for the manylinux1 image breaks so we set the
+      # defines ourselves.
+      # TODO(atash) get better platform-detection support in core so we don't
+      # need to do this manually...
+      environ['CFLAGS'] = '-DGPR_MANYLINUX1=1'
+      environ['GRPC_BUILD_GRPCIO_TOOLS_DEPENDENTS'] = 'TRUE'
+      environ['GRPC_BUILD_MANYLINUX_WHEEL'] = 'TRUE'
+      return create_docker_jobspec(self.name,
+          'tools/dockerfile/grpc_artifact_python_manylinux_%s' % self.arch,
+          'tools/run_tests/artifacts/build_artifact_python.sh',
+          environ=environ,
+          timeout_seconds=60*60)
+    elif self.platform == 'windows':
+      if 'Python27' in self.py_version or 'Python34' in self.py_version:
+        environ['EXT_COMPILER'] = 'mingw32'
+      else:
+        environ['EXT_COMPILER'] = 'msvc'
+      # For some reason, the batch script %random% always runs with the same
+      # seed.  We create a random temp-dir here
+      dir = ''.join(random.choice(string.ascii_uppercase) for _ in range(10))
+      return create_jobspec(self.name,
+                            ['tools\\run_tests\\artifacts\\build_artifact_python.bat',
+                             self.py_version,
+                             '32' if self.arch == 'x86' else '64',
+                             dir
+                            ],
+                            environ=environ,
+                            shell=True)
+    else:
+      environ['PYTHON'] = self.py_version
+      environ['SKIP_PIP_INSTALL'] = 'TRUE'
+      return create_jobspec(self.name,
+                            ['tools/run_tests/artifacts/build_artifact_python.sh'],
+                            environ=environ)
+
+  def __str__(self):
+    return self.name
+
+
+class RubyArtifact:
+  """Builds ruby native gem."""
+
+  def __init__(self, platform, arch):
+    self.name = 'ruby_native_gem_%s_%s' % (platform, arch)
+    self.platform = platform
+    self.arch = arch
+    self.labels = ['artifact', 'ruby', platform, arch]
+
+  def pre_build_jobspecs(self):
+    return []
+
+  def build_jobspec(self):
+    if self.platform == 'windows':
+      raise Exception("Not supported yet")
+    else:
+      if self.platform == 'linux':
+        environ = {}
+        if self.arch == 'x86':
+          environ['SETARCH_CMD'] = 'linux32'
+        return create_docker_jobspec(self.name,
+            'tools/dockerfile/grpc_artifact_linux_%s' % self.arch,
+            'tools/run_tests/artifacts/build_artifact_ruby.sh',
+            environ=environ)
+      else:
+        return create_jobspec(self.name,
+                              ['tools/run_tests/artifacts/build_artifact_ruby.sh'])
+
+
+class CSharpExtArtifact:
+  """Builds C# native extension library"""
+
+  def __init__(self, platform, arch):
+    self.name = 'csharp_ext_%s_%s' % (platform, arch)
+    self.platform = platform
+    self.arch = arch
+    self.labels = ['artifact', 'csharp', platform, arch]
+
+  def pre_build_jobspecs(self):
+    if self.platform == 'windows':
+      return [create_jobspec('prebuild_%s' % self.name,
+                             ['tools\\run_tests\\helper_scripts\\pre_build_c.bat'],
+                             shell=True,
+                             flake_retries=5,
+                             timeout_retries=2)]
+    else:
+      return []
+
+  def build_jobspec(self):
+    if self.platform == 'windows':
+      msbuild_platform = 'Win32' if self.arch == 'x86' else self.arch
+      return create_jobspec(self.name,
+                            ['tools\\run_tests\\artifacts\\build_artifact_csharp.bat',
+                             'vsprojects\\grpc_csharp_ext.sln',
+                             '/p:Configuration=Release',
+                             '/p:PlatformToolset=v120',
+                             '/p:Platform=%s' % msbuild_platform],
+                            shell=True)
+    else:
+      environ = {'CONFIG': 'opt',
+                 'EMBED_OPENSSL': 'true',
+                 'EMBED_ZLIB': 'true',
+                 'CFLAGS': '-DGPR_BACKWARDS_COMPATIBILITY_MODE',
+                 'LDFLAGS': ''}
+      if self.platform == 'linux':
+        return create_docker_jobspec(self.name,
+            'tools/dockerfile/grpc_artifact_linux_%s' % self.arch,
+            'tools/run_tests/artifacts/build_artifact_csharp.sh',
+            environ=environ)
+      else:
+        archflag = _ARCH_FLAG_MAP[self.arch]
+        environ['CFLAGS'] += ' %s %s' % (archflag, _MACOS_COMPAT_FLAG)
+        environ['LDFLAGS'] += ' %s' % archflag
+        return create_jobspec(self.name,
+                              ['tools/run_tests/artifacts/build_artifact_csharp.sh'],
+                              environ=environ)
+
+  def __str__(self):
+    return self.name
+
+
+node_gyp_arch_map = {
+  'x86': 'ia32',
+  'x64': 'x64'
+}
+
+class NodeExtArtifact:
+  """Builds Node native extension"""
+
+  def __init__(self, platform, arch):
+    self.name = 'node_ext_{0}_{1}'.format(platform, arch)
+    self.platform = platform
+    self.arch = arch
+    self.gyp_arch = node_gyp_arch_map[arch]
+    self.labels = ['artifact', 'node', platform, arch]
+
+  def pre_build_jobspecs(self):
+    return []
+
+  def build_jobspec(self):
+    if self.platform == 'windows':
+      return create_jobspec(self.name,
+                            ['tools\\run_tests\\artifacts\\build_artifact_node.bat',
+                             self.gyp_arch],
+                            shell=True)
+    else:
+      if self.platform == 'linux':
+        return create_docker_jobspec(
+            self.name,
+            'tools/dockerfile/grpc_artifact_linux_{}'.format(self.arch),
+            'tools/run_tests/artifacts/build_artifact_node.sh {}'.format(self.gyp_arch))
+      else:
+        return create_jobspec(self.name,
+                              ['tools/run_tests/artifacts/build_artifact_node.sh',
+                               self.gyp_arch])
+
+class PHPArtifact:
+  """Builds PHP PECL package"""
+
+  def __init__(self, platform, arch):
+    self.name = 'php_pecl_package_{0}_{1}'.format(platform, arch)
+    self.platform = platform
+    self.arch = arch
+    self.labels = ['artifact', 'php', platform, arch]
+
+  def pre_build_jobspecs(self):
+    return []
+
+  def build_jobspec(self):
+    if self.platform == 'linux':
+      return create_docker_jobspec(
+          self.name,
+          'tools/dockerfile/grpc_artifact_linux_{}'.format(self.arch),
+          'tools/run_tests/artifacts/build_artifact_php.sh')
+    else:
+      return create_jobspec(self.name,
+                            ['tools/run_tests/artifacts/build_artifact_php.sh'])
+
+class ProtocArtifact:
+  """Builds protoc and protoc-plugin artifacts"""
+
+  def __init__(self, platform, arch):
+    self.name = 'protoc_%s_%s' % (platform, arch)
+    self.platform = platform
+    self.arch = arch
+    self.labels = ['artifact', 'protoc', platform, arch]
+
+  def pre_build_jobspecs(self):
+      return []
+
+  def build_jobspec(self):
+    if self.platform != 'windows':
+      cxxflags = '-DNDEBUG %s' % _ARCH_FLAG_MAP[self.arch]
+      ldflags = '%s' % _ARCH_FLAG_MAP[self.arch]
+      if self.platform != 'macos':
+        ldflags += '  -static-libgcc -static-libstdc++ -s'
+      environ={'CONFIG': 'opt',
+               'CXXFLAGS': cxxflags,
+               'LDFLAGS': ldflags,
+               'PROTOBUF_LDFLAGS_EXTRA': ldflags}
+      if self.platform == 'linux':
+        return create_docker_jobspec(self.name,
+            'tools/dockerfile/grpc_artifact_protoc',
+            'tools/run_tests/artifacts/build_artifact_protoc.sh',
+            environ=environ)
+      else:
+        environ['CXXFLAGS'] += ' -std=c++11 -stdlib=libc++ %s' % _MACOS_COMPAT_FLAG
+        return create_jobspec(self.name,
+            ['tools/run_tests/artifacts/build_artifact_protoc.sh'],
+            environ=environ)
+    else:
+      generator = 'Visual Studio 12 Win64' if self.arch == 'x64' else 'Visual Studio 12' 
+      vcplatform = 'x64' if self.arch == 'x64' else 'Win32'
+      return create_jobspec(self.name,
+                            ['tools\\run_tests\\artifacts\\build_artifact_protoc.bat'],
+                            environ={'generator': generator,
+                                     'Platform': vcplatform})
+
+  def __str__(self):
+    return self.name
+
+
+def targets():
+  """Gets list of supported targets"""
+  return ([Cls(platform, arch)
+           for Cls in (CSharpExtArtifact, NodeExtArtifact, ProtocArtifact)
+           for platform in ('linux', 'macos', 'windows')
+           for arch in ('x86', 'x64')] +
+          [PythonArtifact('linux', 'x86', 'cp27-cp27m'),
+           PythonArtifact('linux', 'x86', 'cp27-cp27mu'),
+           PythonArtifact('linux', 'x86', 'cp34-cp34m'),
+           PythonArtifact('linux', 'x86', 'cp35-cp35m'),
+           PythonArtifact('linux', 'x64', 'cp27-cp27m'),
+           PythonArtifact('linux', 'x64', 'cp27-cp27mu'),
+           PythonArtifact('linux', 'x64', 'cp34-cp34m'),
+           PythonArtifact('linux', 'x64', 'cp35-cp35m'),
+           PythonArtifact('macos', 'x64', 'python2.7'),
+           PythonArtifact('macos', 'x64', 'python3.4'),
+           PythonArtifact('macos', 'x64', 'python3.5'),
+           PythonArtifact('windows', 'x86', 'Python27_32bits'),
+           PythonArtifact('windows', 'x86', 'Python34_32bits'),
+           PythonArtifact('windows', 'x86', 'Python35_32bits'),
+           PythonArtifact('windows', 'x64', 'Python27'),
+           PythonArtifact('windows', 'x64', 'Python34'),
+           PythonArtifact('windows', 'x64', 'Python35'),
+           RubyArtifact('linux', 'x86'),
+           RubyArtifact('linux', 'x64'),
+           RubyArtifact('macos', 'x64'),
+           PHPArtifact('linux', 'x64'),
+           PHPArtifact('macos', 'x64')])