Merge pull request #11115 from jtattermusch/revamp_artifacts

Revamp artifacts, packages and distribtests

18 files changed, 141 insertions, 65 deletions

diff --git a/tools/run_tests/artifacts/artifact_targets.py b/tools/run_tests/artifacts/artifact_targets.py
index c90da066c3..5a71b96f9e 100644
--- a/tools/run_tests/artifacts/artifact_targets.py
+++ b/tools/run_tests/artifacts/artifact_targets.py
@@ -45,6 +45,7 @@ def create_docker_jobspec(name, dockerfile_dir, shell_command, environ={},
   """Creates jobspec for a task running under docker."""
   environ = environ.copy()
   environ['RUN_COMMAND'] = shell_command
+  environ['ARTIFACTS_OUT'] = 'artifacts/%s' % name
 
   docker_args=[]
   for k,v in environ.items():
@@ -65,9 +66,20 @@ def create_docker_jobspec(name, dockerfile_dir, shell_command, environ={},
   return jobspec
 
 
-def create_jobspec(name, cmdline, environ=None, shell=False,
-                   flake_retries=0, timeout_retries=0, timeout_seconds=30*60):
+def create_jobspec(name, cmdline, environ={}, shell=False,
+                   flake_retries=0, timeout_retries=0, timeout_seconds=30*60,
+                   use_workspace=False,
+                   cpu_cost=1.0):
   """Creates jobspec."""
+  environ = environ.copy()
+  if use_workspace:
+    environ['WORKSPACE_NAME'] = 'workspace_%s' % name
+    environ['ARTIFACTS_OUT'] = os.path.join('..', 'artifacts', name)
+    cmdline = ['bash',
+               'tools/run_tests/artifacts/run_in_workspace.sh'] + cmdline
+  else:
+    environ['ARTIFACTS_OUT'] = os.path.join('artifacts', name)
+
   jobspec = jobset.JobSpec(
           cmdline=cmdline,
           environ=environ,
@@ -75,7 +87,8 @@ def create_jobspec(name, cmdline, environ=None, shell=False,
           timeout_seconds=timeout_seconds,
           flake_retries=flake_retries,
           timeout_retries=timeout_retries,
-          shell=shell)
+          shell=shell,
+          cpu_cost=cpu_cost)
   return jobspec
 
 
@@ -137,13 +150,14 @@ class PythonArtifact:
                              dir
                             ],
                             environ=environ,
-                            shell=True)
+                            use_workspace=True)
     else:
       environ['PYTHON'] = self.py_version
       environ['SKIP_PIP_INSTALL'] = 'TRUE'
       return create_jobspec(self.name,
                             ['tools/run_tests/artifacts/build_artifact_python.sh'],
-                            environ=environ)
+                            environ=environ,
+                            use_workspace=True)
 
   def __str__(self):
     return self.name
@@ -162,20 +176,11 @@ class RubyArtifact:
     return []
 
   def build_jobspec(self):
-    if self.platform == 'windows':
-      raise Exception("Not supported yet")
-    else:
-      if self.platform == 'linux':
-        environ = {}
-        if self.arch == 'x86':
-          environ['SETARCH_CMD'] = 'linux32'
-        return create_docker_jobspec(self.name,
-            'tools/dockerfile/grpc_artifact_linux_%s' % self.arch,
-            'tools/run_tests/artifacts/build_artifact_ruby.sh',
-            environ=environ)
-      else:
-        return create_jobspec(self.name,
-                              ['tools/run_tests/artifacts/build_artifact_ruby.sh'])
+    # Ruby build uses docker internally and docker cannot be nested.
+    # We are using a custom workspace instead.
+    return create_jobspec(self.name,
+                          ['tools/run_tests/artifacts/build_artifact_ruby.sh'],
+                          use_workspace=True)
 
 
 class CSharpExtArtifact:
@@ -196,7 +201,7 @@ class CSharpExtArtifact:
       return create_jobspec(self.name,
                             ['tools\\run_tests\\artifacts\\build_artifact_csharp.bat',
                              cmake_arch_option],
-                            shell=True)
+                            use_workspace=True)
     else:
       environ = {'CONFIG': 'opt',
                  'EMBED_OPENSSL': 'true',
@@ -216,7 +221,8 @@ class CSharpExtArtifact:
         environ['LDFLAGS'] += ' %s' % archflag
         return create_jobspec(self.name,
                               ['tools/run_tests/artifacts/build_artifact_csharp.sh'],
-                              environ=environ)
+                              environ=environ,
+                              use_workspace=True)
 
   def __str__(self):
     return self.name
@@ -242,10 +248,15 @@ class NodeExtArtifact:
 
   def build_jobspec(self):
     if self.platform == 'windows':
+      # Simultaneous builds of node on the same windows machine are flaky.
+      # Set x86 build as exclusive to make sure there is only one node build
+      # at a time. See https://github.com/grpc/grpc/issues/8293
+      cpu_cost = 1e6 if self.arch != 'x64' else 1.0
       return create_jobspec(self.name,
                             ['tools\\run_tests\\artifacts\\build_artifact_node.bat',
                              self.gyp_arch],
-                            shell=True)
+                            use_workspace=True,
+                            cpu_cost=cpu_cost)
     else:
       if self.platform == 'linux':
         return create_docker_jobspec(
@@ -255,7 +266,8 @@ class NodeExtArtifact:
       else:
         return create_jobspec(self.name,
                               ['tools/run_tests/artifacts/build_artifact_node.sh',
-                               self.gyp_arch])
+                               self.gyp_arch],
+                               use_workspace=True)
 
 class PHPArtifact:
   """Builds PHP PECL package"""
@@ -277,7 +289,8 @@ class PHPArtifact:
           'tools/run_tests/artifacts/build_artifact_php.sh')
     else:
       return create_jobspec(self.name,
-                            ['tools/run_tests/artifacts/build_artifact_php.sh'])
+                            ['tools/run_tests/artifacts/build_artifact_php.sh'],
+                            use_workspace=True)
 
 class ProtocArtifact:
   """Builds protoc and protoc-plugin artifacts"""
@@ -310,14 +323,16 @@ class ProtocArtifact:
         environ['CXXFLAGS'] += ' -std=c++11 -stdlib=libc++ %s' % _MACOS_COMPAT_FLAG
         return create_jobspec(self.name,
             ['tools/run_tests/artifacts/build_artifact_protoc.sh'],
-            environ=environ)
+            environ=environ,
+            use_workspace=True)
     else:
       generator = 'Visual Studio 12 Win64' if self.arch == 'x64' else 'Visual Studio 12' 
       vcplatform = 'x64' if self.arch == 'x64' else 'Win32'
       return create_jobspec(self.name,
                             ['tools\\run_tests\\artifacts\\build_artifact_protoc.bat'],
                             environ={'generator': generator,
-                                     'Platform': vcplatform})
+                                     'Platform': vcplatform},
+                            use_workspace=True)
 
   def __str__(self):
     return self.name
@@ -351,7 +366,6 @@ def targets():
            PythonArtifact('windows', 'x64', 'Python34'),
            PythonArtifact('windows', 'x64', 'Python35'),
            PythonArtifact('windows', 'x64', 'Python36'),
-           RubyArtifact('linux', 'x86'),
            RubyArtifact('linux', 'x64'),
            RubyArtifact('macos', 'x64'),
            PHPArtifact('linux', 'x64'),
diff --git a/tools/run_tests/artifacts/build_artifact_csharp.bat b/tools/run_tests/artifacts/build_artifact_csharp.bat
index f84ebc5a35..ebb11bf909 100644
--- a/tools/run_tests/artifacts/build_artifact_csharp.bat
+++ b/tools/run_tests/artifacts/build_artifact_csharp.bat
@@ -37,8 +37,8 @@ cd cmake\build\%ARCHITECTURE%
 cmake --build . --target grpc_csharp_ext --config Release
 cd ..\..\..
 
-mkdir artifacts
-copy /Y cmake\build\Win32\Release\grpc_csharp_ext.dll artifacts || copy /Y cmake\build\x64\Release\grpc_csharp_ext.dll artifacts || goto :error
+mkdir -p %ARTIFACTS_OUT%
+copy /Y cmake\build\Win32\Release\grpc_csharp_ext.dll %ARTIFACTS_OUT% || copy /Y cmake\build\x64\Release\grpc_csharp_ext.dll %ARTIFACTS_OUT% || goto :error
 
 goto :EOF
 
diff --git a/tools/run_tests/artifacts/build_artifact_csharp.sh b/tools/run_tests/artifacts/build_artifact_csharp.sh
index aed04b2745..2bfa749fa3 100755
--- a/tools/run_tests/artifacts/build_artifact_csharp.sh
+++ b/tools/run_tests/artifacts/build_artifact_csharp.sh
@@ -34,5 +34,5 @@ cd $(dirname $0)/../../..
 
 make grpc_csharp_ext
 
-mkdir -p artifacts
-cp libs/opt/libgrpc_csharp_ext.so artifacts || cp libs/opt/libgrpc_csharp_ext.dylib artifacts
+mkdir -p "${ARTIFACTS_OUT}"
+cp libs/opt/libgrpc_csharp_ext.so "${ARTIFACTS_OUT}" || cp libs/opt/libgrpc_csharp_ext.dylib "${ARTIFACTS_OUT}"
diff --git a/tools/run_tests/artifacts/build_artifact_node.bat b/tools/run_tests/artifacts/build_artifact_node.bat
index bfd4461f72..da4d479a8c 100644
--- a/tools/run_tests/artifacts/build_artifact_node.bat
+++ b/tools/run_tests/artifacts/build_artifact_node.bat
@@ -37,7 +37,7 @@ del /f /q BUILD || rmdir build /s /q
 
 call npm update || goto :error
 
-mkdir artifacts
+mkdir -p %ARTIFACTS_OUT%
 
 for %%v in (%node_versions%) do (
   call .\node_modules\.bin\node-pre-gyp.cmd configure build --target=%%v --target_arch=%1
@@ -47,13 +47,13 @@ for %%v in (%node_versions%) do (
   rmdir "%HOMEDRIVE%%HOMEPATH%\.node-gyp\iojs-%%v\include\node\openssl" /S /Q
   call .\node_modules\.bin\node-pre-gyp.cmd build package testpackage --target=%%v --target_arch=%1 || goto :error
 
-  xcopy /Y /I /S build\stage\* artifacts\ || goto :error
+  xcopy /Y /I /S build\stage\* %ARTIFACTS_OUT%\ || goto :error
 )
 
 for %%v in (%electron_versions%) do (
   cmd /V /C "set "HOME=%HOMEDRIVE%%HOMEPATH%\electron-gyp" && call .\node_modules\.bin\node-pre-gyp.cmd configure rebuild package testpackage --runtime=electron --target=%%v --target_arch=%1 --disturl=https://atom.io/download/electron" || goto :error
 
-  xcopy /Y /I /S build\stage\* artifacts\ || goto :error
+  xcopy /Y /I /S build\stage\* %ARTIFACTS_OUT%\ || goto :error
 )
 if %errorlevel% neq 0 exit /b %errorlevel%
 
diff --git a/tools/run_tests/artifacts/build_artifact_node.sh b/tools/run_tests/artifacts/build_artifact_node.sh
index 7a747551e8..3947bded6f 100755
--- a/tools/run_tests/artifacts/build_artifact_node.sh
+++ b/tools/run_tests/artifacts/build_artifact_node.sh
@@ -38,7 +38,7 @@ cd $(dirname $0)/../../..
 
 rm -rf build || true
 
-mkdir -p artifacts
+mkdir -p "${ARTIFACTS_OUT}"
 
 npm update
 
@@ -49,11 +49,11 @@ electron_versions=( 1.0.0 1.1.0 1.2.0 1.3.0 1.4.0 1.5.0 1.6.0 )
 for version in ${node_versions[@]}
 do
   ./node_modules/.bin/node-pre-gyp configure rebuild package testpackage --target=$version --target_arch=$NODE_TARGET_ARCH --grpc_alpine=true
-  cp -r build/stage/* artifacts/
+  cp -r build/stage/* "${ARTIFACTS_OUT}"/
 done
 
 for version in ${electron_versions[@]}
 do
   HOME=~/.electron-gyp ./node_modules/.bin/node-pre-gyp configure rebuild package testpackage --runtime=electron --target=$version --target_arch=$NODE_TARGET_ARCH --disturl=https://atom.io/download/electron
-  cp -r build/stage/* artifacts/
+  cp -r build/stage/* "${ARTIFACTS_OUT}"/
 done
diff --git a/tools/run_tests/artifacts/build_artifact_php.sh b/tools/run_tests/artifacts/build_artifact_php.sh
index c8d55860c1..d7f8c8f028 100755
--- a/tools/run_tests/artifacts/build_artifact_php.sh
+++ b/tools/run_tests/artifacts/build_artifact_php.sh
@@ -33,8 +33,8 @@ set -ex
 
 cd $(dirname $0)/../../..
 
-mkdir -p artifacts
+mkdir -p "${ARTIFACTS_OUT}"
 
 pear package
 
-cp -r grpc-*.tgz artifacts/
+cp -r grpc-*.tgz "${ARTIFACTS_OUT}"/
diff --git a/tools/run_tests/artifacts/build_artifact_protoc.bat b/tools/run_tests/artifacts/build_artifact_protoc.bat
index b2bf86da40..def64bdfee 100644
--- a/tools/run_tests/artifacts/build_artifact_protoc.bat
+++ b/tools/run_tests/artifacts/build_artifact_protoc.bat
@@ -27,7 +27,7 @@
 @rem (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 @rem OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-mkdir artifacts
+mkdir -p %ARTIFACTS_OUT%
 
 setlocal
 cd third_party/protobuf/cmake
@@ -39,8 +39,8 @@ endlocal
 
 call vsprojects/build_plugins.bat || goto :error
 
-xcopy /Y third_party\protobuf\cmake\build\solution\Release\protoc.exe artifacts\ || goto :error
-xcopy /Y vsprojects\Release\*_plugin.exe artifacts\ || xcopy /Y vsprojects\x64\Release\*_plugin.exe artifacts\ || goto :error
+xcopy /Y third_party\protobuf\cmake\build\solution\Release\protoc.exe %ARTIFACTS_OUT%\ || goto :error
+xcopy /Y vsprojects\Release\*_plugin.exe %ARTIFACTS_OUT%\ || xcopy /Y vsprojects\x64\Release\*_plugin.exe %ARTIFACTS_OUT%\ || goto :error
 
 goto :EOF
 
diff --git a/tools/run_tests/artifacts/build_artifact_protoc.sh b/tools/run_tests/artifacts/build_artifact_protoc.sh
index 26c2280eff..df78203739 100755
--- a/tools/run_tests/artifacts/build_artifact_protoc.sh
+++ b/tools/run_tests/artifacts/build_artifact_protoc.sh
@@ -37,5 +37,5 @@ cd $(dirname $0)/../../..
 
 make plugins
 
-mkdir -p artifacts
-cp bins/opt/protobuf/protoc bins/opt/*_plugin artifacts/
+mkdir -p "${ARTIFACTS_OUT}"
+cp bins/opt/protobuf/protoc bins/opt/*_plugin "${ARTIFACTS_OUT}"/
diff --git a/tools/run_tests/artifacts/build_artifact_python.bat b/tools/run_tests/artifacts/build_artifact_python.bat
index 246713a6ce..860b9e831d 100644
--- a/tools/run_tests/artifacts/build_artifact_python.bat
+++ b/tools/run_tests/artifacts/build_artifact_python.bat
@@ -38,8 +38,9 @@ set GRPC_PYTHON_BUILD_WITH_CYTHON=1
 
 @rem Multiple builds are running simultaneously, so to avoid distutils
 @rem file collisions, we build everything in a tmp directory
-if not exist "artifacts" mkdir "artifacts"
-set ARTIFACT_DIR=%cd%\artifacts
+@rem TODO(jtattermusch): it doesn't look like builds are actually running in parallel in the same dir
+mkdir -p %ARTIFACTS_OUT%
+set ARTIFACT_DIR=%cd%\%ARTIFACTS_OUT%
 set BUILD_DIR=C:\Windows\Temp\pygrpc-%3\
 mkdir %BUILD_DIR%
 xcopy /s/e/q %cd%\* %BUILD_DIR%
diff --git a/tools/run_tests/artifacts/build_artifact_python.sh b/tools/run_tests/artifacts/build_artifact_python.sh
index 5a5506029a..a1f0a5857a 100755
--- a/tools/run_tests/artifacts/build_artifact_python.sh
+++ b/tools/run_tests/artifacts/build_artifact_python.sh
@@ -41,8 +41,9 @@ export AUDITWHEEL=${AUDITWHEEL:-auditwheel}
 # Because multiple builds run in parallel, some distutils file
 # operations may collide.  To avoid this, each build is run in
 # a temp directory
-mkdir -p artifacts
-ARTIFACT_DIR="$PWD/artifacts"
+# TODO(jtattermusch): it doesn't look like builds are actually running in parallel in the same dir
+mkdir -p "${ARTIFACTS_OUT}"
+ARTIFACT_DIR="$PWD/${ARTIFACTS_OUT}"
 BUILD_DIR=`mktemp -d "${TMPDIR:-/tmp}/pygrpc.XXXXXX"`
 trap "rm -rf $BUILD_DIR" EXIT
 cp -r * "$BUILD_DIR"
diff --git a/tools/run_tests/artifacts/build_artifact_ruby.sh b/tools/run_tests/artifacts/build_artifact_ruby.sh
index ca461ba561..c92d7a8a89 100755
--- a/tools/run_tests/artifacts/build_artifact_ruby.sh
+++ b/tools/run_tests/artifacts/build_artifact_ruby.sh
@@ -63,6 +63,6 @@ if [ "$SYSTEM" == "Darwin" ] ; then
   rm `ls pkg/*.gem | grep -v darwin`
 fi
 
-mkdir -p artifacts
+mkdir -p "${ARTIFACTS_OUT}"
 
-cp pkg/*.gem artifacts
+cp pkg/*.gem "${ARTIFACTS_OUT}"/
diff --git a/tools/run_tests/artifacts/build_package_node.sh b/tools/run_tests/artifacts/build_package_node.sh
index 8b5e8c0bc1..d06f141869 100755
--- a/tools/run_tests/artifacts/build_package_node.sh
+++ b/tools/run_tests/artifacts/build_package_node.sh
@@ -40,7 +40,7 @@ base=$(pwd)
 artifacts=$base/artifacts
 
 mkdir -p $artifacts
-cp -r $EXTERNAL_GIT_ROOT/architecture={x86,x64},language=node,platform={windows,linux,macos}/artifacts/* $artifacts/ || true
+cp -r $EXTERNAL_GIT_ROOT/platform={windows,linux,macos}/artifacts/node_ext_*/* $artifacts/ || true
 
 npm update
 npm pack
@@ -86,7 +86,7 @@ for arch in {x86,x64}; do
         ;;
     esac
     rm -r bin/*
-    input_dir="$EXTERNAL_GIT_ROOT/architecture=$arch,language=protoc,platform=$plat/artifacts"
+    input_dir="$EXTERNAL_GIT_ROOT/platform=${plat}/artifacts/protoc_${plat}_${arch}"
     cp $input_dir/protoc* bin/
     cp $input_dir/grpc_node_plugin* bin/
     mkdir -p bin/google/protobuf
diff --git a/tools/run_tests/artifacts/build_package_php.sh b/tools/run_tests/artifacts/build_package_php.sh
index 42a8d9f8df..16da58d065 100755
--- a/tools/run_tests/artifacts/build_package_php.sh
+++ b/tools/run_tests/artifacts/build_package_php.sh
@@ -33,4 +33,4 @@ set -ex
 cd $(dirname $0)/../../..
 
 mkdir -p artifacts/
-cp -r $EXTERNAL_GIT_ROOT/architecture={x86,x64},language=php,platform={windows,linux,macos}/artifacts/* artifacts/ || true
+cp -r $EXTERNAL_GIT_ROOT/platform={windows,linux,macos}/artifacts/php_*/* artifacts/ || true
diff --git a/tools/run_tests/artifacts/build_package_python.sh b/tools/run_tests/artifacts/build_package_python.sh
index 4a1c15ceee..9148bb0e59 100755
--- a/tools/run_tests/artifacts/build_package_python.sh
+++ b/tools/run_tests/artifacts/build_package_python.sh
@@ -36,7 +36,7 @@ mkdir -p artifacts/
 
 # All the python packages have been built in the artifact phase already
 # and we only collect them here to deliver them to the distribtest phase.
-cp -r $EXTERNAL_GIT_ROOT/architecture={x86,x64},language=python,platform={windows,linux,macos}/artifacts/* artifacts/ || true
+cp -r $EXTERNAL_GIT_ROOT/platform={windows,linux,macos}/artifacts/python_*/* artifacts/ || true
 
 # TODO: all the artifact builder configurations generate a grpcio-VERSION.tar.gz
 # source distribution package, and only one of them will end up
diff --git a/tools/run_tests/artifacts/build_package_ruby.sh b/tools/run_tests/artifacts/build_package_ruby.sh
index b4d20d8a4c..8388b84cc5 100755
--- a/tools/run_tests/artifacts/build_package_ruby.sh
+++ b/tools/run_tests/artifacts/build_package_ruby.sh
@@ -38,7 +38,7 @@ mkdir -p artifacts/
 
 # All the ruby packages have been built in the artifact phase already
 # and we only collect them here to deliver them to the distribtest phase.
-cp -r $EXTERNAL_GIT_ROOT/architecture={x86,x64},language=ruby,platform={windows,linux,macos}/artifacts/* artifacts/ || true
+cp -r $EXTERNAL_GIT_ROOT/platform={windows,linux,macos}/artifacts/ruby_native_gem_*/* artifacts/ || true
 
 well_known_protos=( any api compiler/plugin descriptor duration empty field_mask source_context struct timestamp type wrappers )
 
@@ -56,7 +56,7 @@ for arch in {x86,x64}; do
       ;;
   esac
   for plat in {windows,linux,macos}; do
-    input_dir="$EXTERNAL_GIT_ROOT/architecture=$arch,language=protoc,platform=$plat/artifacts"
+    input_dir="$EXTERNAL_GIT_ROOT/platform=${plat}/artifacts/protoc_${plat}_${arch}"
     output_dir="$base/src/ruby/tools/bin/${ruby_arch}-${plat}"
     mkdir -p $output_dir/google/protobuf
     mkdir -p $output_dir/google/protobuf/compiler  # needed for plugin.proto
diff --git a/tools/run_tests/artifacts/distribtest_targets.py b/tools/run_tests/artifacts/distribtest_targets.py
index 097fd2d8b5..3cb7d97b6d 100644
--- a/tools/run_tests/artifacts/distribtest_targets.py
+++ b/tools/run_tests/artifacts/distribtest_targets.py
@@ -63,8 +63,14 @@ def create_docker_jobspec(name, dockerfile_dir, shell_command, environ={},
 
 
 def create_jobspec(name, cmdline, environ=None, shell=False,
-                   flake_retries=0, timeout_retries=0):
+                   flake_retries=0, timeout_retries=0,
+                   use_workspace=False):
   """Creates jobspec."""
+  environ = environ.copy()
+  if use_workspace:
+    environ['WORKSPACE_NAME'] = 'workspace_%s' % name
+    cmdline = ['bash',
+               'tools/run_tests/artifacts/run_in_workspace.sh'] + cmdline
   jobspec = jobset.JobSpec(
           cmdline=cmdline,
           environ=environ,
@@ -80,7 +86,7 @@ class CSharpDistribTest(object):
   """Tests C# NuGet package"""
 
   def __init__(self, platform, arch, docker_suffix=None, use_dotnet_cli=False):
-    self.name = 'csharp_nuget_%s_%s' % (platform, arch)
+    self.name = 'csharp_%s_%s' % (platform, arch)
     self.platform = platform
     self.arch = arch
     self.docker_suffix = docker_suffix
@@ -110,16 +116,18 @@ class CSharpDistribTest(object):
     elif self.platform == 'macos':
       return create_jobspec(self.name,
           ['test/distrib/csharp/run_distrib_test%s.sh' % self.script_suffix],
-          environ={'EXTERNAL_GIT_ROOT': '../../..'})
+          environ={'EXTERNAL_GIT_ROOT': '../../../..'},
+          use_workspace=True)
     elif self.platform == 'windows':
       if self.arch == 'x64':
         environ={'MSBUILD_EXTRA_ARGS': '/p:Platform=x64',
                  'DISTRIBTEST_OUTPATH': 'DistribTest\\bin\\x64\\Debug'}
       else:
-        environ={'DISTRIBTEST_OUTPATH': 'DistribTest\\bin\\\Debug'}
+        environ={'DISTRIBTEST_OUTPATH': 'DistribTest\\bin\\Debug'}
       return create_jobspec(self.name,
           ['test\\distrib\\csharp\\run_distrib_test%s.bat' % self.script_suffix],
-          environ=environ)
+          environ=environ,
+          use_workspace=True)
     else:
       raise Exception("Not supported yet.")
 
@@ -161,7 +169,8 @@ class NodeDistribTest(object):
       return create_jobspec(self.name,
                             ['test/distrib/node/run_distrib_test.sh',
                              str(self.node_version)],
-                            environ={'EXTERNAL_GIT_ROOT': '../../..'})
+                            environ={'EXTERNAL_GIT_ROOT': '../../../..'},
+                            use_workspace=True)
     else:
       raise Exception("Not supported yet.")
 
@@ -249,7 +258,8 @@ class PHPDistribTest(object):
     elif self.platform == 'macos':
       return create_jobspec(self.name,
           ['test/distrib/php/run_distrib_test.sh'],
-          environ={'EXTERNAL_GIT_ROOT': '../../..'})
+          environ={'EXTERNAL_GIT_ROOT': '../../../..'},
+          use_workspace=True)
     else:
       raise Exception("Not supported yet.")
 
@@ -339,5 +349,5 @@ def targets():
             NodeDistribTest('linux', 'x64', os, version)
             for os in ('wheezy', 'jessie', 'ubuntu1204', 'ubuntu1404',
                        'ubuntu1504', 'ubuntu1510', 'ubuntu1604')
-            for version in ('0.12', '3', '4', '5')
+            for version in ('4', '5')
           ]
diff --git a/tools/run_tests/artifacts/run_in_workspace.sh b/tools/run_tests/artifacts/run_in_workspace.sh
new file mode 100755
index 0000000000..ed3bfda8c0
--- /dev/null
+++ b/tools/run_tests/artifacts/run_in_workspace.sh
@@ -0,0 +1,47 @@
+#!/bin/bash
+# Copyright 2015, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+# Create a workspace in a subdirectory to allow running multiple builds in isolation.
+# WORKSPACE_NAME env variable needs to contain name of the workspace to create.
+# All cmdline args will be executed as a command.
+set -ex
+
+cd $(dirname $0)/../../..
+export repo_root=$(pwd)
+
+rm -rf "${WORKSPACE_NAME}"
+git clone . "${WORKSPACE_NAME}"
+# clone gRPC submodules, use data from locally cloned submodules where possible
+git submodule foreach 'cd "${repo_root}/${WORKSPACE_NAME}" \
+    && git submodule update --init --reference ${repo_root}/${name} ${name}'
+
+echo "Running in workspace ${WORKSPACE_NAME}"
+cd ${WORKSPACE_NAME}
+$@
diff --git a/tools/run_tests/task_runner.py b/tools/run_tests/task_runner.py
index 0ec7efbbee..7feac29673 100755
--- a/tools/run_tests/task_runner.py
+++ b/tools/run_tests/task_runner.py
@@ -40,6 +40,7 @@ import artifacts.artifact_targets as artifact_targets
 import artifacts.distribtest_targets as distribtest_targets
 import artifacts.package_targets as package_targets
 import python_utils.jobset as jobset
+import python_utils.report_utils as report_utils
 
 _TARGETS = []
 _TARGETS += artifact_targets.targets()
@@ -116,8 +117,10 @@ if not build_jobs:
   sys.exit(1)
 
 jobset.message('START', 'Building targets.', do_newline=True)
-num_failures, _ = jobset.run(
+num_failures, resultset = jobset.run(
     build_jobs, newline_on_success=True, maxjobs=args.jobs)
+report_utils.render_junit_xml_report(resultset, 'report_taskrunner_sponge_log.xml',
+                                     suite_name='tasks')
 if num_failures == 0:
   jobset.message('SUCCESS', 'All targets built successfully.',
                  do_newline=True)