merge with head

19 files changed, 216 insertions, 167 deletions

diff --git a/tools/buildgen/build-cleaner.py b/tools/buildgen/build-cleaner.py
index 8288a8998d..4e592ee3ef 100755
--- a/tools/buildgen/build-cleaner.py
+++ b/tools/buildgen/build-cleaner.py
@@ -37,7 +37,7 @@ import yaml
 
 TEST = (os.environ.get('TEST', 'false') == 'true')
 
-_TOP_LEVEL_KEYS = ['settings', 'filegroups', 'libs', 'targets', 'vspackages']
+_TOP_LEVEL_KEYS = ['settings', 'proto_deps', 'filegroups', 'libs', 'targets', 'vspackages']
 _VERSION_KEYS = ['major', 'minor', 'micro', 'build']
 _ELEM_KEYS = [
     'name',
diff --git a/tools/buildgen/generate_build_additions.sh b/tools/buildgen/generate_build_additions.sh
index f304af0ef6..a2cd8249ef 100644
--- a/tools/buildgen/generate_build_additions.sh
+++ b/tools/buildgen/generate_build_additions.sh
@@ -28,7 +28,7 @@
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-gen_build_yaml_dirs="src/boringssl test/core/end2end test/core/bad_client test/core/bad_ssl"
+gen_build_yaml_dirs="src/boringssl test/core/end2end test/core/bad_client test/core/bad_ssl src/proto"
 gen_build_files=""
 for gen_build_yaml in $gen_build_yaml_dirs
 do
diff --git a/tools/buildgen/plugins/transitive_dependencies.py b/tools/buildgen/plugins/transitive_dependencies.py
index c2d3da3a3b..01e7f61ea9 100644
--- a/tools/buildgen/plugins/transitive_dependencies.py
+++ b/tools/buildgen/plugins/transitive_dependencies.py
@@ -1,4 +1,4 @@
-# Copyright 2015, Google Inc.
+# Copyright 2015-2016, Google Inc.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -36,10 +36,13 @@ of the list of dependencies.
 """
 
 def get_lib(libs, name):
-  return next(lib for lib in libs if lib['name']==name)
+  try:
+    return next(lib for lib in libs if lib['name']==name)
+  except StopIteration:
+    return None
 
 def transitive_deps(lib, libs):
-  if 'deps' in lib:
+  if lib is not None and 'deps' in lib:
     # Recursively call transitive_deps on each dependency, and take the union
     return set.union(set(lib['deps']),
                      *[set(transitive_deps(get_lib(libs, dep), libs))
@@ -58,6 +61,10 @@ def mako_plugin(dictionary):
   node_modules = dictionary.get('node_modules')
   targets = dictionary.get('targets')
 
-  for target_list in (libs, node_modules, targets):
+  for target_list in (libs, targets, node_modules):
     for target in target_list:
       target['transitive_deps'] = transitive_deps(target, libs)
+
+  python_dependencies = dictionary.get('python_dependencies')
+  python_dependencies['transitive_deps'] = (
+      transitive_deps(python_dependencies, libs))
diff --git a/tools/distrib/check_copyright.py b/tools/distrib/check_copyright.py
index f54e5fad80..5e948e2dee 100755
--- a/tools/distrib/check_copyright.py
+++ b/tools/distrib/check_copyright.py
@@ -54,6 +54,9 @@ argp.add_argument('-a', '--ancient',
                   default=0,
                   action='store_const',
                   const=1)
+argp.add_argument('-f', '--fix',
+                  default=False,
+                  action='store_true');
 args = argp.parse_args()
 
 # open the license text
@@ -90,7 +93,7 @@ KNOWN_BAD = set([
 ])
 
 
-RE_YEAR = r'Copyright (?:[0-9]+\-)?([0-9]+), Google Inc\.'
+RE_YEAR = r'Copyright (?P<first_year>[0-9]+\-)?(?P<last_year>[0-9]+), Google Inc\.'
 RE_LICENSE = dict(
     (k, r'\n'.join(
         LICENSE_PREFIX[k] +
@@ -103,6 +106,9 @@ def load(name):
   with open(name) as f:
     return '\n'.join(line.rstrip() for line in f.read().splitlines())
 
+def save(name, text):
+  with open(name, 'w') as f:
+    f.write(text)
 
 assert(re.search(RE_LICENSE['LICENSE'], load('LICENSE')))
 assert(re.search(RE_LICENSE['Makefile'], load('Makefile')))
@@ -117,6 +123,7 @@ def log(cond, why, filename):
 
 
 # scan files, validate the text
+ok = True
 for filename in subprocess.check_output('git ls-tree -r --name-only -r HEAD',
                                         shell=True).splitlines():
   if filename in KNOWN_BAD: continue
@@ -130,14 +137,25 @@ for filename in subprocess.check_output('git ls-tree -r --name-only -r HEAD',
     log(args.skips, 'skip', filename)
     continue
   text = load(filename)
-  ok = True
   m = re.search(re_license, text)
   if m:
+    gdict = m.groupdict()
     last_modified = int(subprocess.check_output('git log -1 --format="%ad" --date=short -- ' + filename, shell=True)[0:4])
-    latest_claimed = int(m.group(1))
+    latest_claimed = int(gdict['last_year'])
     if last_modified > latest_claimed:
       print '%s modified %d but copyright only extends to %d' % (filename, last_modified, latest_claimed)
       ok = False
+      if args.fix:
+        span_start, span_end = m.span(2)
+        if not gdict['first_year']:
+          # prepend the old year to the current one.
+          text = '{}-{}{}'.format(text[:span_end], last_modified, text[span_end:])
+        else:  # already a year range
+          # simply update the last year
+          text = '{}{}{}'.format(text[:span_start], last_modified, text[span_end:])
+        save(filename, text)
+        print 'Fixed!'
+        ok = True
   elif 'DO NOT EDIT' not in text and 'AssemblyInfo.cs' not in filename and filename != 'src/boringssl/err_data.c':
     log(1, 'copyright missing', filename)
     ok = False
diff --git a/tools/distrib/clang_format_code.sh b/tools/distrib/clang_format_code.sh
index bc913cb6c1..612074acdf 100755
--- a/tools/distrib/clang_format_code.sh
+++ b/tools/distrib/clang_format_code.sh
@@ -1,5 +1,5 @@
 #!/bin/bash
-# Copyright 2015, Google Inc.
+# Copyright 2015-2016, Google Inc.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -37,5 +37,5 @@ cd $(dirname $0)/../..
 docker build -t grpc_clang_format tools/dockerfile/grpc_clang_format
 
 # run clang-format against the checked out codebase
-docker run -e TEST=$TEST --rm=true -v `pwd`:/local-code -t grpc_clang_format /clang_format_all_the_things.sh
+docker run -e TEST=$TEST --rm=true -v ${HOST_GIT_ROOT:-`pwd`}:/local-code -t grpc_clang_format /clang_format_all_the_things.sh
 
diff --git a/tools/distrib/python/submit.py b/tools/distrib/python/submit.py
index dffbefd5fe..08ace7c690 100755
--- a/tools/distrib/python/submit.py
+++ b/tools/distrib/python/submit.py
@@ -59,7 +59,7 @@ args = parser.parse_args()
 
 # Move to the root directory of Python GRPC.
 pkgdir = os.path.join(os.path.dirname(os.path.abspath(__file__)),
-                      '../../../src/python/grpcio')
+                      '../../../')
 # Remove previous distributions; they somehow confuse twine.
 try:
   shutil.rmtree(os.path.join(pkgdir, 'dist/'))
diff --git a/tools/dockerfile/grpc_clang_format/clang_format_all_the_things.sh b/tools/dockerfile/grpc_clang_format/clang_format_all_the_things.sh
index 60fd30cd6f..ededc6e809 100755
--- a/tools/dockerfile/grpc_clang_format/clang_format_all_the_things.sh
+++ b/tools/dockerfile/grpc_clang_format/clang_format_all_the_things.sh
@@ -1,5 +1,5 @@
 #!/bin/bash
-# Copyright 2015, Google Inc.
+# Copyright 2015-2016, Google Inc.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -28,6 +28,8 @@
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
+set -e
+
 # directories to run against
 DIRS="src/core src/cpp test/core test/cpp include"
 
@@ -50,9 +52,17 @@ if [ "x$TEST" = "x" ]
 then
   echo $files | xargs $CLANG_FORMAT -i
 else
+  ok=yes
   for file in $files
   do
-    $CLANG_FORMAT $file | diff $file -
+    tmp=`mktemp`
+    $CLANG_FORMAT $file > $tmp
+    diff -u $file $tmp || ok=no
+    rm $tmp
   done
+  if [ $ok == no ]
+  then
+    false
+  fi
 fi
 
diff --git a/tools/jenkins/build_docker_and_run_tests.sh b/tools/jenkins/build_docker_and_run_tests.sh
index 4ab9909f9f..175b3d5651 100755
--- a/tools/jenkins/build_docker_and_run_tests.sh
+++ b/tools/jenkins/build_docker_and_run_tests.sh
@@ -1,5 +1,5 @@
 #!/bin/bash
-# Copyright 2015, Google Inc.
+# Copyright 2015-2016, Google Inc.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -64,6 +64,7 @@ docker run \
   -e CCACHE_DIR=/tmp/ccache \
   -e XDG_CACHE_HOME=/tmp/xdg-cache-home \
   -e THIS_IS_REALLY_NEEDED='see https://github.com/docker/docker/issues/14203 for why docker is awful' \
+  -e HOST_GIT_ROOT=$git_root \
   -i $TTY_FLAG \
   -v "$git_root:/var/local/jenkins/grpc" \
   -v /tmp/ccache:/tmp/ccache \
@@ -78,7 +79,7 @@ docker run \
 
 if [ "$XML_REPORT" != "" ]
 then
-  docker cp "$CONTAINER_NAME:/var/local/git/grpc/$XML_REPORT" $git_root
+  docker cp "$CONTAINER_NAME:/var/local/git/grpc/$XML_REPORT" $git_root || true
 fi
 
 docker cp "$CONTAINER_NAME:/var/local/git/grpc/reports.zip" $git_root || true
diff --git a/tools/jenkins/grpc_interop_python/Dockerfile b/tools/jenkins/grpc_interop_python/Dockerfile
index 6034cbf955..047604b1b7 100644
--- a/tools/jenkins/grpc_interop_python/Dockerfile
+++ b/tools/jenkins/grpc_interop_python/Dockerfile
@@ -48,6 +48,7 @@ RUN apt-get update && apt-get install -y \
   libc6-dbg \
   libc6-dev \
   libgtest-dev \
+  libssl-dev \
   libtool \
   make \
   strace \
diff --git a/tools/jenkins/grpc_interop_python/build_interop.sh b/tools/jenkins/grpc_interop_python/build_interop.sh
index 8f5bfd11e2..39c93677d8 100755
--- a/tools/jenkins/grpc_interop_python/build_interop.sh
+++ b/tools/jenkins/grpc_interop_python/build_interop.sh
@@ -43,5 +43,5 @@ make install-certs
 make
 
 # build Python interop client and server
-CONFIG=opt ./tools/run_tests/build_python.sh 2.7
+CONFIG=opt ./tools/run_tests/build_python.sh
 
diff --git a/tools/jenkins/grpc_jenkins_slave/Dockerfile b/tools/jenkins/grpc_jenkins_slave/Dockerfile
index 294175bd15..b1ac024dfb 100644
--- a/tools/jenkins/grpc_jenkins_slave/Dockerfile
+++ b/tools/jenkins/grpc_jenkins_slave/Dockerfile
@@ -1,4 +1,4 @@
-# Copyright 2015, Google Inc.
+# Copyright 2015-2016, Google Inc.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -38,7 +38,6 @@ RUN apt-get update && apt-get install -y \
   autotools-dev \
   build-essential \
   bzip2 \
-  ccache \
   curl \
   gcc \
   gcc-multilib \
@@ -62,14 +61,6 @@ RUN apt-get update && apt-get install -y \
   wget \
   zip && apt-get clean
 
-# Prepare ccache
-RUN ln -s /usr/bin/ccache /usr/local/bin/gcc
-RUN ln -s /usr/bin/ccache /usr/local/bin/g++
-RUN ln -s /usr/bin/ccache /usr/local/bin/cc
-RUN ln -s /usr/bin/ccache /usr/local/bin/c++
-RUN ln -s /usr/bin/ccache /usr/local/bin/clang
-RUN ln -s /usr/bin/ccache /usr/local/bin/clang++
-
 ##################
 # C++ dependencies
 RUN apt-get update && apt-get -y install libgflags-dev libgtest-dev libc++-dev clang
diff --git a/tools/jenkins/grpc_jenkins_slave_32bits/Dockerfile b/tools/jenkins/grpc_jenkins_slave_32bits/Dockerfile
index ad7df5b090..348a333d3e 100644
--- a/tools/jenkins/grpc_jenkins_slave_32bits/Dockerfile
+++ b/tools/jenkins/grpc_jenkins_slave_32bits/Dockerfile
@@ -1,4 +1,4 @@
-# Copyright 2015, Google Inc.
+# Copyright 2015-2016, Google Inc.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -38,7 +38,6 @@ RUN apt-get update && apt-get install -y \
   autotools-dev \
   build-essential \
   bzip2 \
-  ccache \
   curl \
   gcc \
   gcc-multilib \
@@ -62,14 +61,6 @@ RUN apt-get update && apt-get install -y \
   wget \
   zip && apt-get clean
 
-# Prepare ccache
-RUN ln -s /usr/bin/ccache /usr/local/bin/gcc
-RUN ln -s /usr/bin/ccache /usr/local/bin/g++
-RUN ln -s /usr/bin/ccache /usr/local/bin/cc
-RUN ln -s /usr/bin/ccache /usr/local/bin/c++
-RUN ln -s /usr/bin/ccache /usr/local/bin/clang
-RUN ln -s /usr/bin/ccache /usr/local/bin/clang++
-
 ##################
 # C++ dependencies
 RUN apt-get update && apt-get -y install libgflags-dev libgtest-dev libc++-dev clang
diff --git a/tools/jenkins/run_jenkins.sh b/tools/jenkins/run_jenkins.sh
index 4bb6c39a1c..84b4ea51ed 100755
--- a/tools/jenkins/run_jenkins.sh
+++ b/tools/jenkins/run_jenkins.sh
@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-# Copyright 2015, Google Inc.
+# Copyright 2015-2016, Google Inc.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -54,7 +54,7 @@ if [ "$platform" == "linux" ]
 then
   echo "building $language on Linux"
 
-  ./tools/run_tests/run_tests.py --use_docker -t -l $language -c $config -x report.xml $@ || true
+  ./tools/run_tests/run_tests.py --use_docker -t -l $language -c $config -x report.xml $@ || TESTS_FAILED="true"
 
 elif [ "$platform" == "windows" ]
 then
@@ -63,19 +63,19 @@ then
   # Prevent msbuild from picking up "platform" env variable, which would break the build
   unset platform
 
-  python tools/run_tests/run_tests.py -t -l $language -c $config -x report.xml $@ || true
+  python tools/run_tests/run_tests.py -t -l $language -c $config -x report.xml $@ || TESTS_FAILED="true"
 
 elif [ "$platform" == "macos" ]
 then
   echo "building $language on MacOS"
 
-  ./tools/run_tests/run_tests.py -t -l $language -c $config -x report.xml $@ || true
+  ./tools/run_tests/run_tests.py -t -l $language -c $config -x report.xml $@ || TESTS_FAILED="true"
 
 elif [ "$platform" == "freebsd" ]
 then
   echo "building $language on FreeBSD"
 
-  MAKE=gmake ./tools/run_tests/run_tests.py -t -l $language -c $config -x report.xml $@ || true
+  MAKE=gmake ./tools/run_tests/run_tests.py -t -l $language -c $config -x report.xml $@ || TESTS_FAILED="true"
 
 else
   echo "Unknown platform $platform"
@@ -87,3 +87,8 @@ then
   mkdir -p reports
   echo 'No reports generated.' > reports/index.html
 fi
+
+if [ "$TESTS_FAILED" != "" ]
+then
+  exit 1
+fi
diff --git a/tools/run_tests/build_python.sh b/tools/run_tests/build_python.sh
index 57080ce934..e0fcbb602d 100755
--- a/tools/run_tests/build_python.sh
+++ b/tools/run_tests/build_python.sh
@@ -1,5 +1,5 @@
 #!/bin/bash
-# Copyright 2015, Google Inc.
+# Copyright 2015-2016, Google Inc.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -34,16 +34,14 @@ set -ex
 cd $(dirname $0)/../..
 
 ROOT=`pwd`
-GRPCIO=$ROOT/src/python/grpcio
 export LD_LIBRARY_PATH=$ROOT/libs/$CONFIG
 export DYLD_LIBRARY_PATH=$ROOT/libs/$CONFIG
 export PATH=$ROOT/bins/$CONFIG:$ROOT/bins/$CONFIG/protobuf:$PATH
-export CFLAGS="-I$ROOT/include -std=c89"
+export CFLAGS="-I$ROOT/include -std=gnu99"
 export LDFLAGS="-L$ROOT/libs/$CONFIG"
 export GRPC_PYTHON_BUILD_WITH_CYTHON=1
 export GRPC_PYTHON_ENABLE_CYTHON_TRACING=1
 
-cd $GRPCIO
 tox --notest
 
-$GRPCIO/.tox/py27/bin/python $GRPCIO/setup.py build
+$ROOT/.tox/py27/bin/python $ROOT/setup.py build
diff --git a/tools/run_tests/jobset.py b/tools/run_tests/jobset.py
index 48afbaf4cb..e33433daf2 100755
--- a/tools/run_tests/jobset.py
+++ b/tools/run_tests/jobset.py
@@ -1,4 +1,4 @@
-# Copyright 2015, Google Inc.
+# Copyright 2015-2016, Google Inc.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -178,7 +178,7 @@ class JobSpec(object):
 
   def __cmp__(self, other):
     return self.identity() == other.identity()
-    
+
   def __repr__(self):
     return 'JobSpec(shortname=%s, cmdline=%s)' % (self.shortname, self.cmdline)
 
@@ -191,7 +191,7 @@ class JobResult(object):
     self.num_failures = 0
     self.retries = 0
     self.message = ''
-    
+
 
 class Job(object):
   """Manages one job."""
@@ -239,9 +239,11 @@ class Job(object):
 
   def state(self, update_cache):
     """Poll current state of the job. Prints messages at completion."""
-    self._tempfile.seek(0)
-    stdout = self._tempfile.read()
-    self.result.message = stdout[-_MAX_RESULT_SIZE:]
+    def stdout(self=self):
+      self._tempfile.seek(0)
+      stdout = self._tempfile.read()
+      self.result.message = stdout[-_MAX_RESULT_SIZE:]
+      return stdout
     if self._state == _RUNNING and self._process.poll() is not None:
       elapsed = time.time() - self._start
       self.result.elapsed_time = elapsed
@@ -249,7 +251,7 @@ class Job(object):
         if self._retries < self._spec.flake_retries:
           message('FLAKE', '%s [ret=%d, pid=%d]' % (
             self._spec.shortname, self._process.returncode, self._process.pid),
-            stdout, do_newline=True)
+            stdout(), do_newline=True)
           self._retries += 1
           self.result.num_failures += 1
           self.result.retries = self._timeout_retries + self._retries
@@ -259,7 +261,7 @@ class Job(object):
           if not self._suppress_failure_message:
             message('FAILED', '%s [ret=%d, pid=%d]' % (
                 self._spec.shortname, self._process.returncode, self._process.pid),
-                stdout, do_newline=True)
+                stdout(), do_newline=True)
           self.result.state = 'FAILED'
           self.result.num_failures += 1
           self.result.returncode = self._process.returncode
@@ -273,7 +275,7 @@ class Job(object):
           update_cache.finished(self._spec.identity(), self._bin_hash)
     elif self._state == _RUNNING and time.time() - self._start > self._spec.timeout_seconds:
       if self._timeout_retries < self._spec.timeout_retries:
-        message('TIMEOUT_FLAKE', '%s [pid=%d]' % (self._spec.shortname, self._process.pid), stdout, do_newline=True)
+        message('TIMEOUT_FLAKE', '%s [pid=%d]' % (self._spec.shortname, self._process.pid), stdout(), do_newline=True)
         self._timeout_retries += 1
         self.result.num_failures += 1
         self.result.retries = self._timeout_retries + self._retries
@@ -282,7 +284,7 @@ class Job(object):
         self._process.terminate()
         self.start()
       else:
-        message('TIMEOUT', '%s [pid=%d]' % (self._spec.shortname, self._process.pid), stdout, do_newline=True)
+        message('TIMEOUT', '%s [pid=%d]' % (self._spec.shortname, self._process.pid), stdout(), do_newline=True)
         self.kill()
         self.result.state = 'TIMEOUT'
         self.result.num_failures += 1
@@ -297,7 +299,7 @@ class Job(object):
 
   def suppress_failure_message(self):
     self._suppress_failure_message = True
-    
+
 
 class Jobset(object):
   """Manages one run of jobs."""
@@ -448,4 +450,3 @@ def run(cmdlines,
       js.set_remaining(remaining)
   js.finish()
   return js.get_num_failures(), js.resultset
-
diff --git a/tools/run_tests/run_interop_tests.py b/tools/run_tests/run_interop_tests.py
index e69e9877c5..40bbe3cc3c 100755
--- a/tools/run_tests/run_interop_tests.py
+++ b/tools/run_tests/run_interop_tests.py
@@ -298,11 +298,8 @@ class PythonLanguage:
 
   def client_cmd(self, args):
     return [
-        'src/python/grpcio/.tox/py27/bin/python',
-        'src/python/grpcio/setup.py',
-        'run_interop',
-        '--client',
-        '--args=\'{}\''.format(' '.join(args))
+        'tox -einterop_client --',
+        ' '.join(args)
     ]
 
   def cloud_to_prod_env(self):
@@ -310,11 +307,8 @@ class PythonLanguage:
 
   def server_cmd(self, args):
     return [
-        'src/python/grpcio/.tox/py27/bin/python',
-        'src/python/grpcio/setup.py',
-        'run_interop',
-        '--server',
-        '--args=\'{}\''.format(' '.join(args) + ' --use_tls=true')
+        'tox -einterop_server --',
+        ' '.join(args) + ' --use_tls=true'
     ]
 
   def global_env(self):
diff --git a/tools/run_tests/run_python.sh b/tools/run_tests/run_python.sh
index 042b40485d..ffe9c12af1 100755
--- a/tools/run_tests/run_python.sh
+++ b/tools/run_tests/run_python.sh
@@ -1,5 +1,5 @@
 #!/bin/bash
-# Copyright 2015, Google Inc.
+# Copyright 2015-2016, Google Inc.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -34,7 +34,6 @@ set -ex
 cd $(dirname $0)/../..
 
 ROOT=`pwd`
-GRPCIO=$ROOT/src/python/grpcio
 export LD_LIBRARY_PATH=$ROOT/libs/$CONFIG
 export DYLD_LIBRARY_PATH=$ROOT/libs/$CONFIG
 export PATH=$ROOT/bins/$CONFIG:$ROOT/bins/$CONFIG/protobuf:$PATH
@@ -43,9 +42,8 @@ export LDFLAGS="-L$ROOT/libs/$CONFIG"
 export GRPC_PYTHON_BUILD_WITH_CYTHON=1
 export GRPC_PYTHON_ENABLE_CYTHON_TRACING=1
 
-cd $GRPCIO
 tox
 
 mkdir -p $ROOT/reports
 rm -rf $ROOT/reports/python-coverage
-(mv -T $GRPCIO/htmlcov $ROOT/reports/python-coverage) || true
+(mv -T $ROOT/htmlcov $ROOT/reports/python-coverage) || true
diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py
index a7c15d3d2d..0de20a634a 100755
--- a/tools/run_tests/run_tests.py
+++ b/tools/run_tests/run_tests.py
@@ -53,6 +53,7 @@ import jobset
 import report_utils
 import watch_dirs
 
+
 ROOT = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '../..'))
 os.chdir(ROOT)
 
@@ -685,23 +686,30 @@ if args.use_docker:
   sys.exit(0)
 
 # update submodules if necessary
-if args.update_submodules:
-  for spec in args.update_submodules:
-    spec = spec.split(':', 1)
-    if len(spec) == 1:
-      submodule = spec[0]
-      branch = 'master'
-    elif len(spec) == 2:
-      submodule = spec[0]
-      branch = spec[1]
-    cwd = 'third_party/%s' % submodule
-    def git(cmd, cwd=cwd):
-      print 'in %s: git %s' % (cwd, cmd)
-      subprocess.check_call('git %s' % cmd, cwd=cwd, shell=True)
-    git('fetch')
-    git('checkout %s' % branch)
-    git('pull origin %s' % branch)
-  subprocess.check_call('tools/buildgen/generate_projects.sh', shell=True)
+need_to_regenerate_projects = False
+for spec in args.update_submodules:
+  spec = spec.split(':', 1)
+  if len(spec) == 1:
+    submodule = spec[0]
+    branch = 'master'
+  elif len(spec) == 2:
+    submodule = spec[0]
+    branch = spec[1]
+  cwd = 'third_party/%s' % submodule
+  def git(cmd, cwd=cwd):
+    print 'in %s: git %s' % (cwd, cmd)
+    subprocess.check_call('git %s' % cmd, cwd=cwd, shell=True)
+  git('fetch')
+  git('checkout %s' % branch)
+  git('pull origin %s' % branch)
+  if os.path.exists('src/%s/gen_build_yaml.py' % submodule):
+    need_to_regenerate_projects = True
+if need_to_regenerate_projects:
+  if jobset.platform_string() == 'linux':
+    subprocess.check_call('tools/buildgen/generate_projects.sh', shell=True)
+  else:
+    print 'WARNING: may need to regenerate projects, but since we are not on'
+    print '         Linux this step is being skipped. Compilation MAY fail.'
 
 
 # grab config
@@ -718,9 +726,11 @@ if 'all' in args.language:
   lang_list = _LANGUAGES.keys()
 else:
   lang_list = args.language
-# We don't support code coverage on ObjC
-if 'gcov' in args.config and 'objc' in lang_list:
-  lang_list.remove('objc')
+# We don't support code coverage on some languages
+if 'gcov' in args.config:
+  for bad in ['objc', 'sanity', 'build']:
+    if bad in lang_list:
+      lang_list.remove(bad)
 
 languages = set(_LANGUAGES[l] for l in lang_list)
 
@@ -953,6 +963,15 @@ def _calculate_num_runs_failures(list_of_results):
   return num_runs, num_failures
 
 
+# _build_and_run results
+class BuildAndRunError(object):
+
+  BUILD = object()
+  TEST = object()
+  POST_TEST = object()
+
+
+# returns a list of things that failed (or an empty list on success)
 def _build_and_run(
     check_cancelled, newline_on_success, cache, xml_report=None, build_only=False):
   """Do one pass of building & running tests."""
@@ -961,10 +980,10 @@ def _build_and_run(
       build_steps, maxjobs=1, stop_on_failure=True,
       newline_on_success=newline_on_success, travis=args.travis)
   if num_failures:
-    return 1
-    
+    return [BuildAndRunError.BUILD]
+
   if build_only:
-    return 0
+    return []
 
   # start antagonists
   antagonists = [subprocess.Popen(['tools/run_tests/antagonist.py'])
@@ -1022,12 +1041,16 @@ def _build_and_run(
   number_failures, _ = jobset.run(
       post_tests_steps, maxjobs=1, stop_on_failure=True,
       newline_on_success=newline_on_success, travis=args.travis)
-  if num_test_failures or number_failures:
-    return 2
+
+  out = []
+  if number_failures:
+    out.append(BuildAndRunError.POST_TEST)
+  if num_test_failures:
+    out.append(BuildAndRunError.TEST)
 
   if cache: cache.save()
 
-  return 0
+  return out
 
 
 test_cache = TestCache(runs_per_test == 1)
@@ -1040,11 +1063,11 @@ if forever:
     initial_time = dw.most_recent_change()
     have_files_changed = lambda: dw.most_recent_change() != initial_time
     previous_success = success
-    success = _build_and_run(check_cancelled=have_files_changed,
-                             newline_on_success=False,
-                             cache=test_cache,
-                             build_only=args.build_only) == 0
-    if not previous_success and success:
+    errors = _build_and_run(check_cancelled=have_files_changed,
+                            newline_on_success=False,
+                            cache=test_cache,
+                            build_only=args.build_only) == 0
+    if not previous_success and not errors:
       jobset.message('SUCCESS',
                      'All tests are now passing properly',
                      do_newline=True)
@@ -1052,13 +1075,21 @@ if forever:
     while not have_files_changed():
       time.sleep(1)
 else:
-  result = _build_and_run(check_cancelled=lambda: False,
+  errors = _build_and_run(check_cancelled=lambda: False,
                           newline_on_success=args.newline_on_success,
                           cache=test_cache,
                           xml_report=args.xml_report,
                           build_only=args.build_only)
-  if result == 0:
+  if not errors:
     jobset.message('SUCCESS', 'All tests passed', do_newline=True)
   else:
     jobset.message('FAILED', 'Some tests failed', do_newline=True)
-  sys.exit(result)
+  exit_code = 0
+  if BuildAndRunError.BUILD in errors:
+    exit_code |= 1
+  if BuildAndRunError.TEST in errors and not args.travis:
+    exit_code |= 2
+  if BuildAndRunError.POST_TEST in errors:
+    exit_code |= 4
+  sys.exit(exit_code)
+
diff --git a/tools/run_tests/sources_and_headers.json b/tools/run_tests/sources_and_headers.json
index 3af1bbaccc..8d86fa3bf3 100644
--- a/tools/run_tests/sources_and_headers.json
+++ b/tools/run_tests/sources_and_headers.json
@@ -1641,9 +1641,9 @@
       "grpc++_test_config"
     ], 
     "headers": [
-      "test/cpp/util/metrics_server.h", 
-      "test/proto/metrics.grpc.pb.h", 
-      "test/proto/metrics.pb.h"
+      "src/proto/grpc/testing/metrics.grpc.pb.h", 
+      "src/proto/grpc/testing/metrics.pb.h", 
+      "test/cpp/util/metrics_server.h"
     ], 
     "language": "c++", 
     "name": "metrics_client", 
@@ -1773,12 +1773,12 @@
       "grpc_test_util"
     ], 
     "headers": [
-      "test/proto/empty.grpc.pb.h", 
-      "test/proto/empty.pb.h", 
-      "test/proto/messages.grpc.pb.h", 
-      "test/proto/messages.pb.h", 
-      "test/proto/test.grpc.pb.h", 
-      "test/proto/test.pb.h"
+      "src/proto/grpc/testing/empty.grpc.pb.h", 
+      "src/proto/grpc/testing/empty.pb.h", 
+      "src/proto/grpc/testing/messages.grpc.pb.h", 
+      "src/proto/grpc/testing/messages.pb.h", 
+      "src/proto/grpc/testing/test.grpc.pb.h", 
+      "src/proto/grpc/testing/test.pb.h"
     ], 
     "language": "c++", 
     "name": "reconnect_interop_client", 
@@ -1799,12 +1799,12 @@
       "test_tcp_server"
     ], 
     "headers": [
-      "test/proto/empty.grpc.pb.h", 
-      "test/proto/empty.pb.h", 
-      "test/proto/messages.grpc.pb.h", 
-      "test/proto/messages.pb.h", 
-      "test/proto/test.grpc.pb.h", 
-      "test/proto/test.pb.h"
+      "src/proto/grpc/testing/empty.grpc.pb.h", 
+      "src/proto/grpc/testing/empty.pb.h", 
+      "src/proto/grpc/testing/messages.grpc.pb.h", 
+      "src/proto/grpc/testing/messages.pb.h", 
+      "src/proto/grpc/testing/test.grpc.pb.h", 
+      "src/proto/grpc/testing/test.pb.h"
     ], 
     "language": "c++", 
     "name": "reconnect_interop_server", 
@@ -1935,18 +1935,18 @@
       "grpc_test_util"
     ], 
     "headers": [
+      "src/proto/grpc/testing/empty.grpc.pb.h", 
+      "src/proto/grpc/testing/empty.pb.h", 
+      "src/proto/grpc/testing/messages.grpc.pb.h", 
+      "src/proto/grpc/testing/messages.pb.h", 
+      "src/proto/grpc/testing/metrics.grpc.pb.h", 
+      "src/proto/grpc/testing/metrics.pb.h", 
+      "src/proto/grpc/testing/test.grpc.pb.h", 
+      "src/proto/grpc/testing/test.pb.h", 
       "test/cpp/interop/client_helper.h", 
       "test/cpp/interop/interop_client.h", 
       "test/cpp/interop/stress_interop_client.h", 
-      "test/cpp/util/metrics_server.h", 
-      "test/proto/empty.grpc.pb.h", 
-      "test/proto/empty.pb.h", 
-      "test/proto/messages.grpc.pb.h", 
-      "test/proto/messages.pb.h", 
-      "test/proto/metrics.grpc.pb.h", 
-      "test/proto/metrics.pb.h", 
-      "test/proto/test.grpc.pb.h", 
-      "test/proto/test.pb.h"
+      "test/cpp/util/metrics_server.h"
     ], 
     "language": "c++", 
     "name": "stress_test", 
@@ -2021,7 +2021,10 @@
       "grpc_test_util", 
       "grpc_zookeeper"
     ], 
-    "headers": [], 
+    "headers": [
+      "src/proto/grpc/testing/echo.grpc.pb.h", 
+      "src/proto/grpc/testing/echo.pb.h"
+    ], 
     "language": "c++", 
     "name": "zookeeper_test", 
     "src": [
@@ -3953,14 +3956,14 @@
       "grpc_test_util"
     ], 
     "headers": [
+      "src/proto/grpc/testing/duplicate/echo_duplicate.grpc.pb.h", 
+      "src/proto/grpc/testing/duplicate/echo_duplicate.pb.h", 
+      "src/proto/grpc/testing/echo.grpc.pb.h", 
+      "src/proto/grpc/testing/echo.pb.h", 
+      "src/proto/grpc/testing/echo_messages.grpc.pb.h", 
+      "src/proto/grpc/testing/echo_messages.pb.h", 
       "test/cpp/util/cli_call.h", 
       "test/cpp/util/create_test_channel.h", 
-      "test/cpp/util/echo.grpc.pb.h", 
-      "test/cpp/util/echo.pb.h", 
-      "test/cpp/util/echo_duplicate.grpc.pb.h", 
-      "test/cpp/util/echo_duplicate.pb.h", 
-      "test/cpp/util/messages.grpc.pb.h", 
-      "test/cpp/util/messages.pb.h", 
       "test/cpp/util/string_ref_helper.h", 
       "test/cpp/util/subprocess.h"
     ], 
@@ -4163,9 +4166,9 @@
       "grpc_test_util"
     ], 
     "headers": [
-      "test/cpp/interop/client_helper.h", 
-      "test/proto/messages.grpc.pb.h", 
-      "test/proto/messages.pb.h"
+      "src/proto/grpc/testing/messages.grpc.pb.h", 
+      "src/proto/grpc/testing/messages.pb.h", 
+      "test/cpp/interop/client_helper.h"
     ], 
     "language": "c++", 
     "name": "interop_client_helper", 
@@ -4186,13 +4189,13 @@
       "interop_client_helper"
     ], 
     "headers": [
-      "test/cpp/interop/interop_client.h", 
-      "test/proto/empty.grpc.pb.h", 
-      "test/proto/empty.pb.h", 
-      "test/proto/messages.grpc.pb.h", 
-      "test/proto/messages.pb.h", 
-      "test/proto/test.grpc.pb.h", 
-      "test/proto/test.pb.h"
+      "src/proto/grpc/testing/empty.grpc.pb.h", 
+      "src/proto/grpc/testing/empty.pb.h", 
+      "src/proto/grpc/testing/messages.grpc.pb.h", 
+      "src/proto/grpc/testing/messages.pb.h", 
+      "src/proto/grpc/testing/test.grpc.pb.h", 
+      "src/proto/grpc/testing/test.pb.h", 
+      "test/cpp/interop/interop_client.h"
     ], 
     "language": "c++", 
     "name": "interop_client_main", 
@@ -4231,12 +4234,12 @@
       "interop_server_helper"
     ], 
     "headers": [
-      "test/proto/empty.grpc.pb.h", 
-      "test/proto/empty.pb.h", 
-      "test/proto/messages.grpc.pb.h", 
-      "test/proto/messages.pb.h", 
-      "test/proto/test.grpc.pb.h", 
-      "test/proto/test.pb.h"
+      "src/proto/grpc/testing/empty.grpc.pb.h", 
+      "src/proto/grpc/testing/empty.pb.h", 
+      "src/proto/grpc/testing/messages.grpc.pb.h", 
+      "src/proto/grpc/testing/messages.pb.h", 
+      "src/proto/grpc/testing/test.grpc.pb.h", 
+      "src/proto/grpc/testing/test.pb.h"
     ], 
     "language": "c++", 
     "name": "interop_server_main", 
@@ -4251,29 +4254,29 @@
       "grpc_test_util"
     ], 
     "headers": [
+      "src/proto/grpc/testing/control.grpc.pb.h", 
+      "src/proto/grpc/testing/control.pb.h", 
+      "src/proto/grpc/testing/messages.grpc.pb.h", 
+      "src/proto/grpc/testing/messages.pb.h", 
+      "src/proto/grpc/testing/payloads.grpc.pb.h", 
+      "src/proto/grpc/testing/payloads.pb.h", 
+      "src/proto/grpc/testing/perf_db.grpc.pb.h", 
+      "src/proto/grpc/testing/perf_db.pb.h", 
+      "src/proto/grpc/testing/services.grpc.pb.h", 
+      "src/proto/grpc/testing/services.pb.h", 
+      "src/proto/grpc/testing/stats.grpc.pb.h", 
+      "src/proto/grpc/testing/stats.pb.h", 
       "test/cpp/qps/client.h", 
       "test/cpp/qps/driver.h", 
       "test/cpp/qps/histogram.h", 
       "test/cpp/qps/interarrival.h", 
-      "test/cpp/qps/perf_db.grpc.pb.h", 
-      "test/cpp/qps/perf_db.pb.h", 
       "test/cpp/qps/perf_db_client.h", 
       "test/cpp/qps/qps_worker.h", 
       "test/cpp/qps/report.h", 
       "test/cpp/qps/server.h", 
       "test/cpp/qps/stats.h", 
       "test/cpp/qps/timer.h", 
-      "test/cpp/util/benchmark_config.h", 
-      "test/proto/benchmarks/control.grpc.pb.h", 
-      "test/proto/benchmarks/control.pb.h", 
-      "test/proto/benchmarks/payloads.grpc.pb.h", 
-      "test/proto/benchmarks/payloads.pb.h", 
-      "test/proto/benchmarks/services.grpc.pb.h", 
-      "test/proto/benchmarks/services.pb.h", 
-      "test/proto/benchmarks/stats.grpc.pb.h", 
-      "test/proto/benchmarks/stats.pb.h", 
-      "test/proto/messages.grpc.pb.h", 
-      "test/proto/messages.pb.h"
+      "test/cpp/util/benchmark_config.h"
     ], 
     "language": "c++", 
     "name": "qps",