Merge remote-tracking branch 'upstream/master' into cares_buildin

13 files changed, 78 insertions, 26 deletions

diff --git a/tools/distrib/check_copyright.py b/tools/distrib/check_copyright.py
index cb0bc38238..6218e7c4d4 100755
--- a/tools/distrib/check_copyright.py
+++ b/tools/distrib/check_copyright.py
@@ -90,11 +90,26 @@ LICENSE_PREFIX = {
   'Makefile':   r'#\s*',
   'Dockerfile': r'#\s*',
   'LICENSE':    '',
+  'BUILD':      r'#\s*',
 }
 
-KNOWN_BAD = set([
+_EXEMPT = frozenset((
+  # Generated protocol compiler output.
+  'examples/python/helloworld/helloworld_pb2.py',
+  'examples/python/helloworld/helloworld_pb2_grpc.py',
+  'examples/python/multiplex/helloworld_pb2.py',
+  'examples/python/multiplex/helloworld_pb2_grpc.py',
+  'examples/python/multiplex/route_guide_pb2.py',
+  'examples/python/multiplex/route_guide_pb2_grpc.py',
+  'examples/python/route_guide/route_guide_pb2.py',
+  'examples/python/route_guide/route_guide_pb2_grpc.py',
+
+  'src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h',
+  'src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.c',
+
+  # An older file originally from outside gRPC.
   'src/php/tests/bootstrap.php',
-])
+))
 
 
 RE_YEAR = r'Copyright (?P<first_year>[0-9]+\-)?(?P<last_year>[0-9]+), Google Inc\.'
@@ -142,7 +157,8 @@ except subprocess.CalledProcessError:
   sys.exit(0)
 
 for filename in filename_list:
-  if filename in KNOWN_BAD: continue
+  if filename in _EXEMPT:
+    continue
   ext = os.path.splitext(filename)[1]
   base = os.path.basename(filename)
   if ext in RE_LICENSE:
diff --git a/tools/distrib/check_nanopb_output.sh b/tools/distrib/check_nanopb_output.sh
index c0707051a6..eb64e23daf 100755
--- a/tools/distrib/check_nanopb_output.sh
+++ b/tools/distrib/check_nanopb_output.sh
@@ -37,7 +37,7 @@ readonly PROTOBUF_INSTALL_PREFIX="$(mktemp -d)"
 pushd third_party/protobuf
 ./autogen.sh
 ./configure --prefix="$PROTOBUF_INSTALL_PREFIX"
-make
+make -j 8
 make install
 #ldconfig
 popd
@@ -51,7 +51,7 @@ fi
 # stack up and change to nanopb's proto generator directory
 pushd third_party/nanopb/generator/proto
 export PATH="$PROTOC_BIN_PATH:$PATH"
-make
+make -j 8
 # back to the root directory
 popd
 
diff --git a/tools/distrib/clang_format_code.sh b/tools/distrib/clang_format_code.sh
index 858e074898..13e018709f 100755
--- a/tools/distrib/clang_format_code.sh
+++ b/tools/distrib/clang_format_code.sh
@@ -32,9 +32,15 @@ set -ex
 
 # change to root directory
 cd $(dirname $0)/../..
+REPO_ROOT=$(pwd)
 
-# build clang-format docker image
-docker build -t grpc_clang_format tools/dockerfile/grpc_clang_format
+if [ "$CLANG_FORMAT_SKIP_DOCKER" == "" ]
+then
+  # build clang-format docker image
+  docker build -t grpc_clang_format tools/dockerfile/grpc_clang_format
 
-# run clang-format against the checked out codebase
-docker run -e TEST=$TEST -e CHANGED_FILES="$CHANGED_FILES" --rm=true -v ${HOST_GIT_ROOT:-`pwd`}:/local-code -t grpc_clang_format /clang_format_all_the_things.sh
+  # run clang-format against the checked out codebase
+  docker run -e TEST=$TEST -e CHANGED_FILES="$CHANGED_FILES" -e CLANG_FORMAT_ROOT="/local-code" --rm=true -v "${REPO_ROOT}":/local-code -t grpc_clang_format /clang_format_all_the_things.sh
+else
+  CLANG_FORMAT_ROOT="${REPO_ROOT}" tools/dockerfile/grpc_clang_format/clang_format_all_the_things.sh
+fi
diff --git a/tools/distrib/python/grpcio_tools/grpc/__init__.py b/tools/distrib/pyformat_code.sh
index 70ac5edd48..e3ebf1c490 100644..100755
--- a/tools/distrib/python/grpcio_tools/grpc/__init__.py
+++ b/tools/distrib/pyformat_code.sh
@@ -1,4 +1,5 @@
-# Copyright 2016, Google Inc.
+#!/bin/bash
+# Copyright 2015, Google Inc.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -27,4 +28,33 @@
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-__import__('pkg_resources').declare_namespace(__name__)
+set -ex
+
+# change to root directory
+cd $(dirname $0)/../..
+
+DIRS=src/python
+EXCLUSIONS='src/python/grpcio/grpc_*.py src/python/grpcio_health_checking/grpc_*.py src/python/grpcio_reflection/grpc_*.py src/python/grpcio_tests/grpc_*.py'
+
+VIRTUALENV=python_format_venv
+
+virtualenv $VIRTUALENV
+PYTHON=`realpath $VIRTUALENV/bin/python`
+$PYTHON -m pip install --upgrade futures yapf
+
+exclusion_args=""
+for exclusion in $EXCLUSIONS; do
+  exclusion_args="$exclusion_args --exclude $exclusion"
+done
+
+script_result=0
+for dir in $DIRS; do
+  tempdir=`mktemp -d`
+  cp -RT $dir $tempdir
+  $PYTHON -m yapf -i -r -p $exclusion_args $dir
+  if ! diff -rq $dir $tempdir; then
+    script_result=1
+  fi
+  rm -rf $tempdir
+done
+exit $script_result
diff --git a/tools/distrib/python/docgen.py b/tools/distrib/python/docgen.py
index 622317920d..38ffcd6e0e 100755
--- a/tools/distrib/python/docgen.py
+++ b/tools/distrib/python/docgen.py
@@ -94,6 +94,7 @@ if args.submit:
   # specified repository, edit it, and push it. It's up to the user to then go
   # onto GitHub and make a PR against grpc/grpc:gh-pages.
   repo_parent_dir = tempfile.mkdtemp()
+  print('Documentation parent directory: {}'.format(repo_parent_dir))
   repo_dir = os.path.join(repo_parent_dir, 'grpc')
   python_doc_dir = os.path.join(repo_dir, 'python')
   doc_branch = args.doc_branch
diff --git a/tools/distrib/python/grpcio_tools/MANIFEST.in b/tools/distrib/python/grpcio_tools/MANIFEST.in
index 7712834d64..11ce367747 100644
--- a/tools/distrib/python/grpcio_tools/MANIFEST.in
+++ b/tools/distrib/python/grpcio_tools/MANIFEST.in
@@ -2,6 +2,6 @@ include grpc_version.py
 include protoc_deps.py
 include protoc_lib_deps.py
 include README.rst
-graft grpc
+graft grpc_tools
 graft grpc_root
 graft third_party
diff --git a/tools/distrib/python/grpcio_tools/grpc/tools/__init__.py b/tools/distrib/python/grpcio_tools/grpc_tools/__init__.py
index d5ad73a74a..d5ad73a74a 100644
--- a/tools/distrib/python/grpcio_tools/grpc/tools/__init__.py
+++ b/tools/distrib/python/grpcio_tools/grpc_tools/__init__.py
diff --git a/tools/distrib/python/grpcio_tools/grpc/tools/_protoc_compiler.pyx b/tools/distrib/python/grpcio_tools/grpc_tools/_protoc_compiler.pyx
index a6530127c0..81034fad5e 100644
--- a/tools/distrib/python/grpcio_tools/grpc/tools/_protoc_compiler.pyx
+++ b/tools/distrib/python/grpcio_tools/grpc_tools/_protoc_compiler.pyx
@@ -29,7 +29,7 @@
 
 from libc cimport stdlib
 
-cdef extern from "grpc/tools/main.h":
+cdef extern from "grpc_tools/main.h":
   int protoc_main(int argc, char *argv[])
 
 def run_main(list args not None):
diff --git a/tools/distrib/python/grpcio_tools/grpc/tools/command.py b/tools/distrib/python/grpcio_tools/grpc_tools/command.py
index 424fd90411..31b3331a66 100644
--- a/tools/distrib/python/grpcio_tools/grpc/tools/command.py
+++ b/tools/distrib/python/grpcio_tools/grpc_tools/command.py
@@ -33,7 +33,7 @@ import sys
 
 import setuptools
 
-from grpc.tools import protoc
+from grpc_tools import protoc
 
 
 def build_package_protos(package_root):
@@ -45,11 +45,11 @@ def build_package_protos(package_root):
         proto_files.append(os.path.abspath(os.path.join(root, filename)))
 
   well_known_protos_include = pkg_resources.resource_filename(
-      'grpc.tools', '_proto')
+      'grpc_tools', '_proto')
 
   for proto_file in proto_files:
     command = [
-        'grpc.tools.protoc',
+        'grpc_tools.protoc',
         '--proto_path={}'.format(inclusion_root),
         '--proto_path={}'.format(well_known_protos_include),
         '--python_out={}'.format(inclusion_root),
diff --git a/tools/distrib/python/grpcio_tools/grpc/tools/main.cc b/tools/distrib/python/grpcio_tools/grpc_tools/main.cc
index 8391839513..0c2fa3180a 100644
--- a/tools/distrib/python/grpcio_tools/grpc/tools/main.cc
+++ b/tools/distrib/python/grpcio_tools/grpc_tools/main.cc
@@ -32,7 +32,7 @@
 
 #include "src/compiler/python_generator.h"
 
-#include "grpc/tools/main.h"
+#include "grpc_tools/main.h"
 
 int protoc_main(int argc, char* argv[]) {
   google::protobuf::compiler::CommandLineInterface cli;
diff --git a/tools/distrib/python/grpcio_tools/grpc/tools/main.h b/tools/distrib/python/grpcio_tools/grpc_tools/main.h
index ea2860ff02..ea2860ff02 100644
--- a/tools/distrib/python/grpcio_tools/grpc/tools/main.h
+++ b/tools/distrib/python/grpcio_tools/grpc_tools/main.h
diff --git a/tools/distrib/python/grpcio_tools/grpc/tools/protoc.py b/tools/distrib/python/grpcio_tools/grpc_tools/protoc.py
index e1256a7dd9..63fddb2f06 100644
--- a/tools/distrib/python/grpcio_tools/grpc/tools/protoc.py
+++ b/tools/distrib/python/grpcio_tools/grpc_tools/protoc.py
@@ -32,7 +32,7 @@
 import pkg_resources
 import sys
 
-from grpc.tools import _protoc_compiler
+from grpc_tools import _protoc_compiler
 
 def main(command_arguments):
   """Run the protocol buffer compiler with the given command-line arguments.
@@ -45,5 +45,5 @@ def main(command_arguments):
   return _protoc_compiler.run_main(command_arguments)
 
 if __name__ == '__main__':
-  proto_include = pkg_resources.resource_filename('grpc.tools', '_proto')
+  proto_include = pkg_resources.resource_filename('grpc_tools', '_proto')
   sys.exit(main(sys.argv + ['-I{}'.format(proto_include)]))
diff --git a/tools/distrib/python/grpcio_tools/setup.py b/tools/distrib/python/grpcio_tools/setup.py
index a07a586fb2..502d7ef27b 100644
--- a/tools/distrib/python/grpcio_tools/setup.py
+++ b/tools/distrib/python/grpcio_tools/setup.py
@@ -108,7 +108,7 @@ PROTO_FILES = [
 CC_INCLUDE = os.path.normpath(protoc_lib_deps.CC_INCLUDE)
 PROTO_INCLUDE = os.path.normpath(protoc_lib_deps.PROTO_INCLUDE)
 
-GRPC_PYTHON_TOOLS_PACKAGE = 'grpc.tools'
+GRPC_PYTHON_TOOLS_PACKAGE = 'grpc_tools'
 GRPC_PYTHON_PROTO_RESOURCES_NAME = '_proto'
 
 DEFINE_MACROS = ()
@@ -154,16 +154,16 @@ def package_data():
 
 def extension_modules():
   if BUILD_WITH_CYTHON:
-    plugin_sources = [os.path.join('grpc', 'tools', '_protoc_compiler.pyx')]
+    plugin_sources = [os.path.join('grpc_tools', '_protoc_compiler.pyx')]
   else:
-    plugin_sources = [os.path.join('grpc', 'tools', '_protoc_compiler.cpp')]
+    plugin_sources = [os.path.join('grpc_tools', '_protoc_compiler.cpp')]
   plugin_sources += [
-    os.path.join('grpc', 'tools', 'main.cc'),
+    os.path.join('grpc_tools', 'main.cc'),
     os.path.join('grpc_root', 'src', 'compiler', 'python_generator.cc')] + [
     os.path.join(CC_INCLUDE, cc_file)
     for cc_file in CC_FILES]
   plugin_ext = extension.Extension(
-      name='grpc.tools._protoc_compiler',
+      name='grpc_tools._protoc_compiler',
       sources=plugin_sources,
       include_dirs=[
           '.',
@@ -184,12 +184,11 @@ def extension_modules():
     return extensions
 
 setuptools.setup(
-  name='grpcio_tools',
+  name='grpcio-tools',
   version=grpc_version.VERSION,
   license='3-clause BSD',
   ext_modules=extension_modules(),
   packages=setuptools.find_packages('.'),
-  namespace_packages=['grpc'],
   install_requires=[
     'protobuf>=3.0.0',
     'grpcio>={version}'.format(version=grpc_version.VERSION),