Merge pull request #17303 from lidizheng/sanity-python-3

Run pylint test in Python 3

20 files changed, 46 insertions, 32 deletions

diff --git a/tools/distrib/pylint_code.sh b/tools/distrib/pylint_code.sh
index 8a5f7af6c6..0050777503 100755
--- a/tools/distrib/pylint_code.sh
+++ b/tools/distrib/pylint_code.sh
@@ -32,12 +32,12 @@ TEST_DIRS=(
 )
 
 VIRTUALENV=python_pylint_venv
-python -m virtualenv $VIRTUALENV
+python3 -m virtualenv $VIRTUALENV
 
 PYTHON=$VIRTUALENV/bin/python
 
-$PYTHON -m pip install --upgrade pip==10.0.1
-$PYTHON -m pip install pylint==1.9.2
+$PYTHON -m pip install --upgrade pip==18.1
+$PYTHON -m pip install --upgrade pylint==2.2.2
 
 EXIT=0
 for dir in "${DIRS[@]}"; do
diff --git a/tools/dockerfile/test/sanity/Dockerfile b/tools/dockerfile/test/sanity/Dockerfile
index e6bdb4ee03..aeee02a50f 100644
--- a/tools/dockerfile/test/sanity/Dockerfile
+++ b/tools/dockerfile/test/sanity/Dockerfile
@@ -12,8 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM debian:jessie
-
+FROM debian:stretch
+  
 # Install Git and basic packages.
 RUN apt-get update && apt-get install -y \
   autoconf \
@@ -53,20 +53,19 @@ RUN apt-get update && apt-get install -y time && apt-get clean
 RUN apt-get update && apt-get install -y python-pip && apt-get clean
 RUN pip install --upgrade google-api-python-client oauth2client
 
-#====================
-# Python dependencies
+# Install Python 2.7
+RUN apt-get update && apt-get install -y python2.7 python-all-dev
+RUN curl https://bootstrap.pypa.io/get-pip.py | python2.7
 
-# Install dependencies
+# Add Debian 'testing' repository
+RUN echo 'deb http://ftp.de.debian.org/debian testing main' >> /etc/apt/sources.list
+RUN echo 'APT::Default-Release "stable";' | tee -a /etc/apt/apt.conf.d/00local
 
-RUN apt-get update && apt-get install -y \
-    python-all-dev \
-    python3-all-dev \
-    python-pip
 
-# Install Python packages from PyPI
-RUN pip install --upgrade pip==10.0.1
-RUN pip install virtualenv
-RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.5.2.post1 six==1.10.0 twisted==17.5.0
+RUN mkdir /var/local/jenkins
+
+# Define the default command.
+CMD ["bash"]
 
 #=================
 # C++ dependencies
@@ -74,16 +73,18 @@ RUN apt-get update && apt-get -y install libgflags-dev libgtest-dev libc++-dev c
 
 #========================
 # Sanity test dependencies
+RUN apt-get update && apt-get -t testing install -y python3.7 python3-all-dev
+RUN curl https://bootstrap.pypa.io/get-pip.py | python3.7
+# Make Python 3.7 the default Python 3 version
+RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.7 1
 RUN apt-get update && apt-get install -y \
-      python-pip \
       autoconf \
       automake \
       libtool \
       curl \
-      python-virtualenv \
-      python-lxml \
       shellcheck
-RUN pip install simplejson mako
+RUN python2 -m pip install simplejson mako virtualenv lxml
+RUN python3 -m pip install simplejson mako virtualenv lxml
 
 RUN apt-get update && apt-get -y install wget xz-utils
 RUN wget http://releases.llvm.org/5.0.0/clang+llvm-5.0.0-linux-x86_64-ubuntu14.04.tar.xz
@@ -94,8 +95,5 @@ RUN ln -s /clang+llvm-5.0.0-linux-x86_64-ubuntu14.04/bin/clang-tidy /usr/local/b
 ENV CLANG_TIDY=clang-tidy
 
 
-RUN mkdir /var/local/jenkins
-
-
 # Define the default command.
 CMD ["bash"]
diff --git a/tools/run_tests/artifacts/build_artifact_protoc.sh b/tools/run_tests/artifacts/build_artifact_protoc.sh
index 6d433f2dad..a5b6e2f348 100755
--- a/tools/run_tests/artifacts/build_artifact_protoc.sh
+++ b/tools/run_tests/artifacts/build_artifact_protoc.sh
@@ -14,6 +14,7 @@
 # limitations under the License.
 
 # Use devtoolset environment that has GCC 4.8 before set -ex
+# shellcheck disable=SC1091
 source scl_source enable devtoolset-2
 
 set -ex
diff --git a/tools/run_tests/artifacts/build_artifact_ruby.sh b/tools/run_tests/artifacts/build_artifact_ruby.sh
index 5ab4cf21b4..c910374376 100755
--- a/tools/run_tests/artifacts/build_artifact_ruby.sh
+++ b/tools/run_tests/artifacts/build_artifact_ruby.sh
@@ -18,7 +18,9 @@ SYSTEM=$(uname | cut -f 1 -d_)
 
 cd "$(dirname "$0")/../../.."
 set +ex
+# shellcheck disable=SC1091
 [[ -s /etc/profile.d/rvm.sh ]] && . /etc/profile.d/rvm.sh
+# shellcheck disable=SC1090
 [[ -s "$HOME/.rvm/scripts/rvm" ]] && source "$HOME/.rvm/scripts/rvm"
 set -ex
 
diff --git a/tools/run_tests/artifacts/run_in_workspace.sh b/tools/run_tests/artifacts/run_in_workspace.sh
index 20181e077c..f4719b0a4a 100755
--- a/tools/run_tests/artifacts/run_in_workspace.sh
+++ b/tools/run_tests/artifacts/run_in_workspace.sh
@@ -19,7 +19,8 @@
 set -ex
 
 cd "$(dirname "$0")/../../.."
-export repo_root=$(pwd)
+repo_root=$(pwd)
+export repo_root
 
 # TODO: fix file to pass shellcheck
 
diff --git a/tools/run_tests/dockerize/build_and_run_docker.sh b/tools/run_tests/dockerize/build_and_run_docker.sh
index 3f01fbc7b7..9d0efc8b40 100755
--- a/tools/run_tests/dockerize/build_and_run_docker.sh
+++ b/tools/run_tests/dockerize/build_and_run_docker.sh
@@ -20,7 +20,7 @@ set -ex
 
 cd "$(dirname "$0")/../../.."
 git_root=$(pwd)
-cd -
+cd - # shellcheck disable=SC2103
 
 # Inputs
 # DOCKERFILE_DIR - Directory in which Dockerfile file is located.
diff --git a/tools/run_tests/dockerize/build_docker_and_run_tests.sh b/tools/run_tests/dockerize/build_docker_and_run_tests.sh
index 1741b3268b..a63e1be96d 100755
--- a/tools/run_tests/dockerize/build_docker_and_run_tests.sh
+++ b/tools/run_tests/dockerize/build_docker_and_run_tests.sh
@@ -20,7 +20,7 @@ set -ex
 
 cd "$(dirname "$0")/../../.."
 git_root=$(pwd)
-cd -
+cd - # shellcheck disable=SC2103
 
 # Inputs
 # DOCKERFILE_DIR - Directory in which Dockerfile file is located.
@@ -48,7 +48,7 @@ docker_instance_git_root=/var/local/jenkins/grpc
 # Run tests inside docker
 DOCKER_EXIT_CODE=0
 # TODO: silence complaint about $TTY_FLAG expansion in some other way
-# shellcheck disable=SC2086
+# shellcheck disable=SC2086,SC2154
 docker run \
   --cap-add SYS_PTRACE \
   -e "RUN_TESTS_COMMAND=$RUN_TESTS_COMMAND" \
diff --git a/tools/run_tests/dockerize/build_interop_image.sh b/tools/run_tests/dockerize/build_interop_image.sh
index 126dd4065e..0f88787639 100755
--- a/tools/run_tests/dockerize/build_interop_image.sh
+++ b/tools/run_tests/dockerize/build_interop_image.sh
@@ -16,7 +16,7 @@
 # This script is invoked by run_interop_tests.py to build the docker image
 # for interop testing. You should never need to call this script on your own.
 
-set -x
+set -ex
 
 # Params:
 #  INTEROP_IMAGE - name of tag of the final interop image
diff --git a/tools/run_tests/dockerize/docker_run_tests.sh b/tools/run_tests/dockerize/docker_run_tests.sh
index b7686e48ba..2a3b6f53ef 100755
--- a/tools/run_tests/dockerize/docker_run_tests.sh
+++ b/tools/run_tests/dockerize/docker_run_tests.sh
@@ -18,7 +18,7 @@
 
 set -e
 
-export CONFIG=$config
+export CONFIG=${config:-opt}
 export ASAN_SYMBOLIZER_PATH=/usr/bin/llvm-symbolizer
 export PATH=$PATH:/usr/bin/llvm-symbolizer
 
diff --git a/tools/run_tests/helper_scripts/run_grpc-node.sh b/tools/run_tests/helper_scripts/run_grpc-node.sh
index 747aae7fd5..d14753a4d5 100755
--- a/tools/run_tests/helper_scripts/run_grpc-node.sh
+++ b/tools/run_tests/helper_scripts/run_grpc-node.sh
@@ -16,6 +16,8 @@
 # This script runs grpc/grpc-node tests with their grpc submodule updated
 # to this reference
 
+set -ex
+
 # cd to gRPC root directory
 cd "$(dirname "$0")/../../.."
 
diff --git a/tools/run_tests/helper_scripts/run_tests_in_workspace.sh b/tools/run_tests/helper_scripts/run_tests_in_workspace.sh
index 790c041881..fa7a7aac0a 100755
--- a/tools/run_tests/helper_scripts/run_tests_in_workspace.sh
+++ b/tools/run_tests/helper_scripts/run_tests_in_workspace.sh
@@ -20,7 +20,8 @@
 set -ex
 
 cd "$(dirname "$0")/../../.."
-export repo_root="$(pwd)"
+repo_root="$(pwd)"
+export repo_root
 
 rm -rf "${WORKSPACE_NAME}"
 git clone . "${WORKSPACE_NAME}"
diff --git a/tools/run_tests/interop/with_nvm.sh b/tools/run_tests/interop/with_nvm.sh
index 887f9f6a9f..55f4b2b875 100755
--- a/tools/run_tests/interop/with_nvm.sh
+++ b/tools/run_tests/interop/with_nvm.sh
@@ -15,5 +15,6 @@
 # limitations under the License.
 
 # Makes sure NVM is loaded before executing the command passed as an argument
+# shellcheck disable=SC1090
 source ~/.nvm/nvm.sh
 "$@"
diff --git a/tools/run_tests/interop/with_rvm.sh b/tools/run_tests/interop/with_rvm.sh
index 41e6efcb56..82bce9da12 100755
--- a/tools/run_tests/interop/with_rvm.sh
+++ b/tools/run_tests/interop/with_rvm.sh
@@ -15,5 +15,6 @@
 # limitations under the License.
 
 # Makes sure RVM is loaded before executing the command passed as an argument
+# shellcheck disable=SC1091
 source /usr/local/rvm/scripts/rvm
 "$@"
diff --git a/tools/run_tests/performance/build_performance.sh b/tools/run_tests/performance/build_performance.sh
index ab6bffdc34..6c22e2d37d 100755
--- a/tools/run_tests/performance/build_performance.sh
+++ b/tools/run_tests/performance/build_performance.sh
@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# shellcheck disable=SC1090
 source ~/.rvm/scripts/rvm
 set -ex
 
diff --git a/tools/run_tests/performance/build_performance_go.sh b/tools/run_tests/performance/build_performance_go.sh
index 812728d4ce..3aa203a6ee 100755
--- a/tools/run_tests/performance/build_performance_go.sh
+++ b/tools/run_tests/performance/build_performance_go.sh
@@ -17,7 +17,8 @@ set -ex
 
 cd "$(dirname "$0")/../../.."
 
-export GOPATH=$(pwd)/../gopath
+GOPATH=$(pwd)/../gopath
+export GOPATH
 
 # Get grpc-go and the dependencies but get rid of the upstream/master version
 go get google.golang.org/grpc
diff --git a/tools/run_tests/performance/build_performance_node.sh b/tools/run_tests/performance/build_performance_node.sh
index 12e0872264..b5b5d9a1a2 100755
--- a/tools/run_tests/performance/build_performance_node.sh
+++ b/tools/run_tests/performance/build_performance_node.sh
@@ -15,6 +15,7 @@
 
 set +ex
 
+# shellcheck disable=SC1090
 . "$HOME/.nvm/nvm.sh"
 
 nvm install 10
diff --git a/tools/run_tests/performance/run_worker_go.sh b/tools/run_tests/performance/run_worker_go.sh
index f8e821a265..1127f4f25a 100755
--- a/tools/run_tests/performance/run_worker_go.sh
+++ b/tools/run_tests/performance/run_worker_go.sh
@@ -17,6 +17,7 @@ set -ex
 
 cd "$(dirname "$0")/../../.."
 
-export GOPATH=$(pwd)/../gopath
+GOPATH=$(pwd)/../gopath
+export GOPATH
 
 "${GOPATH}/bin/worker" "$@"
diff --git a/tools/run_tests/performance/run_worker_node.sh b/tools/run_tests/performance/run_worker_node.sh
index 3e5dd18edb..658cd50811 100755
--- a/tools/run_tests/performance/run_worker_node.sh
+++ b/tools/run_tests/performance/run_worker_node.sh
@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# shellcheck disable=SC1090
 . "$HOME/.nvm/nvm.sh"
 
 nvm use 10
diff --git a/tools/run_tests/performance/run_worker_php.sh b/tools/run_tests/performance/run_worker_php.sh
index 2fe2493e60..97292a9978 100755
--- a/tools/run_tests/performance/run_worker_php.sh
+++ b/tools/run_tests/performance/run_worker_php.sh
@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# shellcheck disable=SC1090
 source ~/.rvm/scripts/rvm
 set -ex
 
diff --git a/tools/run_tests/performance/run_worker_ruby.sh b/tools/run_tests/performance/run_worker_ruby.sh
index 729c5cec97..ed44846336 100755
--- a/tools/run_tests/performance/run_worker_ruby.sh
+++ b/tools/run_tests/performance/run_worker_ruby.sh
@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# shellcheck disable=SC1090
 source ~/.rvm/scripts/rvm
 set -ex