fixed local cloning of grpc/grpc submodules on docker

21 files changed, 110 insertions, 21 deletions

diff --git a/test/distrib/cpp/run_distrib_test.sh b/test/distrib/cpp/run_distrib_test.sh
index bc84b84b8f..cd4158eb5d 100755
--- a/test/distrib/cpp/run_distrib_test.sh
+++ b/test/distrib/cpp/run_distrib_test.sh
@@ -30,9 +30,13 @@
 
 set -ex
 
-git clone --recursive $EXTERNAL_GIT_ROOT
+git clone $EXTERNAL_GIT_ROOT
 cd grpc
 
+# clone submodules
+git submodule | awk -v EXTERNAL_GIT_ROOT=$EXTERNAL_GIT_ROOT '{ system("git \
+submodule update --init --reference " EXTERNAL_GIT_ROOT$2 " " $2) }'
+
 cd third_party/protobuf && ./autogen.sh && \
 ./configure && make -j4 && make check && make install && ldconfig
 
diff --git a/tools/dockerfile/grpc_check_generated_pb_files/check_pb_files.sh b/tools/dockerfile/grpc_check_generated_pb_files/check_pb_files.sh
index 62e41755ec..fa9fd8fc72 100755
--- a/tools/dockerfile/grpc_check_generated_pb_files/check_pb_files.sh
+++ b/tools/dockerfile/grpc_check_generated_pb_files/check_pb_files.sh
@@ -31,10 +31,14 @@
 set -e
 
 mkdir -p /var/local/git
-git clone --recursive /var/local/jenkins/grpc /var/local/git/grpc
+git clone /var/local/jenkins/grpc /var/local/git/grpc
 
 cd /var/local/git/grpc
 
+# clone gRPC submodules
+git submodule | awk '{ system("git submodule update --init --reference \
+./../../jenkins/grpc" $2 " " $2) }'
+
 # build grpc cpp plugin for generating grpc pb files
 make grpc_cpp_plugin
 
diff --git a/tools/dockerfile/interoptest/grpc_interop_csharp/build_interop.sh b/tools/dockerfile/interoptest/grpc_interop_csharp/build_interop.sh
index fd5436c44f..4ec41b9c30 100755
--- a/tools/dockerfile/interoptest/grpc_interop_csharp/build_interop.sh
+++ b/tools/dockerfile/interoptest/grpc_interop_csharp/build_interop.sh
@@ -32,12 +32,16 @@
 set -e
 
 mkdir -p /var/local/git
-git clone --recursive /var/local/jenkins/grpc /var/local/git/grpc
+git clone /var/local/jenkins/grpc /var/local/git/grpc
 
 # copy service account keys if available
 cp -r /var/local/jenkins/service_account $HOME || true
 
 cd /var/local/git/grpc
 
+# clone gRPC submodules
+git submodule | awk '{ system("git submodule update --init --reference \
+./../../jenkins/grpc" $2 " " $2) }'
+
 # build C# interop client & server
 tools/run_tests/run_tests.py -l csharp -c dbg --build_only
diff --git a/tools/dockerfile/interoptest/grpc_interop_csharpcoreclr/build_interop.sh b/tools/dockerfile/interoptest/grpc_interop_csharpcoreclr/build_interop.sh
index 77cd65cce3..073c4625bc 100755
--- a/tools/dockerfile/interoptest/grpc_interop_csharpcoreclr/build_interop.sh
+++ b/tools/dockerfile/interoptest/grpc_interop_csharpcoreclr/build_interop.sh
@@ -32,12 +32,16 @@
 set -e
 
 mkdir -p /var/local/git
-git clone --recursive /var/local/jenkins/grpc /var/local/git/grpc
+git clone /var/local/jenkins/grpc /var/local/git/grpc
 
 # copy service account keys if available
 cp -r /var/local/jenkins/service_account $HOME || true
 
 cd /var/local/git/grpc
 
+# clone gRPC submodules
+git submodule | awk '{ system("git submodule update --init --reference \
+./../../jenkins/grpc" $2 " " $2) }'
+
 # build C# interop client & server
 tools/run_tests/run_tests.py -l csharp -c dbg --compiler coreclr --build_only
diff --git a/tools/dockerfile/interoptest/grpc_interop_cxx/build_interop.sh b/tools/dockerfile/interoptest/grpc_interop_cxx/build_interop.sh
index 1c0828d23a..5de62a41c0 100755
--- a/tools/dockerfile/interoptest/grpc_interop_cxx/build_interop.sh
+++ b/tools/dockerfile/interoptest/grpc_interop_cxx/build_interop.sh
@@ -32,13 +32,17 @@
 set -e
 
 mkdir -p /var/local/git
-git clone --recursive /var/local/jenkins/grpc /var/local/git/grpc
+git clone /var/local/jenkins/grpc /var/local/git/grpc
 
 # copy service account keys if available
 cp -r /var/local/jenkins/service_account $HOME || true
 
 cd /var/local/git/grpc
 
+# clone gRPC submodules
+git submodule | awk '{ system("git submodule update --init --reference \
+./../../jenkins/grpc" $2 " " $2) }'
+
 make install-certs
 
 # build C++ interop client & server
diff --git a/tools/dockerfile/interoptest/grpc_interop_http2/build_interop.sh b/tools/dockerfile/interoptest/grpc_interop_http2/build_interop.sh
index 46ddaf929a..64418bc63d 100755
--- a/tools/dockerfile/interoptest/grpc_interop_http2/build_interop.sh
+++ b/tools/dockerfile/interoptest/grpc_interop_http2/build_interop.sh
@@ -32,7 +32,11 @@
 set -e
 
 mkdir -p /var/local/git
-git clone --recursive /var/local/jenkins/grpc /var/local/git/grpc
+git clone /var/local/jenkins/grpc /var/local/git/grpc
+
+# clone gRPC submodules
+(cd /var/local/git/grpc/ && exec git submodule | awk '{ system("git submodule \
+update --init --reference ./../../jenkins/grpc" $2 " " $2) }')
 
 # copy service account keys if available
 cp -r /var/local/jenkins/service_account $HOME || true
diff --git a/tools/dockerfile/interoptest/grpc_interop_node/build_interop.sh b/tools/dockerfile/interoptest/grpc_interop_node/build_interop.sh
index 976f55d9ab..85750ed91c 100755
--- a/tools/dockerfile/interoptest/grpc_interop_node/build_interop.sh
+++ b/tools/dockerfile/interoptest/grpc_interop_node/build_interop.sh
@@ -32,13 +32,17 @@
 set -e
 
 mkdir -p /var/local/git
-git clone --recursive /var/local/jenkins/grpc /var/local/git/grpc
+git clone /var/local/jenkins/grpc /var/local/git/grpc
 
 # copy service account keys if available
 cp -r /var/local/jenkins/service_account $HOME || true
 
 cd /var/local/git/grpc
 
+# clone gRPC submodules
+git submodule | awk '{ system("git submodule update --init --reference \
+./../../jenkins/grpc" $2 " " $2) }'
+
 # build Node interop client & server
 npm install -g node-gyp
 npm install --unsafe-perm --build-from-source
diff --git a/tools/dockerfile/interoptest/grpc_interop_php/build_interop.sh b/tools/dockerfile/interoptest/grpc_interop_php/build_interop.sh
index a84a450221..ea14904933 100755
--- a/tools/dockerfile/interoptest/grpc_interop_php/build_interop.sh
+++ b/tools/dockerfile/interoptest/grpc_interop_php/build_interop.sh
@@ -32,12 +32,17 @@
 set -ex
 
 mkdir -p /var/local/git
-git clone --recursive /var/local/jenkins/grpc /var/local/git/grpc
+git clone /var/local/jenkins/grpc /var/local/git/grpc
 
 # copy service account keys if available
 cp -r /var/local/jenkins/service_account $HOME || true
 
 cd /var/local/git/grpc
+
+# clone gRPC submodules
+git submodule | awk '{ system("git submodule update --init --reference \
+./../../jenkins/grpc" $2 " " $2) }'
+
 rvm --default use ruby-2.1
 
 # gRPC core and protobuf need to be installed
diff --git a/tools/dockerfile/interoptest/grpc_interop_php7/build_interop.sh b/tools/dockerfile/interoptest/grpc_interop_php7/build_interop.sh
index 261dded282..0e045631c2 100755
--- a/tools/dockerfile/interoptest/grpc_interop_php7/build_interop.sh
+++ b/tools/dockerfile/interoptest/grpc_interop_php7/build_interop.sh
@@ -32,12 +32,17 @@
 set -ex
 
 mkdir -p /var/local/git
-git clone --recursive /var/local/jenkins/grpc /var/local/git/grpc
+git clone /var/local/jenkins/grpc /var/local/git/grpc
 
 # copy service account keys if available
 cp -r /var/local/jenkins/service_account $HOME || true
 
 cd /var/local/git/grpc
+
+# clone gRPC submodules
+git submodule | awk '{ system("git submodule update --init --reference \
+./../../jenkins/grpc" $2 " " $2) }'
+
 rvm --default use ruby-2.1
 
 # gRPC core and protobuf need to be installed
diff --git a/tools/dockerfile/interoptest/grpc_interop_python/build_interop.sh b/tools/dockerfile/interoptest/grpc_interop_python/build_interop.sh
index f29c59da8e..55d1b6f915 100755
--- a/tools/dockerfile/interoptest/grpc_interop_python/build_interop.sh
+++ b/tools/dockerfile/interoptest/grpc_interop_python/build_interop.sh
@@ -32,11 +32,15 @@
 set -e
 
 mkdir -p /var/local/git
-git clone --recursive /var/local/jenkins/grpc /var/local/git/grpc
+git clone /var/local/jenkins/grpc /var/local/git/grpc
 
 # copy service account keys if available
 cp -r /var/local/jenkins/service_account $HOME || true
 
 cd /var/local/git/grpc
 
+# clone gRPC submodules
+git submodule | awk '{ system("git submodule update --init --reference \
+./../../jenkins/grpc" $2 " " $2) }'
+
 tools/run_tests/run_tests.py -l python -c opt --build_only
diff --git a/tools/dockerfile/interoptest/grpc_interop_ruby/build_interop.sh b/tools/dockerfile/interoptest/grpc_interop_ruby/build_interop.sh
index 97b3860f98..0290de9209 100755
--- a/tools/dockerfile/interoptest/grpc_interop_ruby/build_interop.sh
+++ b/tools/dockerfile/interoptest/grpc_interop_ruby/build_interop.sh
@@ -32,12 +32,17 @@
 set -e
 
 mkdir -p /var/local/git
-git clone --recursive /var/local/jenkins/grpc /var/local/git/grpc
+git clone /var/local/jenkins/grpc /var/local/git/grpc
 
 # copy service account keys if available
 cp -r /var/local/jenkins/service_account $HOME || true
 
 cd /var/local/git/grpc
+
+# clone gRPC submodules
+git submodule | awk '{ system("git submodule update --init --reference \
+./../../jenkins/grpc" $2 " " $2) }'
+
 rvm --default use ruby-2.1
 
 # build Ruby interop client and server
diff --git a/tools/dockerfile/stress_test/grpc_interop_stress_csharp/build_interop_stress.sh b/tools/dockerfile/stress_test/grpc_interop_stress_csharp/build_interop_stress.sh
index 1f4bf893cc..a4d7feef95 100755
--- a/tools/dockerfile/stress_test/grpc_interop_stress_csharp/build_interop_stress.sh
+++ b/tools/dockerfile/stress_test/grpc_interop_stress_csharp/build_interop_stress.sh
@@ -32,13 +32,17 @@
 set -e
 
 mkdir -p /var/local/git
-git clone --recursive /var/local/jenkins/grpc /var/local/git/grpc
+git clone /var/local/jenkins/grpc /var/local/git/grpc
 
 # Copy service account keys if available
 cp -r /var/local/jenkins/service_account $HOME || true
 
 cd /var/local/git/grpc
 
+# clone gRPC submodules
+git submodule | awk '{ system("git submodule update --init --reference \
+./../../jenkins/grpc" $2 " " $2) }'
+
 # Build C++ metrics client (to query the metrics from csharp stress client)
 make metrics_client -j
 
diff --git a/tools/dockerfile/stress_test/grpc_interop_stress_cxx/build_interop_stress.sh b/tools/dockerfile/stress_test/grpc_interop_stress_cxx/build_interop_stress.sh
index b67b1a1664..d704f86496 100755
--- a/tools/dockerfile/stress_test/grpc_interop_stress_cxx/build_interop_stress.sh
+++ b/tools/dockerfile/stress_test/grpc_interop_stress_cxx/build_interop_stress.sh
@@ -32,13 +32,17 @@
 set -e
 
 mkdir -p /var/local/git
-git clone --recursive /var/local/jenkins/grpc /var/local/git/grpc
+git clone /var/local/jenkins/grpc /var/local/git/grpc
 
 # copy service account keys if available
 cp -r /var/local/jenkins/service_account $HOME || true
 
 cd /var/local/git/grpc
 
+# clone gRPC submodules
+git submodule | awk '{ system("git submodule update --init --reference \
+./../../jenkins/grpc" $2 " " $2) }'
+
 make install-certs
 
 BUILD_TYPE=${BUILD_TYPE:=opt}
diff --git a/tools/dockerfile/stress_test/grpc_interop_stress_go/build_interop_stress.sh b/tools/dockerfile/stress_test/grpc_interop_stress_go/build_interop_stress.sh
index 919d885c17..2b3b69be6e 100755
--- a/tools/dockerfile/stress_test/grpc_interop_stress_go/build_interop_stress.sh
+++ b/tools/dockerfile/stress_test/grpc_interop_stress_go/build_interop_stress.sh
@@ -38,7 +38,11 @@ git clone --recursive /var/local/jenkins/grpc-go src/google.golang.org/grpc
 
 # Clone the 'grpc' repo. We just need this for the wrapper scripts under
 # grpc/tools/gcp/stress_tests
-git clone --recursive /var/local/jenkins/grpc /var/local/git/grpc
+git clone /var/local/jenkins/grpc /var/local/git/grpc
+
+# clone gRPC submodules
+(cd /var/local/git/grpc/ && exec git submodule | awk '{ system("git submodule \
+update --init --reference ./../../jenkins/grpc" $2 " " $2) }')
 
 # copy service account keys if available
 cp -r /var/local/jenkins/service_account $HOME || true
diff --git a/tools/dockerfile/stress_test/grpc_interop_stress_java/build_interop_stress.sh b/tools/dockerfile/stress_test/grpc_interop_stress_java/build_interop_stress.sh
index d4fdfbbac9..99d287b21b 100755
--- a/tools/dockerfile/stress_test/grpc_interop_stress_java/build_interop_stress.sh
+++ b/tools/dockerfile/stress_test/grpc_interop_stress_java/build_interop_stress.sh
@@ -36,7 +36,11 @@ mkdir -p /var/local/git
 git clone --recursive --depth 1 /var/local/jenkins/grpc-java /var/local/git/grpc-java
 
 # grpc repo (for metrics client and for the stress test wrapper scripts)
-git clone --recursive /var/local/jenkins/grpc /var/local/git/grpc
+git clone /var/local/jenkins/grpc /var/local/git/grpc
+
+# clone gRPC submodules
+(cd /var/local/git/grpc/ && exec git submodule | awk '{ system("git submodule \
+update --init --reference ./../../jenkins/grpc" $2 " " $2) }')
 
 # Copy service account keys if available
 cp -r /var/local/jenkins/service_account $HOME || true
diff --git a/tools/dockerfile/stress_test/grpc_interop_stress_node/build_interop_stress.sh b/tools/dockerfile/stress_test/grpc_interop_stress_node/build_interop_stress.sh
index 976f55d9ab..85750ed91c 100755
--- a/tools/dockerfile/stress_test/grpc_interop_stress_node/build_interop_stress.sh
+++ b/tools/dockerfile/stress_test/grpc_interop_stress_node/build_interop_stress.sh
@@ -32,13 +32,17 @@
 set -e
 
 mkdir -p /var/local/git
-git clone --recursive /var/local/jenkins/grpc /var/local/git/grpc
+git clone /var/local/jenkins/grpc /var/local/git/grpc
 
 # copy service account keys if available
 cp -r /var/local/jenkins/service_account $HOME || true
 
 cd /var/local/git/grpc
 
+# clone gRPC submodules
+git submodule | awk '{ system("git submodule update --init --reference \
+./../../jenkins/grpc" $2 " " $2) }'
+
 # build Node interop client & server
 npm install -g node-gyp
 npm install --unsafe-perm --build-from-source
diff --git a/tools/dockerfile/stress_test/grpc_interop_stress_php/build_interop_stress.sh b/tools/dockerfile/stress_test/grpc_interop_stress_php/build_interop_stress.sh
index 87262f1d62..997cda6a35 100755
--- a/tools/dockerfile/stress_test/grpc_interop_stress_php/build_interop_stress.sh
+++ b/tools/dockerfile/stress_test/grpc_interop_stress_php/build_interop_stress.sh
@@ -32,12 +32,17 @@
 set -ex
 
 mkdir -p /var/local/git
-git clone --recursive /var/local/jenkins/grpc /var/local/git/grpc
+git clone /var/local/jenkins/grpc /var/local/git/grpc
 
 # copy service account keys if available
 cp -r /var/local/jenkins/service_account $HOME || true
 
 cd /var/local/git/grpc
+
+# clone gRPC submodules
+git submodule | awk '{ system("git submodule update --init --reference \
+./../../jenkins/grpc" $2 " " $2) }'
+
 rvm --default use ruby-2.1
 
 make install-certs
diff --git a/tools/dockerfile/stress_test/grpc_interop_stress_python/build_interop_stress.sh b/tools/dockerfile/stress_test/grpc_interop_stress_python/build_interop_stress.sh
index e65332f2f3..e3ccd8e00d 100755
--- a/tools/dockerfile/stress_test/grpc_interop_stress_python/build_interop_stress.sh
+++ b/tools/dockerfile/stress_test/grpc_interop_stress_python/build_interop_stress.sh
@@ -32,13 +32,17 @@
 set -e
 
 mkdir -p /var/local/git
-git clone --recursive /var/local/jenkins/grpc /var/local/git/grpc
+git clone /var/local/jenkins/grpc /var/local/git/grpc
 
 # copy service account keys if available
 cp -r /var/local/jenkins/service_account $HOME || true
 
 cd /var/local/git/grpc
 
+# clone gRPC submodules
+git submodule | awk '{ system("git submodule update --init --reference \
+./../../jenkins/grpc" $2 " " $2) }'
+
 tools/run_tests/run_tests.py -l python -c opt --build_only
 
 # Build c++ interop client
diff --git a/tools/dockerfile/stress_test/grpc_interop_stress_ruby/build_interop_stress.sh b/tools/dockerfile/stress_test/grpc_interop_stress_ruby/build_interop_stress.sh
index 1b7567d87a..be39fb484b 100755
--- a/tools/dockerfile/stress_test/grpc_interop_stress_ruby/build_interop_stress.sh
+++ b/tools/dockerfile/stress_test/grpc_interop_stress_ruby/build_interop_stress.sh
@@ -32,12 +32,17 @@
 set -e
 
 mkdir -p /var/local/git
-git clone --recursive /var/local/jenkins/grpc /var/local/git/grpc
+git clone /var/local/jenkins/grpc /var/local/git/grpc
 
 # Copy service account keys if available
 cp -r /var/local/jenkins/service_account $HOME || true
 
 cd /var/local/git/grpc
+
+# clone gRPC submodules
+git submodule | awk '{ system("git submodule update --init --reference \
+./../../jenkins/grpc" $2 " " $2) }'
+
 rvm --default use ruby-2.1
 
 # Build Ruby interop client and server
diff --git a/tools/run_tests/dockerize/docker_run.sh b/tools/run_tests/dockerize/docker_run.sh
index f04b1cfb55..bb875e275e 100755
--- a/tools/run_tests/dockerize/docker_run.sh
+++ b/tools/run_tests/dockerize/docker_run.sh
@@ -36,7 +36,11 @@ set -ex
 if [ "$RELATIVE_COPY_PATH" == "" ]
 then
   mkdir -p /var/local/git
-  git clone --recursive "$EXTERNAL_GIT_ROOT" /var/local/git/grpc
+  git clone "$EXTERNAL_GIT_ROOT" /var/local/git/grpc
+  # clone gRPC submodules
+  (cd var/local/git/grpc && exec git submodule | awk -v \
+  EXTERNAL_GIT_ROOT=$EXTERNAL_GIT_ROOT '{ system("git submodule update --init \
+  --reference " EXTERNAL_GIT_ROOT$2 " " $2) }')
 else
   mkdir -p "/var/local/git/grpc/$RELATIVE_COPY_PATH"
   cp -r "$EXTERNAL_GIT_ROOT/$RELATIVE_COPY_PATH"/* "/var/local/git/grpc/$RELATIVE_COPY_PATH"
diff --git a/tools/run_tests/dockerize/docker_run_tests.sh b/tools/run_tests/dockerize/docker_run_tests.sh
index 8c6143d24f..5537158960 100755
--- a/tools/run_tests/dockerize/docker_run_tests.sh
+++ b/tools/run_tests/dockerize/docker_run_tests.sh
@@ -42,7 +42,11 @@ export PATH=$PATH:/usr/bin/llvm-symbolizer
 chown $(whoami) $XDG_CACHE_HOME
 
 mkdir -p /var/local/git
-git clone --recursive /var/local/jenkins/grpc /var/local/git/grpc
+git clone /var/local/jenkins/grpc /var/local/git/grpc
+
+# clone gRPC submodules
+(cd /var/local/git/grpc/ && exec git submodule | awk '{ system("git submodule \
+update --init --reference ./../../jenkins/grpc" $2 " " $2) }')
 
 mkdir -p reports