Merge pull request #15424 from jtattermusch/tools_jenkins_cleanup

Cleanup tools/jenkins directory

10 files changed, 158 insertions, 6 deletions

diff --git a/tools/internal_ci/linux/grpc_bazel_build.cfg b/tools/internal_ci/linux/grpc_bazel_build.cfg
index 4d3a2ade5d..59f5da9a67 100644
--- a/tools/internal_ci/linux/grpc_bazel_build.cfg
+++ b/tools/internal_ci/linux/grpc_bazel_build.cfg
@@ -19,5 +19,5 @@ build_file: "grpc/tools/internal_ci/linux/grpc_bazel.sh"
 timeout_mins: 60
 env_vars {
   key: "BAZEL_SCRIPT"
-  value: "tools/jenkins/run_bazel_basic_in_docker.sh"
+  value: "tools/internal_ci/linux/grpc_bazel_build_in_docker.sh"
 }
diff --git a/tools/internal_ci/linux/grpc_bazel_build_in_docker.sh b/tools/internal_ci/linux/grpc_bazel_build_in_docker.sh
new file mode 100755
index 0000000000..24598f43f0
--- /dev/null
+++ b/tools/internal_ci/linux/grpc_bazel_build_in_docker.sh
@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+# Copyright 2017 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Test basic Bazel features
+#
+# NOTE: No empty lines should appear in this file before igncr is set!
+set -ex -o igncr || set -ex
+
+mkdir -p /var/local/git
+git clone /var/local/jenkins/grpc /var/local/git/grpc
+(cd /var/local/jenkins/grpc/ && git submodule foreach 'cd /var/local/git/grpc \
+&& git submodule update --init --reference /var/local/jenkins/grpc/${name} \
+${name}')
+cd /var/local/git/grpc
+bazel build --spawn_strategy=standalone --genrule_strategy=standalone :all test/... examples/...
diff --git a/tools/internal_ci/linux/grpc_bazel_test.cfg b/tools/internal_ci/linux/grpc_bazel_test.cfg
index 46198b9600..6b03be9518 100644
--- a/tools/internal_ci/linux/grpc_bazel_test.cfg
+++ b/tools/internal_ci/linux/grpc_bazel_test.cfg
@@ -19,5 +19,5 @@ build_file: "grpc/tools/internal_ci/linux/grpc_bazel.sh"
 timeout_mins: 240
 env_vars {
   key: "BAZEL_SCRIPT"
-  value: "tools/jenkins/run_bazel_full_in_docker.sh"
+  value: "tools/internal_ci/linux/grpc_bazel_test_in_docker.sh"
 }
diff --git a/tools/internal_ci/linux/grpc_bazel_test_in_docker.sh b/tools/internal_ci/linux/grpc_bazel_test_in_docker.sh
new file mode 100755
index 0000000000..9663debe69
--- /dev/null
+++ b/tools/internal_ci/linux/grpc_bazel_test_in_docker.sh
@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+# Copyright 2017 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Test full Bazel
+#
+# NOTE: No empty lines should appear in this file before igncr is set!
+set -ex -o igncr || set -ex
+
+mkdir -p /var/local/git
+git clone /var/local/jenkins/grpc /var/local/git/grpc
+(cd /var/local/jenkins/grpc/ && git submodule foreach 'cd /var/local/git/grpc \
+&& git submodule update --init --reference /var/local/jenkins/grpc/${name} \
+${name}')
+cd /var/local/git/grpc/test
+bazel test --spawn_strategy=standalone --genrule_strategy=standalone ...
diff --git a/tools/internal_ci/linux/grpc_microbenchmark_diff.sh b/tools/internal_ci/linux/grpc_microbenchmark_diff.sh
index 45add1b02d..9834aaa053 100755
--- a/tools/internal_ci/linux/grpc_microbenchmark_diff.sh
+++ b/tools/internal_ci/linux/grpc_microbenchmark_diff.sh
@@ -25,9 +25,9 @@ cd $(dirname $0)/../../..
 source tools/internal_ci/helper_scripts/prepare_build_linux_perf_rc
 
 tools/run_tests/start_port_server.py
-tools/jenkins/run_c_cpp_test.sh tools/profiling/bloat/bloat_diff.py \
+tools/internal_ci/linux/run_if_c_cpp_modified.sh tools/profiling/bloat/bloat_diff.py \
   -d origin/$ghprbTargetBranch || FAILED="true"
-tools/jenkins/run_c_cpp_test.sh tools/profiling/microbenchmarks/bm_diff/bm_main.py \
+tools/internal_ci/linux/run_if_c_cpp_modified.sh tools/profiling/microbenchmarks/bm_diff/bm_main.py \
   -d origin/$ghprbTargetBranch \
   -b $BENCHMARKS_TO_RUN || FAILED="true"
 
diff --git a/tools/internal_ci/linux/grpc_performance_profile_master.sh b/tools/internal_ci/linux/grpc_performance_profile_master.sh
index 40bbfe89dc..fbff17466b 100755
--- a/tools/internal_ci/linux/grpc_performance_profile_master.sh
+++ b/tools/internal_ci/linux/grpc_performance_profile_master.sh
@@ -20,7 +20,7 @@ cd $(dirname $0)/../../..
 
 source tools/internal_ci/helper_scripts/prepare_build_linux_perf_rc
 
-tools/jenkins/run_performance_profile_hourly.sh || FAILED="true"
+tools/internal_ci/linux/run_performance_profile_hourly.sh || FAILED="true"
 
 # kill port_server.py to prevent the build from hanging
 ps aux | grep port_server\\.py | awk '{print $2}' | xargs kill -9
diff --git a/tools/internal_ci/linux/grpc_trickle_diff.sh b/tools/internal_ci/linux/grpc_trickle_diff.sh
index 624031afa7..4ed1b73f9e 100755
--- a/tools/internal_ci/linux/grpc_trickle_diff.sh
+++ b/tools/internal_ci/linux/grpc_trickle_diff.sh
@@ -25,7 +25,7 @@ cd $(dirname $0)/../../..
 source tools/internal_ci/helper_scripts/prepare_build_linux_perf_rc
 
 tools/run_tests/start_port_server.py
-tools/jenkins/run_c_cpp_test.sh tools/profiling/microbenchmarks/bm_diff/bm_main.py \
+tools/internal_ci/linux/run_if_c_cpp_modified.sh tools/profiling/microbenchmarks/bm_diff/bm_main.py \
   -d origin/$ghprbTargetBranch \
   -b bm_fullstack_trickle \
   -l 4 \
diff --git a/tools/internal_ci/linux/run_if_c_cpp_modified.sh b/tools/internal_ci/linux/run_if_c_cpp_modified.sh
new file mode 100755
index 0000000000..736d759423
--- /dev/null
+++ b/tools/internal_ci/linux/run_if_c_cpp_modified.sh
@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+# Copyright 2017 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# This script is invoked by a pull request job and executes all
+# args passed to this script if the pull request affect C/C++ code
+set -ex
+
+# Enter the gRPC repo root
+cd $(dirname $0)/../../..
+
+# TODO(jtattermusch): the "ghprbTargetBranch" is Jenkins specific and probably
+# does not work on kokoro?
+AFFECTS_C_CPP=`python -c 'import os; \
+               import sys; \
+               sys.path.insert(0, "tools/run_tests/python_utils"); \
+               import filter_pull_request_tests as filter; \
+               github_target_branch = os.environ.get("ghprbTargetBranch"); \
+               print(filter.affects_c_cpp("origin/%s" % github_target_branch))'`
+
+if [ $AFFECTS_C_CPP == "False" ] ; then
+  echo "This pull request does not affect C/C++. Tests do not need to be run."
+else
+  $@
+fi
diff --git a/tools/internal_ci/linux/run_performance_profile_daily.sh b/tools/internal_ci/linux/run_performance_profile_daily.sh
new file mode 100755
index 0000000000..45c7a99805
--- /dev/null
+++ b/tools/internal_ci/linux/run_performance_profile_daily.sh
@@ -0,0 +1,34 @@
+#!/bin/bash
+# Copyright 2016 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -ex
+
+cd $(dirname $0)/../../..
+
+# try to use pypy for generating reports
+# each trace dumps 7-8gig of text to disk, and processing this into a report is
+# heavyweight - so any speed boost is worthwhile
+# TODO(ctiller): consider rewriting report generation in C++ for performance
+if which pypy >/dev/null; then
+  PYTHON=pypy
+else
+  PYTHON=python2.7
+fi
+
+BENCHMARKS_TO_RUN="bm_fullstack_unary_ping_pong bm_fullstack_streaming_ping_pong bm_fullstack_streaming_pump bm_closure bm_cq bm_call_create bm_error bm_chttp2_hpack bm_chttp2_transport bm_pollset bm_metadata"
+
+./tools/run_tests/start_port_server.py || true
+
+$PYTHON tools/run_tests/run_microbenchmark.py --collect summary perf latency -b $BENCHMARKS_TO_RUN
diff --git a/tools/internal_ci/linux/run_performance_profile_hourly.sh b/tools/internal_ci/linux/run_performance_profile_hourly.sh
new file mode 100755
index 0000000000..edf85c2e2c
--- /dev/null
+++ b/tools/internal_ci/linux/run_performance_profile_hourly.sh
@@ -0,0 +1,28 @@
+#!/bin/bash
+# Copyright 2016 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -ex
+
+cd $(dirname $0)/../../..
+
+./tools/run_tests/start_port_server.py || true
+
+CPUS=`python -c 'import multiprocessing; print multiprocessing.cpu_count()'`
+
+make CONFIG=opt memory_profile_test memory_profile_client memory_profile_server -j $CPUS
+bins/opt/memory_profile_test
+bq load microbenchmarks.memory memory_usage.csv
+
+tools/run_tests/run_microbenchmark.py --collect summary --bigquery_upload