aboutsummaryrefslogtreecommitdiffhomepage
path: root/tools
diff options
context:
space:
mode:
authorGravatar Craig Tiller <ctiller@google.com>2017-09-29 11:18:26 -0700
committerGravatar Craig Tiller <ctiller@google.com>2017-09-29 11:18:26 -0700
commit1e868f0f9539925e51aa52269c848082d23b7c4e (patch)
tree49fd965a5f58e2605864df54a1d70621367eee36 /tools
parent710334577ce1b2de94f656ec0762eeba6effd29b (diff)
parent903f06fe3f8b1b971f1b633dff45488ca68c6708 (diff)
Merge github.com:grpc/grpc into flowctl+millis
Diffstat (limited to 'tools')
-rwxr-xr-xtools/codegen/core/gen_stats_data.py4
-rwxr-xr-xtools/distrib/pylint_code.sh2
-rwxr-xr-xtools/distrib/python/docgen.py2
-rwxr-xr-xtools/distrib/yapf_code.sh2
-rw-r--r--tools/dockerfile/grpc_clang_format/Dockerfile8
-rw-r--r--tools/dockerfile/interoptest/grpc_interop_csharp/Dockerfile2
-rw-r--r--tools/dockerfile/interoptest/grpc_interop_csharpcoreclr/Dockerfile2
-rw-r--r--tools/dockerfile/interoptest/grpc_interop_cxx/Dockerfile2
-rw-r--r--tools/dockerfile/interoptest/grpc_interop_go/Dockerfile2
-rw-r--r--tools/dockerfile/interoptest/grpc_interop_go1.7/Dockerfile2
-rw-r--r--tools/dockerfile/interoptest/grpc_interop_go1.8/Dockerfile2
-rw-r--r--tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile2
-rw-r--r--tools/dockerfile/interoptest/grpc_interop_java/Dockerfile2
-rw-r--r--tools/dockerfile/interoptest/grpc_interop_java_oracle8/Dockerfile2
-rw-r--r--tools/dockerfile/interoptest/grpc_interop_node/Dockerfile2
-rw-r--r--tools/dockerfile/interoptest/grpc_interop_python/Dockerfile2
-rw-r--r--tools/dockerfile/interoptest/grpc_interop_ruby/Dockerfile2
-rw-r--r--tools/dockerfile/test/csharp_jessie_x64/Dockerfile2
-rw-r--r--tools/dockerfile/test/cxx_alpine_x64/Dockerfile2
-rw-r--r--tools/dockerfile/test/cxx_jessie_x64/Dockerfile2
-rw-r--r--tools/dockerfile/test/cxx_jessie_x86/Dockerfile2
-rw-r--r--tools/dockerfile/test/cxx_ubuntu1404_x64/Dockerfile2
-rw-r--r--tools/dockerfile/test/cxx_ubuntu1604_x64/Dockerfile2
-rw-r--r--tools/dockerfile/test/fuzzer/Dockerfile2
-rw-r--r--tools/dockerfile/test/multilang_jessie_x64/Dockerfile2
-rw-r--r--tools/dockerfile/test/node_jessie_x64/Dockerfile2
-rw-r--r--tools/dockerfile/test/php7_jessie_x64/Dockerfile2
-rw-r--r--tools/dockerfile/test/php_jessie_x64/Dockerfile2
-rw-r--r--tools/dockerfile/test/python_alpine_x64/Dockerfile2
-rw-r--r--tools/dockerfile/test/python_jessie_x64/Dockerfile2
-rw-r--r--tools/dockerfile/test/python_pyenv_x64/Dockerfile2
-rw-r--r--tools/dockerfile/test/ruby_jessie_x64/Dockerfile2
-rw-r--r--tools/dockerfile/test/sanity/Dockerfile35
-rw-r--r--tools/doxygen/Doxyfile.c++.internal2
-rw-r--r--tools/doxygen/Doxyfile.core.internal4
-rwxr-xr-xtools/gce/linux_performance_worker_init.sh1
-rw-r--r--tools/github/pr_latency.py163
-rwxr-xr-xtools/internal_ci/linux/grpc_performance_profile_daily.sh2
-rw-r--r--tools/interop_matrix/README.md16
-rwxr-xr-xtools/jenkins/run_full_performance.sh2
-rwxr-xr-xtools/jenkins/run_performance_profile_daily.sh2
-rwxr-xr-xtools/profiling/microbenchmarks/bm_diff/bm_main.py16
-rwxr-xr-xtools/profiling/microbenchmarks/bm_diff/bm_run.py9
-rw-r--r--tools/run_tests/generated/sources_and_headers.json6
-rwxr-xr-xtools/run_tests/helper_scripts/build_python.sh2
-rwxr-xr-xtools/run_tests/helper_scripts/run_grpc-node.sh28
-rw-r--r--tools/run_tests/performance/massage_qps_stats.py8
-rw-r--r--tools/run_tests/performance/scenario_result_schema.json70
-rw-r--r--tools/run_tests/python_utils/filter_pull_request_tests.py2
-rwxr-xr-xtools/run_tests/run_tests.py67
-rwxr-xr-xtools/run_tests/run_tests_matrix.py2
-rwxr-xr-xtools/run_tests/sanity/check_test_filtering.py2
52 files changed, 373 insertions, 138 deletions
diff --git a/tools/codegen/core/gen_stats_data.py b/tools/codegen/core/gen_stats_data.py
index 10ad0cc831..8359734c84 100755
--- a/tools/codegen/core/gen_stats_data.py
+++ b/tools/codegen/core/gen_stats_data.py
@@ -147,8 +147,7 @@ def gen_bucket_code(histogram):
shift_data = find_ideal_shift(code_bounds[first_nontrivial:], 256 * histogram.buckets)
#print first_nontrivial, shift_data, bounds
#if shift_data is not None: print [hex(x >> shift_data[0]) for x in code_bounds[first_nontrivial:]]
- code = '\n/* Automatically generated by tools/codegen/core/gen_stats_data.py */\n'
- code += 'value = GPR_CLAMP(value, 0, %d);\n' % histogram.max
+ code = 'value = GPR_CLAMP(value, 0, %d);\n' % histogram.max
map_table = gen_map_table(code_bounds[first_nontrivial:], shift_data)
if first_nontrivial is None:
code += ('GRPC_STATS_INC_HISTOGRAM((exec_ctx), GRPC_STATS_HISTOGRAM_%s, value);\n'
@@ -408,3 +407,4 @@ with open('src/core/lib/debug/stats_data_bq_schema.sql', 'w') as S:
for counter in inst_map['Counter']:
columns.append(('%s_per_iteration' % counter.name, 'FLOAT'))
print >>S, ',\n'.join('%s:%s' % x for x in columns)
+
diff --git a/tools/distrib/pylint_code.sh b/tools/distrib/pylint_code.sh
index 3c9235b548..7175f1e15b 100755
--- a/tools/distrib/pylint_code.sh
+++ b/tools/distrib/pylint_code.sh
@@ -29,7 +29,7 @@ VIRTUALENV=python_pylint_venv
virtualenv $VIRTUALENV
PYTHON=$(realpath $VIRTUALENV/bin/python)
-$PYTHON -m pip install --upgrade pip
+$PYTHON -m pip install --upgrade pip==9.0.1
$PYTHON -m pip install pylint==1.6.5
for dir in "${DIRS[@]}"; do
diff --git a/tools/distrib/python/docgen.py b/tools/distrib/python/docgen.py
index 6f6d43c7d9..1822e51d09 100755
--- a/tools/distrib/python/docgen.py
+++ b/tools/distrib/python/docgen.py
@@ -60,7 +60,7 @@ environment.update({
subprocess_arguments_list = [
{'args': ['virtualenv', VIRTUALENV_DIR], 'env': environment},
- {'args': [VIRTUALENV_PIP_PATH, 'install', '--upgrade', 'pip'],
+ {'args': [VIRTUALENV_PIP_PATH, 'install', '--upgrade', 'pip==9.0.1'],
'env': environment},
{'args': [VIRTUALENV_PIP_PATH, 'install', '-r', REQUIREMENTS_PATH],
'env': environment},
diff --git a/tools/distrib/yapf_code.sh b/tools/distrib/yapf_code.sh
index dbb6b5c41f..e5beb70117 100755
--- a/tools/distrib/yapf_code.sh
+++ b/tools/distrib/yapf_code.sh
@@ -33,7 +33,7 @@ VIRTUALENV=yapf_virtual_environment
virtualenv $VIRTUALENV
PYTHON=$(realpath "${VIRTUALENV}/bin/python")
-$PYTHON -m pip install --upgrade pip
+$PYTHON -m pip install --upgrade pip==9.0.1
$PYTHON -m pip install --upgrade futures
$PYTHON -m pip install yapf==0.16.0
diff --git a/tools/dockerfile/grpc_clang_format/Dockerfile b/tools/dockerfile/grpc_clang_format/Dockerfile
index 647cb52a7b..dff07feeb6 100644
--- a/tools/dockerfile/grpc_clang_format/Dockerfile
+++ b/tools/dockerfile/grpc_clang_format/Dockerfile
@@ -12,12 +12,12 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-FROM ubuntu:15.10
+FROM debian:jessie
RUN apt-get update && apt-get -y install wget
-RUN echo deb http://llvm.org/apt/wily/ llvm-toolchain-wily-3.8 main >> /etc/apt/sources.list
-RUN echo deb-src http://llvm.org/apt/wily/ llvm-toolchain-wily-3.8 main >> /etc/apt/sources.list
-RUN wget -O - http://llvm.org/apt/llvm-snapshot.gpg.key| apt-key add -
+RUN echo "deb http://llvm.org/apt/jessie/ llvm-toolchain-jessie-3.8 main" >> /etc/apt/sources.list
+RUN echo "deb-src http://llvm.org/apt/jessie/ llvm-toolchain-jessie-3.8 main" >> /etc/apt/sources.list
+RUN wget -O - http://llvm.org/apt/llvm-snapshot.gpg.key | apt-key add -
RUN apt-get update && apt-get -y install clang-format-3.8
ADD clang_format_all_the_things.sh /
diff --git a/tools/dockerfile/interoptest/grpc_interop_csharp/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_csharp/Dockerfile
index dbf58023c5..ea82476b65 100644
--- a/tools/dockerfile/interoptest/grpc_interop_csharp/Dockerfile
+++ b/tools/dockerfile/interoptest/grpc_interop_csharp/Dockerfile
@@ -60,7 +60,7 @@ RUN apt-get update && apt-get install -y \
python-pip
# Install Python packages from PyPI
-RUN pip install pip --upgrade
+RUN pip install --upgrade pip==9.0.1
RUN pip install virtualenv
RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0
diff --git a/tools/dockerfile/interoptest/grpc_interop_csharpcoreclr/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_csharpcoreclr/Dockerfile
index 4ccfbc43c3..56b8be89ec 100644
--- a/tools/dockerfile/interoptest/grpc_interop_csharpcoreclr/Dockerfile
+++ b/tools/dockerfile/interoptest/grpc_interop_csharpcoreclr/Dockerfile
@@ -59,7 +59,7 @@ RUN apt-get update && apt-get install -y \
python-pip
# Install Python packages from PyPI
-RUN pip install pip --upgrade
+RUN pip install --upgrade pip==9.0.1
RUN pip install virtualenv
RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0
diff --git a/tools/dockerfile/interoptest/grpc_interop_cxx/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_cxx/Dockerfile
index 7cfe98cbc8..38d377ce2f 100644
--- a/tools/dockerfile/interoptest/grpc_interop_cxx/Dockerfile
+++ b/tools/dockerfile/interoptest/grpc_interop_cxx/Dockerfile
@@ -60,7 +60,7 @@ RUN apt-get update && apt-get install -y \
python-pip
# Install Python packages from PyPI
-RUN pip install pip --upgrade
+RUN pip install --upgrade pip==9.0.1
RUN pip install virtualenv
RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0
diff --git a/tools/dockerfile/interoptest/grpc_interop_go/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_go/Dockerfile
index febe2fa251..73c41a4838 100644
--- a/tools/dockerfile/interoptest/grpc_interop_go/Dockerfile
+++ b/tools/dockerfile/interoptest/grpc_interop_go/Dockerfile
@@ -28,7 +28,7 @@ RUN apt-get update && apt-get install -y \
python-pip
# Install Python packages from PyPI
-RUN pip install pip --upgrade
+RUN pip install --upgrade pip==9.0.1
RUN pip install virtualenv
RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0
diff --git a/tools/dockerfile/interoptest/grpc_interop_go1.7/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_go1.7/Dockerfile
index 3a516cbb62..7c083de365 100644
--- a/tools/dockerfile/interoptest/grpc_interop_go1.7/Dockerfile
+++ b/tools/dockerfile/interoptest/grpc_interop_go1.7/Dockerfile
@@ -28,7 +28,7 @@ RUN apt-get update && apt-get install -y \
python-pip
# Install Python packages from PyPI
-RUN pip install pip --upgrade
+RUN pip install --upgrade pip==9.0.1
RUN pip install virtualenv
RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0
diff --git a/tools/dockerfile/interoptest/grpc_interop_go1.8/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_go1.8/Dockerfile
index acb640a81d..61efc18df9 100644
--- a/tools/dockerfile/interoptest/grpc_interop_go1.8/Dockerfile
+++ b/tools/dockerfile/interoptest/grpc_interop_go1.8/Dockerfile
@@ -28,7 +28,7 @@ RUN apt-get update && apt-get install -y \
python-pip
# Install Python packages from PyPI
-RUN pip install pip --upgrade
+RUN pip install --upgrade pip==9.0.1
RUN pip install virtualenv
RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0
diff --git a/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile
index 354b7bfdb5..278b09a666 100644
--- a/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile
+++ b/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile
@@ -28,7 +28,7 @@ RUN apt-get update && apt-get install -y \
python-pip
# Install Python packages from PyPI
-RUN pip install pip --upgrade
+RUN pip install --upgrade pip==9.0.1
RUN pip install virtualenv
RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0
diff --git a/tools/dockerfile/interoptest/grpc_interop_java/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_java/Dockerfile
index 92a542ff76..d56632440f 100644
--- a/tools/dockerfile/interoptest/grpc_interop_java/Dockerfile
+++ b/tools/dockerfile/interoptest/grpc_interop_java/Dockerfile
@@ -43,7 +43,7 @@ RUN apt-get update && apt-get install -y \
python-pip
# Install Python packages from PyPI
-RUN pip install pip --upgrade
+RUN pip install --upgrade pip==9.0.1
RUN pip install virtualenv
RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0
diff --git a/tools/dockerfile/interoptest/grpc_interop_java_oracle8/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_java_oracle8/Dockerfile
index 92a542ff76..d56632440f 100644
--- a/tools/dockerfile/interoptest/grpc_interop_java_oracle8/Dockerfile
+++ b/tools/dockerfile/interoptest/grpc_interop_java_oracle8/Dockerfile
@@ -43,7 +43,7 @@ RUN apt-get update && apt-get install -y \
python-pip
# Install Python packages from PyPI
-RUN pip install pip --upgrade
+RUN pip install --upgrade pip==9.0.1
RUN pip install virtualenv
RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0
diff --git a/tools/dockerfile/interoptest/grpc_interop_node/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_node/Dockerfile
index 4343d56cb4..f4c3e4103e 100644
--- a/tools/dockerfile/interoptest/grpc_interop_node/Dockerfile
+++ b/tools/dockerfile/interoptest/grpc_interop_node/Dockerfile
@@ -60,7 +60,7 @@ RUN apt-get update && apt-get install -y \
python-pip
# Install Python packages from PyPI
-RUN pip install pip --upgrade
+RUN pip install --upgrade pip==9.0.1
RUN pip install virtualenv
RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0
diff --git a/tools/dockerfile/interoptest/grpc_interop_python/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_python/Dockerfile
index 271c6e75e9..d165307031 100644
--- a/tools/dockerfile/interoptest/grpc_interop_python/Dockerfile
+++ b/tools/dockerfile/interoptest/grpc_interop_python/Dockerfile
@@ -60,7 +60,7 @@ RUN apt-get update && apt-get install -y \
python-pip
# Install Python packages from PyPI
-RUN pip install pip --upgrade
+RUN pip install --upgrade pip==9.0.1
RUN pip install virtualenv
RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0
diff --git a/tools/dockerfile/interoptest/grpc_interop_ruby/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_ruby/Dockerfile
index 7bcada68e0..2217b10b8b 100644
--- a/tools/dockerfile/interoptest/grpc_interop_ruby/Dockerfile
+++ b/tools/dockerfile/interoptest/grpc_interop_ruby/Dockerfile
@@ -60,7 +60,7 @@ RUN apt-get update && apt-get install -y \
python-pip
# Install Python packages from PyPI
-RUN pip install pip --upgrade
+RUN pip install --upgrade pip==9.0.1
RUN pip install virtualenv
RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0
diff --git a/tools/dockerfile/test/csharp_jessie_x64/Dockerfile b/tools/dockerfile/test/csharp_jessie_x64/Dockerfile
index 40d46fcf58..3e31e67f5f 100644
--- a/tools/dockerfile/test/csharp_jessie_x64/Dockerfile
+++ b/tools/dockerfile/test/csharp_jessie_x64/Dockerfile
@@ -64,7 +64,7 @@ RUN apt-get update && apt-get install -y \
python-pip
# Install Python packages from PyPI
-RUN pip install pip --upgrade
+RUN pip install --upgrade pip==9.0.1
RUN pip install virtualenv
RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0
diff --git a/tools/dockerfile/test/cxx_alpine_x64/Dockerfile b/tools/dockerfile/test/cxx_alpine_x64/Dockerfile
index 1ae50c106f..af5e7d6f14 100644
--- a/tools/dockerfile/test/cxx_alpine_x64/Dockerfile
+++ b/tools/dockerfile/test/cxx_alpine_x64/Dockerfile
@@ -36,7 +36,7 @@ RUN apk update && apk add \
zip
# Install Python packages from PyPI
-RUN pip install pip --upgrade
+RUN pip install --upgrade pip==9.0.1
RUN pip install virtualenv
RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0
diff --git a/tools/dockerfile/test/cxx_jessie_x64/Dockerfile b/tools/dockerfile/test/cxx_jessie_x64/Dockerfile
index 888a37baca..3492dd72c5 100644
--- a/tools/dockerfile/test/cxx_jessie_x64/Dockerfile
+++ b/tools/dockerfile/test/cxx_jessie_x64/Dockerfile
@@ -64,7 +64,7 @@ RUN apt-get update && apt-get install -y \
python-pip
# Install Python packages from PyPI
-RUN pip install pip --upgrade
+RUN pip install --upgrade pip==9.0.1
RUN pip install virtualenv
RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0
diff --git a/tools/dockerfile/test/cxx_jessie_x86/Dockerfile b/tools/dockerfile/test/cxx_jessie_x86/Dockerfile
index 319f1e1889..f8cbf35d39 100644
--- a/tools/dockerfile/test/cxx_jessie_x86/Dockerfile
+++ b/tools/dockerfile/test/cxx_jessie_x86/Dockerfile
@@ -64,7 +64,7 @@ RUN apt-get update && apt-get install -y \
python-pip
# Install Python packages from PyPI
-RUN pip install pip --upgrade
+RUN pip install --upgrade pip==9.0.1
RUN pip install virtualenv
RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0
diff --git a/tools/dockerfile/test/cxx_ubuntu1404_x64/Dockerfile b/tools/dockerfile/test/cxx_ubuntu1404_x64/Dockerfile
index 61f005d9da..6966d6b8d3 100644
--- a/tools/dockerfile/test/cxx_ubuntu1404_x64/Dockerfile
+++ b/tools/dockerfile/test/cxx_ubuntu1404_x64/Dockerfile
@@ -64,7 +64,7 @@ RUN apt-get update && apt-get install -y \
python-pip
# Install Python packages from PyPI
-RUN pip install pip --upgrade
+RUN pip install --upgrade pip==9.0.1
RUN pip install virtualenv
RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0
diff --git a/tools/dockerfile/test/cxx_ubuntu1604_x64/Dockerfile b/tools/dockerfile/test/cxx_ubuntu1604_x64/Dockerfile
index f35247eccb..016034aa50 100644
--- a/tools/dockerfile/test/cxx_ubuntu1604_x64/Dockerfile
+++ b/tools/dockerfile/test/cxx_ubuntu1604_x64/Dockerfile
@@ -64,7 +64,7 @@ RUN apt-get update && apt-get install -y \
python-pip
# Install Python packages from PyPI
-RUN pip install pip --upgrade
+RUN pip install --upgrade pip==9.0.1
RUN pip install virtualenv
RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0
diff --git a/tools/dockerfile/test/fuzzer/Dockerfile b/tools/dockerfile/test/fuzzer/Dockerfile
index ce1badfeb1..50104ad7ae 100644
--- a/tools/dockerfile/test/fuzzer/Dockerfile
+++ b/tools/dockerfile/test/fuzzer/Dockerfile
@@ -64,7 +64,7 @@ RUN apt-get update && apt-get install -y \
python-pip
# Install Python packages from PyPI
-RUN pip install pip --upgrade
+RUN pip install --upgrade pip==9.0.1
RUN pip install virtualenv
RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0
diff --git a/tools/dockerfile/test/multilang_jessie_x64/Dockerfile b/tools/dockerfile/test/multilang_jessie_x64/Dockerfile
index 59fe4d8f93..1a4b681576 100644
--- a/tools/dockerfile/test/multilang_jessie_x64/Dockerfile
+++ b/tools/dockerfile/test/multilang_jessie_x64/Dockerfile
@@ -121,7 +121,7 @@ RUN apt-get update && apt-get install -y \
python-pip
# Install Python packages from PyPI
-RUN pip install pip --upgrade
+RUN pip install --upgrade pip==9.0.1
RUN pip install virtualenv
RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0
diff --git a/tools/dockerfile/test/node_jessie_x64/Dockerfile b/tools/dockerfile/test/node_jessie_x64/Dockerfile
index 103be8412b..4f18dbae79 100644
--- a/tools/dockerfile/test/node_jessie_x64/Dockerfile
+++ b/tools/dockerfile/test/node_jessie_x64/Dockerfile
@@ -75,7 +75,7 @@ RUN apt-get update && apt-get install -y \
python-pip
# Install Python packages from PyPI
-RUN pip install pip --upgrade
+RUN pip install --upgrade pip==9.0.1
RUN pip install virtualenv
RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0
diff --git a/tools/dockerfile/test/php7_jessie_x64/Dockerfile b/tools/dockerfile/test/php7_jessie_x64/Dockerfile
index f6d426bcd6..1399502840 100644
--- a/tools/dockerfile/test/php7_jessie_x64/Dockerfile
+++ b/tools/dockerfile/test/php7_jessie_x64/Dockerfile
@@ -75,7 +75,7 @@ RUN apt-get update && apt-get install -y \
python-pip
# Install Python packages from PyPI
-RUN pip install pip --upgrade
+RUN pip install --upgrade pip==9.0.1
RUN pip install virtualenv
RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0
diff --git a/tools/dockerfile/test/php_jessie_x64/Dockerfile b/tools/dockerfile/test/php_jessie_x64/Dockerfile
index ae82a8d99f..56dc6045e1 100644
--- a/tools/dockerfile/test/php_jessie_x64/Dockerfile
+++ b/tools/dockerfile/test/php_jessie_x64/Dockerfile
@@ -64,7 +64,7 @@ RUN apt-get update && apt-get install -y \
python-pip
# Install Python packages from PyPI
-RUN pip install pip --upgrade
+RUN pip install --upgrade pip==9.0.1
RUN pip install virtualenv
RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0
diff --git a/tools/dockerfile/test/python_alpine_x64/Dockerfile b/tools/dockerfile/test/python_alpine_x64/Dockerfile
index 7bd11d7407..7584ab8767 100644
--- a/tools/dockerfile/test/python_alpine_x64/Dockerfile
+++ b/tools/dockerfile/test/python_alpine_x64/Dockerfile
@@ -36,7 +36,7 @@ RUN apk update && apk add \
zip
# Install Python packages from PyPI
-RUN pip install pip --upgrade
+RUN pip install --upgrade pip==9.0.1
RUN pip install virtualenv
RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0
diff --git a/tools/dockerfile/test/python_jessie_x64/Dockerfile b/tools/dockerfile/test/python_jessie_x64/Dockerfile
index d5d781cd1a..8d89f5022f 100644
--- a/tools/dockerfile/test/python_jessie_x64/Dockerfile
+++ b/tools/dockerfile/test/python_jessie_x64/Dockerfile
@@ -64,7 +64,7 @@ RUN apt-get update && apt-get install -y \
python-pip
# Install Python packages from PyPI
-RUN pip install pip --upgrade
+RUN pip install --upgrade pip==9.0.1
RUN pip install virtualenv
RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0
diff --git a/tools/dockerfile/test/python_pyenv_x64/Dockerfile b/tools/dockerfile/test/python_pyenv_x64/Dockerfile
index 3b4ad12b6d..f8cbdc5726 100644
--- a/tools/dockerfile/test/python_pyenv_x64/Dockerfile
+++ b/tools/dockerfile/test/python_pyenv_x64/Dockerfile
@@ -64,7 +64,7 @@ RUN apt-get update && apt-get install -y \
python-pip
# Install Python packages from PyPI
-RUN pip install pip --upgrade
+RUN pip install --upgrade pip==9.0.1
RUN pip install virtualenv
RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0
diff --git a/tools/dockerfile/test/ruby_jessie_x64/Dockerfile b/tools/dockerfile/test/ruby_jessie_x64/Dockerfile
index 3d879bb0c9..5d7f80bf88 100644
--- a/tools/dockerfile/test/ruby_jessie_x64/Dockerfile
+++ b/tools/dockerfile/test/ruby_jessie_x64/Dockerfile
@@ -64,7 +64,7 @@ RUN apt-get update && apt-get install -y \
python-pip
# Install Python packages from PyPI
-RUN pip install pip --upgrade
+RUN pip install --upgrade pip==9.0.1
RUN pip install virtualenv
RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0
diff --git a/tools/dockerfile/test/sanity/Dockerfile b/tools/dockerfile/test/sanity/Dockerfile
index 44732a5ae4..487ce15e2e 100644
--- a/tools/dockerfile/test/sanity/Dockerfile
+++ b/tools/dockerfile/test/sanity/Dockerfile
@@ -12,7 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-FROM ubuntu:15.10
+FROM debian:jessie
# Install Git and basic packages.
RUN apt-get update && apt-get install -y \
@@ -64,7 +64,7 @@ RUN apt-get update && apt-get install -y \
python-pip
# Install Python packages from PyPI
-RUN pip install pip --upgrade
+RUN pip install --upgrade pip==9.0.1
RUN pip install virtualenv
RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0
@@ -82,19 +82,28 @@ RUN pip install simplejson mako
#======================================
# More sanity test dependencies (bazel)
-RUN apt-get install -y openjdk-8-jdk
-# Check out Bazel version 0.4.1 since this version allows running
-# ./compile.sh without a local protoc dependency
-# TODO(mattkwong): install dependencies to support latest Bazel version if newer
-# version is needed
-RUN git clone https://github.com/bazelbuild/bazel.git /bazel && \
- cd /bazel && git checkout tags/0.4.1 && ./compile.sh
-RUN ln -s /bazel/output/bazel /bin/
+RUN echo "deb http://http.debian.net/debian jessie-backports main" >> /etc/apt/sources.list
+RUN apt-get update
+RUN apt-get install -y -t jessie-backports openjdk-8-jdk
+
+#========================
+# Bazel installation
+RUN echo "deb [arch=amd64] http://storage.googleapis.com/bazel-apt stable jdk1.8" > /etc/apt/sources.list.d/bazel.list
+RUN curl https://bazel.build/bazel-release.pub.gpg | apt-key add -
+RUN apt-get -y update
+RUN apt-get -y install bazel
+
+# Pin Bazel to 0.4.4
+# Installing Bazel via apt-get first is required before installing 0.4.4 to
+# allow gRPC to build without errors. See https://github.com/grpc/grpc/issues/10553
+RUN curl -fSsL -O https://github.com/bazelbuild/bazel/releases/download/0.4.4/bazel-0.4.4-installer-linux-x86_64.sh
+RUN chmod +x ./bazel-0.4.4-installer-linux-x86_64.sh
+RUN ./bazel-0.4.4-installer-linux-x86_64.sh
RUN apt-get update && apt-get -y install wget
-RUN echo deb http://llvm.org/apt/wily/ llvm-toolchain-wily-3.8 main >> /etc/apt/sources.list
-RUN echo deb-src http://llvm.org/apt/wily/ llvm-toolchain-wily-3.8 main >> /etc/apt/sources.list
-RUN wget -O - http://llvm.org/apt/llvm-snapshot.gpg.key| apt-key add -
+RUN echo "deb http://llvm.org/apt/jessie/ llvm-toolchain-jessie-3.8 main" >> /etc/apt/sources.list
+RUN echo "deb-src http://llvm.org/apt/jessie/ llvm-toolchain-jessie-3.8 main" >> /etc/apt/sources.list
+RUN wget -O - http://llvm.org/apt/llvm-snapshot.gpg.key | apt-key add -
RUN apt-get update && apt-get -y install clang-format-3.8
# Prepare ccache
diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal
index 6b2764298c..d220365fc0 100644
--- a/tools/doxygen/Doxyfile.c++.internal
+++ b/tools/doxygen/Doxyfile.c++.internal
@@ -945,6 +945,8 @@ src/core/lib/channel/handshaker_registry.h \
src/core/lib/compression/algorithm_metadata.h \
src/core/lib/compression/message_compress.h \
src/core/lib/compression/stream_compression.h \
+src/core/lib/compression/stream_compression_gzip.h \
+src/core/lib/compression/stream_compression_identity.h \
src/core/lib/debug/stats.h \
src/core/lib/debug/stats_data.h \
src/core/lib/debug/trace.h \
diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal
index 364a4f7e2f..430566bfbe 100644
--- a/tools/doxygen/Doxyfile.core.internal
+++ b/tools/doxygen/Doxyfile.core.internal
@@ -1080,6 +1080,10 @@ src/core/lib/compression/message_compress.c \
src/core/lib/compression/message_compress.h \
src/core/lib/compression/stream_compression.c \
src/core/lib/compression/stream_compression.h \
+src/core/lib/compression/stream_compression_gzip.c \
+src/core/lib/compression/stream_compression_gzip.h \
+src/core/lib/compression/stream_compression_identity.c \
+src/core/lib/compression/stream_compression_identity.h \
src/core/lib/debug/stats.c \
src/core/lib/debug/stats.h \
src/core/lib/debug/stats_data.c \
diff --git a/tools/gce/linux_performance_worker_init.sh b/tools/gce/linux_performance_worker_init.sh
index 88d8de7402..8d900f1d16 100755
--- a/tools/gce/linux_performance_worker_init.sh
+++ b/tools/gce/linux_performance_worker_init.sh
@@ -72,6 +72,7 @@ sudo apt-get install -y netperf
sudo apt-get install -y libgflags-dev libgtest-dev libc++-dev clang
# Python dependencies
+sudo pip install --upgrade pip==9.0.1
sudo pip install tabulate
sudo pip install google-api-python-client
sudo pip install virtualenv
diff --git a/tools/github/pr_latency.py b/tools/github/pr_latency.py
new file mode 100644
index 0000000000..5d635835e5
--- /dev/null
+++ b/tools/github/pr_latency.py
@@ -0,0 +1,163 @@
+#!/usr/bin/env python
+# Copyright 2017 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Measure the time between PR creation and completion of all tests.
+
+You'll need a github API token to avoid being rate-limited. See
+https://help.github.com/articles/creating-a-personal-access-token-for-the-command-line/
+
+This script goes over the most recent 100 pull requests. For PRs with a single
+commit, it uses the PR's creation as the initial time; othewise, it uses the
+date of the last commit. This is somewhat fragile, and imposed by the fact that
+GitHub reports a PR's updated timestamp for any event that modifies the PR (e.g.
+comments), not just the addition of new commits.
+
+In addition, it ignores latencies greater than five hours, as that's likely due
+to a manual re-run of tests.
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import json
+import logging
+import pprint
+import urllib2
+
+from datetime import datetime, timedelta
+
+logging.basicConfig(format='%(asctime)s %(message)s')
+
+PRS = 'https://api.github.com/repos/grpc/grpc/pulls?state=open&per_page=100'
+COMMITS = 'https://api.github.com/repos/grpc/grpc/pulls/{pr_number}/commits'
+
+
+def gh(url):
+ request = urllib2.Request(url)
+ if TOKEN:
+ request.add_header('Authorization', 'token {}'.format(TOKEN))
+ response = urllib2.urlopen(request)
+ return response.read()
+
+
+def print_csv_header():
+ print('pr,base_time,test_time,latency_seconds,successes,failures,errors')
+
+
+def output(pr, base_time, test_time, diff_time, successes, failures, errors, mode='human'):
+ if mode == 'human':
+ print("PR #{} base time: {} UTC, Tests completed at: {} UTC. Latency: {}."
+ "\n\tSuccesses: {}, Failures: {}, Errors: {}".format(
+ pr, base_time, test_time, diff_time, successes, failures, errors))
+ elif mode == 'csv':
+ print(','.join([str(pr), str(base_time),
+ str(test_time), str(int((test_time-base_time).total_seconds())),
+ str(successes), str(failures), str(errors)]))
+
+
+def parse_timestamp(datetime_str):
+ return datetime.strptime(datetime_str, '%Y-%m-%dT%H:%M:%SZ')
+
+
+def to_posix_timestamp(dt):
+ return str((dt - datetime(1970, 1, 1)).total_seconds())
+
+
+def get_pr_data():
+ latest_prs = json.loads(gh(PRS))
+ res = [{'number': pr['number'],
+ 'created_at': parse_timestamp(pr['created_at']),
+ 'updated_at': parse_timestamp(pr['updated_at']),
+ 'statuses_url': pr['statuses_url']}
+ for pr in latest_prs]
+ return res
+
+
+def get_commits_data(pr_number):
+ commits = json.loads(gh(COMMITS.format(pr_number=pr_number)))
+ return {'num_commits': len(commits),
+ 'most_recent_date': parse_timestamp(commits[-1]['commit']['author']['date'])}
+
+
+def get_status_data(statuses_url, system):
+ status_url = statuses_url.replace('statuses', 'status')
+ statuses = json.loads(gh(status_url + '?per_page=100'))
+ successes = 0
+ failures = 0
+ errors = 0
+ latest_datetime = None
+ if not statuses: return None
+ if system == 'kokoro': string_in_target_url = 'kokoro'
+ elif system == 'jenkins': string_in_target_url = 'grpc-testing'
+ for status in statuses['statuses']:
+ if not status['target_url'] or string_in_target_url not in status['target_url']: continue # Ignore jenkins
+ if status['state'] == 'pending': return None
+ elif status['state'] == 'success': successes += 1
+ elif status['state'] == 'failure': failures += 1
+ elif status['state'] == 'error': errors += 1
+ if not latest_datetime:
+ latest_datetime = parse_timestamp(status['updated_at'])
+ else:
+ latest_datetime = max(latest_datetime, parse_timestamp(status['updated_at']))
+ # First status is the most recent one.
+ if any([successes, failures, errors]) and sum([successes, failures, errors]) > 15:
+ return {'latest_datetime': latest_datetime,
+ 'successes': successes,
+ 'failures': failures,
+ 'errors': errors}
+ else: return None
+
+
+def build_args_parser():
+ import argparse
+ parser = argparse.ArgumentParser()
+ parser.add_argument('--format', type=str, choices=['human', 'csv'],
+ default='human',
+ help='Output format: are you a human or a machine?')
+ parser.add_argument('--system', type=str, choices=['jenkins', 'kokoro'],
+ required=True, help='Consider only the given CI system')
+ parser.add_argument('--token', type=str, default='',
+ help='GitHub token to use its API with a higher rate limit')
+ return parser
+
+
+def main():
+ import sys
+ global TOKEN
+ args_parser = build_args_parser()
+ args = args_parser.parse_args()
+ TOKEN = args.token
+ if args.format == 'csv': print_csv_header()
+ for pr_data in get_pr_data():
+ commit_data = get_commits_data(pr_data['number'])
+ # PR with a single commit -> use the PRs creation time.
+ # else -> use the latest commit's date.
+ base_timestamp = pr_data['updated_at']
+ if commit_data['num_commits'] > 1:
+ base_timestamp = commit_data['most_recent_date']
+ else:
+ base_timestamp = pr_data['created_at']
+ last_status = get_status_data(pr_data['statuses_url'], args.system)
+ if last_status:
+ diff = last_status['latest_datetime'] - base_timestamp
+ if diff < timedelta(hours=5):
+ output(pr_data['number'], base_timestamp, last_status['latest_datetime'],
+ diff, last_status['successes'], last_status['failures'],
+ last_status['errors'], mode=args.format)
+
+
+if __name__ == '__main__':
+ main()
diff --git a/tools/internal_ci/linux/grpc_performance_profile_daily.sh b/tools/internal_ci/linux/grpc_performance_profile_daily.sh
index 25523e21b8..34d41bc04c 100755
--- a/tools/internal_ci/linux/grpc_performance_profile_daily.sh
+++ b/tools/internal_ci/linux/grpc_performance_profile_daily.sh
@@ -22,6 +22,8 @@ source tools/internal_ci/helper_scripts/prepare_build_linux_perf_rc
CPUS=`python -c 'import multiprocessing; print multiprocessing.cpu_count()'`
+./tools/run_tests/start_port_server.py || true
+
make CONFIG=opt memory_profile_test memory_profile_client memory_profile_server -j $CPUS
bins/opt/memory_profile_test
bq load microbenchmarks.memory memory_usage.csv
diff --git a/tools/interop_matrix/README.md b/tools/interop_matrix/README.md
index f92dc690e2..c2f354399f 100644
--- a/tools/interop_matrix/README.md
+++ b/tools/interop_matrix/README.md
@@ -5,6 +5,21 @@ This directory contains scripts that facilitate building and running gRPC tests
The setup builds gRPC docker images for each language/runtime and upload it to Google Container Registry (GCR). These images, encapsulating gRPC stack
from specific releases/tag, are used to test version compatiblity between gRPC release versions.
+## Step-by-step instructions for adding a new release to compatibility test
+We have continuous nightly test setup to test gRPC backward compatibility between old clients and latest server. When a gRPC developer creates a new gRPC release, s/he is also responsible to add the just-released gRPC client to the nightly test. The steps are:
+- Add (or update) an entry in ./client_matrix.py file to reference the github tag for the release.
+- Build new client docker image(s). For example, for java release `v1.9.9`, do
+ - `tools/interop_matrix/create_matrix_images.py --git_checkout --release=v1.9.9 --language=java`
+- Verify that the new docker image was built successfully and uploaded to GCR. For example,
+ - `gcloud beta container images list-tags gcr.io/grpc-testing/grpc_interop_java_oracle8`
+ - should show an image entry with tag `v1.9.9`.
+- Verify the just-created docker client image would pass backward compatibility test (it should). For example,
+ - `gcloud docker -- pull gcr.io/grpc-testing/grpc_interop_java_oracle8:v1.9.9` followed by
+ - `docker_image=gcr.io/grpc-testing/grpc_interop_java_oracle8:v1.9.9 ./testcases/java__master`
+- git commit the change and merge it to upstream/master.
+- (Optional) clean up the tmp directory to where grpc source is cloned at `/export/hda3/tmp/grpc_matrix/`.
+For more details on each step, refer to sections below.
+
## Instructions for creating GCR images
- Edit `./client_matrix.py` to include desired gRPC release.
- Run `tools/interop_matrix/create_matrix_images.py`. Useful options:
@@ -45,3 +60,4 @@ For example:
Note:
- File path starting with `tools/` or `template/` are relative to the grpc repo root dir. File path starting with `./` are relative to current directory (`tools/interop_matrix`).
+- Creating and referencing images in GCR require read and write permission to Google Container Registry path gcr.io/grpc-testing.
diff --git a/tools/jenkins/run_full_performance.sh b/tools/jenkins/run_full_performance.sh
index 0f101f6da9..a9661c7e26 100755
--- a/tools/jenkins/run_full_performance.sh
+++ b/tools/jenkins/run_full_performance.sh
@@ -21,7 +21,7 @@ cd $(dirname $0)/../..
# run 8core client vs 8core server
tools/run_tests/run_performance_tests.py \
- -l c++ csharp node ruby java python go node_express \
+ -l c++ csharp node ruby java python go node_express php \
--netperf \
--category scalable \
--bq_result_table performance_test.performance_experiment \
diff --git a/tools/jenkins/run_performance_profile_daily.sh b/tools/jenkins/run_performance_profile_daily.sh
index 04a2464aee..48d82a9b7f 100755
--- a/tools/jenkins/run_performance_profile_daily.sh
+++ b/tools/jenkins/run_performance_profile_daily.sh
@@ -29,4 +29,6 @@ fi
BENCHMARKS_TO_RUN="bm_fullstack_unary_ping_pong bm_fullstack_streaming_ping_pong bm_fullstack_streaming_pump bm_closure bm_cq bm_call_create bm_error bm_chttp2_hpack bm_chttp2_transport bm_pollset bm_metadata"
+./tools/run_tests/start_port_server.py || true
+
$PYTHON tools/run_tests/run_microbenchmark.py --collect summary perf latency -b $BENCHMARKS_TO_RUN
diff --git a/tools/profiling/microbenchmarks/bm_diff/bm_main.py b/tools/profiling/microbenchmarks/bm_diff/bm_main.py
index 5aa11ac391..516d110b97 100755
--- a/tools/profiling/microbenchmarks/bm_diff/bm_main.py
+++ b/tools/profiling/microbenchmarks/bm_diff/bm_main.py
@@ -23,6 +23,7 @@ import bm_diff
import sys
import os
+import random
import argparse
import multiprocessing
import subprocess
@@ -32,6 +33,12 @@ sys.path.append(
os.path.dirname(sys.argv[0]), '..', '..', 'run_tests', 'python_utils'))
import comment_on_pr
+sys.path.append(
+ os.path.join(
+ os.path.dirname(sys.argv[0]), '..', '..', '..', 'run_tests',
+ 'python_utils'))
+import jobset
+
def _args():
argp = argparse.ArgumentParser(
@@ -125,8 +132,13 @@ def main(args):
subprocess.check_call(['git', 'checkout', where_am_i])
subprocess.check_call(['git', 'submodule', 'update'])
- bm_run.run('new', args.benchmarks, args.jobs, args.loops, args.regex, args.counters)
- bm_run.run(old, args.benchmarks, args.jobs, args.loops, args.regex, args.counters)
+ jobs_list = []
+ jobs_list += bm_run.create_jobs('new', args.benchmarks, args.loops, args.regex, args.counters)
+ jobs_list += bm_run.create_jobs(old, args.benchmarks, args.loops, args.regex, args.counters)
+
+ # shuffle all jobs to eliminate noise from GCE CPU drift
+ random.shuffle(jobs_list, random.SystemRandom().random)
+ jobset.run(jobs_list, maxjobs=args.jobs)
diff, note = bm_diff.diff(args.benchmarks, args.loops, args.regex, args.track, old,
'new', args.counters)
diff --git a/tools/profiling/microbenchmarks/bm_diff/bm_run.py b/tools/profiling/microbenchmarks/bm_diff/bm_run.py
index 206f7c5845..81db5a226a 100755
--- a/tools/profiling/microbenchmarks/bm_diff/bm_run.py
+++ b/tools/profiling/microbenchmarks/bm_diff/bm_run.py
@@ -95,11 +95,12 @@ def _collect_bm_data(bm, cfg, name, regex, idx, loops):
shortname='%s %s %s %s %d/%d' % (bm, line, cfg, name, idx + 1,
loops),
verbose_success=True,
+ cpu_cost=2,
timeout_seconds=60 * 60)) # one hour
return jobs_list
-def run(name, benchmarks, jobs, loops, regex, counters):
+def create_jobs(name, benchmarks, loops, regex, counters):
jobs_list = []
for loop in range(0, loops):
for bm in benchmarks:
@@ -108,9 +109,11 @@ def run(name, benchmarks, jobs, loops, regex, counters):
jobs_list += _collect_bm_data(bm, 'counters', name, regex, loop,
loops)
random.shuffle(jobs_list, random.SystemRandom().random)
- jobset.run(jobs_list, maxjobs=jobs)
+ return jobs_list
if __name__ == '__main__':
args = _args()
- run(args.name, args.benchmarks, args.jobs, args.loops, args.regex, args.counters)
+ jobs_list = create_jobs(args.name, args.benchmarks, args.loops,
+ args.regex, args.counters)
+ jobset.run(jobs_list, maxjobs=args.jobs)
diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json
index 4e5e9f4a7f..5e825e2082 100644
--- a/tools/run_tests/generated/sources_and_headers.json
+++ b/tools/run_tests/generated/sources_and_headers.json
@@ -8007,6 +8007,8 @@
"src/core/lib/compression/compression.c",
"src/core/lib/compression/message_compress.c",
"src/core/lib/compression/stream_compression.c",
+ "src/core/lib/compression/stream_compression_gzip.c",
+ "src/core/lib/compression/stream_compression_identity.c",
"src/core/lib/debug/stats.c",
"src/core/lib/debug/stats_data.c",
"src/core/lib/http/format_request.c",
@@ -8160,6 +8162,8 @@
"src/core/lib/compression/algorithm_metadata.h",
"src/core/lib/compression/message_compress.h",
"src/core/lib/compression/stream_compression.h",
+ "src/core/lib/compression/stream_compression_gzip.h",
+ "src/core/lib/compression/stream_compression_identity.h",
"src/core/lib/debug/stats.h",
"src/core/lib/debug/stats_data.h",
"src/core/lib/http/format_request.h",
@@ -8292,6 +8296,8 @@
"src/core/lib/compression/algorithm_metadata.h",
"src/core/lib/compression/message_compress.h",
"src/core/lib/compression/stream_compression.h",
+ "src/core/lib/compression/stream_compression_gzip.h",
+ "src/core/lib/compression/stream_compression_identity.h",
"src/core/lib/debug/stats.h",
"src/core/lib/debug/stats_data.h",
"src/core/lib/http/format_request.h",
diff --git a/tools/run_tests/helper_scripts/build_python.sh b/tools/run_tests/helper_scripts/build_python.sh
index be650553dc..e3620821f3 100755
--- a/tools/run_tests/helper_scripts/build_python.sh
+++ b/tools/run_tests/helper_scripts/build_python.sh
@@ -152,7 +152,7 @@ pip_install_dir() {
cd $PWD
}
-$VENV_PYTHON -m pip install --upgrade pip
+$VENV_PYTHON -m pip install --upgrade pip==9.0.1
$VENV_PYTHON -m pip install setuptools
$VENV_PYTHON -m pip install cython
$VENV_PYTHON -m pip install six enum34 protobuf futures
diff --git a/tools/run_tests/helper_scripts/run_grpc-node.sh b/tools/run_tests/helper_scripts/run_grpc-node.sh
new file mode 100755
index 0000000000..25f149f579
--- /dev/null
+++ b/tools/run_tests/helper_scripts/run_grpc-node.sh
@@ -0,0 +1,28 @@
+#!/bin/bash
+# Copyright 2015 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# This script runs grpc/grpc-node tests with their grpc submodule updated
+# to this reference
+
+# cd to gRPC root directory
+cd $(dirname $0)/../../..
+
+CURRENT_COMMIT=$(git rev-parse --verify HEAD)
+
+rm -rf ./../grpc-node
+git clone --recursive https://github.com/grpc/grpc-node ./../grpc-node
+cd ./../grpc-node
+
+./test-grpc-submodule.sh $CURRENT_COMMIT
diff --git a/tools/run_tests/performance/massage_qps_stats.py b/tools/run_tests/performance/massage_qps_stats.py
index a1a6aa4526..ebbfe6c26c 100644
--- a/tools/run_tests/performance/massage_qps_stats.py
+++ b/tools/run_tests/performance/massage_qps_stats.py
@@ -107,8 +107,6 @@ def massage_qps_stats(scenario_result):
stats["core_executor_wakeup_initiated"] = massage_qps_stats_helpers.counter(core_stats, "executor_wakeup_initiated")
stats["core_executor_queue_drained"] = massage_qps_stats_helpers.counter(core_stats, "executor_queue_drained")
stats["core_executor_push_retries"] = massage_qps_stats_helpers.counter(core_stats, "executor_push_retries")
- stats["core_executor_threads_created"] = massage_qps_stats_helpers.counter(core_stats, "executor_threads_created")
- stats["core_executor_threads_used"] = massage_qps_stats_helpers.counter(core_stats, "executor_threads_used")
stats["core_server_requested_calls"] = massage_qps_stats_helpers.counter(core_stats, "server_requested_calls")
stats["core_server_slowpath_requests_queued"] = massage_qps_stats_helpers.counter(core_stats, "server_slowpath_requests_queued")
h = massage_qps_stats_helpers.histogram(core_stats, "call_initial_size")
@@ -183,12 +181,6 @@ def massage_qps_stats(scenario_result):
stats["core_http2_send_flowctl_per_write_50p"] = massage_qps_stats_helpers.percentile(h.buckets, 50, h.boundaries)
stats["core_http2_send_flowctl_per_write_95p"] = massage_qps_stats_helpers.percentile(h.buckets, 95, h.boundaries)
stats["core_http2_send_flowctl_per_write_99p"] = massage_qps_stats_helpers.percentile(h.buckets, 99, h.boundaries)
- h = massage_qps_stats_helpers.histogram(core_stats, "executor_closures_per_wakeup")
- stats["core_executor_closures_per_wakeup"] = ",".join("%f" % x for x in h.buckets)
- stats["core_executor_closures_per_wakeup_bkts"] = ",".join("%f" % x for x in h.boundaries)
- stats["core_executor_closures_per_wakeup_50p"] = massage_qps_stats_helpers.percentile(h.buckets, 50, h.boundaries)
- stats["core_executor_closures_per_wakeup_95p"] = massage_qps_stats_helpers.percentile(h.buckets, 95, h.boundaries)
- stats["core_executor_closures_per_wakeup_99p"] = massage_qps_stats_helpers.percentile(h.buckets, 99, h.boundaries)
h = massage_qps_stats_helpers.histogram(core_stats, "server_cqs_checked")
stats["core_server_cqs_checked"] = ",".join("%f" % x for x in h.buckets)
stats["core_server_cqs_checked_bkts"] = ",".join("%f" % x for x in h.boundaries)
diff --git a/tools/run_tests/performance/scenario_result_schema.json b/tools/run_tests/performance/scenario_result_schema.json
index be52166915..169221d18c 100644
--- a/tools/run_tests/performance/scenario_result_schema.json
+++ b/tools/run_tests/performance/scenario_result_schema.json
@@ -547,16 +547,6 @@
},
{
"mode": "NULLABLE",
- "name": "core_executor_threads_created",
- "type": "INTEGER"
- },
- {
- "mode": "NULLABLE",
- "name": "core_executor_threads_used",
- "type": "INTEGER"
- },
- {
- "mode": "NULLABLE",
"name": "core_server_requested_calls",
"type": "INTEGER"
},
@@ -867,31 +857,6 @@
},
{
"mode": "NULLABLE",
- "name": "core_executor_closures_per_wakeup",
- "type": "STRING"
- },
- {
- "mode": "NULLABLE",
- "name": "core_executor_closures_per_wakeup_bkts",
- "type": "STRING"
- },
- {
- "mode": "NULLABLE",
- "name": "core_executor_closures_per_wakeup_50p",
- "type": "FLOAT"
- },
- {
- "mode": "NULLABLE",
- "name": "core_executor_closures_per_wakeup_95p",
- "type": "FLOAT"
- },
- {
- "mode": "NULLABLE",
- "name": "core_executor_closures_per_wakeup_99p",
- "type": "FLOAT"
- },
- {
- "mode": "NULLABLE",
"name": "core_server_cqs_checked",
"type": "STRING"
},
@@ -1379,16 +1344,6 @@
},
{
"mode": "NULLABLE",
- "name": "core_executor_threads_created",
- "type": "INTEGER"
- },
- {
- "mode": "NULLABLE",
- "name": "core_executor_threads_used",
- "type": "INTEGER"
- },
- {
- "mode": "NULLABLE",
"name": "core_server_requested_calls",
"type": "INTEGER"
},
@@ -1699,31 +1654,6 @@
},
{
"mode": "NULLABLE",
- "name": "core_executor_closures_per_wakeup",
- "type": "STRING"
- },
- {
- "mode": "NULLABLE",
- "name": "core_executor_closures_per_wakeup_bkts",
- "type": "STRING"
- },
- {
- "mode": "NULLABLE",
- "name": "core_executor_closures_per_wakeup_50p",
- "type": "FLOAT"
- },
- {
- "mode": "NULLABLE",
- "name": "core_executor_closures_per_wakeup_95p",
- "type": "FLOAT"
- },
- {
- "mode": "NULLABLE",
- "name": "core_executor_closures_per_wakeup_99p",
- "type": "FLOAT"
- },
- {
- "mode": "NULLABLE",
"name": "core_server_cqs_checked",
"type": "STRING"
},
diff --git a/tools/run_tests/python_utils/filter_pull_request_tests.py b/tools/run_tests/python_utils/filter_pull_request_tests.py
index f99143fdd7..393aef8662 100644
--- a/tools/run_tests/python_utils/filter_pull_request_tests.py
+++ b/tools/run_tests/python_utils/filter_pull_request_tests.py
@@ -47,7 +47,7 @@ class TestSuite:
_CORE_TEST_SUITE = TestSuite(['c'])
_CPP_TEST_SUITE = TestSuite(['c++'])
_CSHARP_TEST_SUITE = TestSuite(['csharp'])
-_NODE_TEST_SUITE = TestSuite(['node'])
+_NODE_TEST_SUITE = TestSuite(['node', 'grpc-node'])
_OBJC_TEST_SUITE = TestSuite(['objc'])
_PHP_TEST_SUITE = TestSuite(['php', 'php7'])
_PYTHON_TEST_SUITE = TestSuite(['python'])
diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py
index f661aed4e2..5750b90619 100755
--- a/tools/run_tests/run_tests.py
+++ b/tools/run_tests/run_tests.py
@@ -444,6 +444,67 @@ class CLanguage(object):
return self.make_target
+# This tests Node on grpc/grpc-node and will become the standard for Node testing
+class RemoteNodeLanguage(object):
+
+ def __init__(self):
+ self.platform = platform_string()
+
+ def configure(self, config, args):
+ self.config = config
+ self.args = args
+ # Note: electron ABI only depends on major and minor version, so that's all
+ # we should specify in the compiler argument
+ _check_compiler(self.args.compiler, ['default', 'node0.12',
+ 'node4', 'node5', 'node6',
+ 'node7', 'node8',
+ 'electron1.3', 'electron1.6'])
+ if self.args.compiler == 'default':
+ self.runtime = 'node'
+ self.node_version = '8'
+ else:
+ if self.args.compiler.startswith('electron'):
+ self.runtime = 'electron'
+ self.node_version = self.args.compiler[8:]
+ else:
+ self.runtime = 'node'
+ # Take off the word "node"
+ self.node_version = self.args.compiler[4:]
+
+ # TODO: update with Windows/electron scripts when available for grpc/grpc-node
+ def test_specs(self):
+ if self.platform == 'windows':
+ return [self.config.job_spec(['tools\\run_tests\\helper_scripts\\run_node.bat'])]
+ else:
+ return [self.config.job_spec(['tools/run_tests/helper_scripts/run_grpc-node.sh'],
+ None,
+ environ=_FORCE_ENVIRON_FOR_WRAPPERS)]
+
+ def pre_build_steps(self):
+ return []
+
+ def make_targets(self):
+ return []
+
+ def make_options(self):
+ return []
+
+ def build_steps(self):
+ return []
+
+ def post_tests_steps(self):
+ return []
+
+ def makefile_name(self):
+ return 'Makefile'
+
+ def dockerfile_dir(self):
+ return 'tools/dockerfile/test/node_jessie_%s' % _docker_arch_suffix(self.args.arch)
+
+ def __str__(self):
+ return 'grpc-node'
+
+
class NodeLanguage(object):
def __init__(self):
@@ -717,6 +778,9 @@ class PythonLanguage(object):
return (pypy32_config,)
elif args.compiler == 'python_alpine':
return (python27_config,)
+ elif args.compiler == 'all_the_cpythons':
+ return (python27_config, python34_config, python35_config,
+ python36_config,)
else:
raise Exception('Compiler %s not supported.' % args.compiler)
@@ -1064,6 +1128,7 @@ with open('tools/run_tests/generated/configs.json') as f:
_LANGUAGES = {
'c++': CLanguage('cxx', 'c++'),
'c': CLanguage('c', 'c'),
+ 'grpc-node': RemoteNodeLanguage(),
'node': NodeLanguage(),
'node_express': NodeExpressLanguage(),
'php': PhpLanguage(),
@@ -1214,7 +1279,7 @@ argp.add_argument('--compiler',
choices=['default',
'gcc4.4', 'gcc4.6', 'gcc4.8', 'gcc4.9', 'gcc5.3', 'gcc_musl',
'clang3.4', 'clang3.5', 'clang3.6', 'clang3.7',
- 'python2.7', 'python3.4', 'python3.5', 'python3.6', 'pypy', 'pypy3', 'python_alpine',
+ 'python2.7', 'python3.4', 'python3.5', 'python3.6', 'pypy', 'pypy3', 'python_alpine', 'all_the_cpythons',
'node0.12', 'node4', 'node5', 'node6', 'node7', 'node8',
'electron1.3', 'electron1.6',
'coreclr',
diff --git a/tools/run_tests/run_tests_matrix.py b/tools/run_tests/run_tests_matrix.py
index 2417c5ae55..34d839e0b8 100755
--- a/tools/run_tests/run_tests_matrix.py
+++ b/tools/run_tests/run_tests_matrix.py
@@ -169,7 +169,7 @@ def _create_test_jobs(extra_args=[], inner_jobs=_DEFAULT_INNER_JOBS):
inner_jobs=inner_jobs,
timeout_seconds=_CPP_RUNTESTS_TIMEOUT)
- test_jobs += _generate_jobs(languages=['ruby', 'php'],
+ test_jobs += _generate_jobs(languages=['grpc-node', 'ruby', 'php'],
configs=['dbg', 'opt'],
platforms=['linux', 'macos'],
labels=['basictests', 'multilang'],
diff --git a/tools/run_tests/sanity/check_test_filtering.py b/tools/run_tests/sanity/check_test_filtering.py
index bea8125fb9..3ebb9389f7 100755
--- a/tools/run_tests/sanity/check_test_filtering.py
+++ b/tools/run_tests/sanity/check_test_filtering.py
@@ -25,7 +25,7 @@ sys.path.insert(0, os.path.abspath('tools/run_tests/'))
from run_tests_matrix import _create_test_jobs, _create_portability_test_jobs
import python_utils.filter_pull_request_tests as filter_pull_request_tests
-_LIST_OF_LANGUAGE_LABELS = ['c', 'c++', 'csharp', 'node', 'objc', 'php', 'php7', 'python', 'ruby']
+_LIST_OF_LANGUAGE_LABELS = ['c', 'c++', 'csharp', 'grpc-node', 'node', 'objc', 'php', 'php7', 'python', 'ruby']
_LIST_OF_PLATFORM_LABELS = ['linux', 'macos', 'windows']
class TestFilteringTest(unittest.TestCase):