diff options
author | 2017-09-29 11:18:26 -0700 | |
---|---|---|
committer | 2017-09-29 11:18:26 -0700 | |
commit | 1e868f0f9539925e51aa52269c848082d23b7c4e (patch) | |
tree | 49fd965a5f58e2605864df54a1d70621367eee36 /tools | |
parent | 710334577ce1b2de94f656ec0762eeba6effd29b (diff) | |
parent | 903f06fe3f8b1b971f1b633dff45488ca68c6708 (diff) |
Merge github.com:grpc/grpc into flowctl+millis
Diffstat (limited to 'tools')
52 files changed, 373 insertions, 138 deletions
diff --git a/tools/codegen/core/gen_stats_data.py b/tools/codegen/core/gen_stats_data.py index 10ad0cc831..8359734c84 100755 --- a/tools/codegen/core/gen_stats_data.py +++ b/tools/codegen/core/gen_stats_data.py @@ -147,8 +147,7 @@ def gen_bucket_code(histogram): shift_data = find_ideal_shift(code_bounds[first_nontrivial:], 256 * histogram.buckets) #print first_nontrivial, shift_data, bounds #if shift_data is not None: print [hex(x >> shift_data[0]) for x in code_bounds[first_nontrivial:]] - code = '\n/* Automatically generated by tools/codegen/core/gen_stats_data.py */\n' - code += 'value = GPR_CLAMP(value, 0, %d);\n' % histogram.max + code = 'value = GPR_CLAMP(value, 0, %d);\n' % histogram.max map_table = gen_map_table(code_bounds[first_nontrivial:], shift_data) if first_nontrivial is None: code += ('GRPC_STATS_INC_HISTOGRAM((exec_ctx), GRPC_STATS_HISTOGRAM_%s, value);\n' @@ -408,3 +407,4 @@ with open('src/core/lib/debug/stats_data_bq_schema.sql', 'w') as S: for counter in inst_map['Counter']: columns.append(('%s_per_iteration' % counter.name, 'FLOAT')) print >>S, ',\n'.join('%s:%s' % x for x in columns) + diff --git a/tools/distrib/pylint_code.sh b/tools/distrib/pylint_code.sh index 3c9235b548..7175f1e15b 100755 --- a/tools/distrib/pylint_code.sh +++ b/tools/distrib/pylint_code.sh @@ -29,7 +29,7 @@ VIRTUALENV=python_pylint_venv virtualenv $VIRTUALENV PYTHON=$(realpath $VIRTUALENV/bin/python) -$PYTHON -m pip install --upgrade pip +$PYTHON -m pip install --upgrade pip==9.0.1 $PYTHON -m pip install pylint==1.6.5 for dir in "${DIRS[@]}"; do diff --git a/tools/distrib/python/docgen.py b/tools/distrib/python/docgen.py index 6f6d43c7d9..1822e51d09 100755 --- a/tools/distrib/python/docgen.py +++ b/tools/distrib/python/docgen.py @@ -60,7 +60,7 @@ environment.update({ subprocess_arguments_list = [ {'args': ['virtualenv', VIRTUALENV_DIR], 'env': environment}, - {'args': [VIRTUALENV_PIP_PATH, 'install', '--upgrade', 'pip'], + {'args': [VIRTUALENV_PIP_PATH, 'install', '--upgrade', 'pip==9.0.1'], 'env': environment}, {'args': [VIRTUALENV_PIP_PATH, 'install', '-r', REQUIREMENTS_PATH], 'env': environment}, diff --git a/tools/distrib/yapf_code.sh b/tools/distrib/yapf_code.sh index dbb6b5c41f..e5beb70117 100755 --- a/tools/distrib/yapf_code.sh +++ b/tools/distrib/yapf_code.sh @@ -33,7 +33,7 @@ VIRTUALENV=yapf_virtual_environment virtualenv $VIRTUALENV PYTHON=$(realpath "${VIRTUALENV}/bin/python") -$PYTHON -m pip install --upgrade pip +$PYTHON -m pip install --upgrade pip==9.0.1 $PYTHON -m pip install --upgrade futures $PYTHON -m pip install yapf==0.16.0 diff --git a/tools/dockerfile/grpc_clang_format/Dockerfile b/tools/dockerfile/grpc_clang_format/Dockerfile index 647cb52a7b..dff07feeb6 100644 --- a/tools/dockerfile/grpc_clang_format/Dockerfile +++ b/tools/dockerfile/grpc_clang_format/Dockerfile @@ -12,12 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM ubuntu:15.10 +FROM debian:jessie RUN apt-get update && apt-get -y install wget -RUN echo deb http://llvm.org/apt/wily/ llvm-toolchain-wily-3.8 main >> /etc/apt/sources.list -RUN echo deb-src http://llvm.org/apt/wily/ llvm-toolchain-wily-3.8 main >> /etc/apt/sources.list -RUN wget -O - http://llvm.org/apt/llvm-snapshot.gpg.key| apt-key add - +RUN echo "deb http://llvm.org/apt/jessie/ llvm-toolchain-jessie-3.8 main" >> /etc/apt/sources.list +RUN echo "deb-src http://llvm.org/apt/jessie/ llvm-toolchain-jessie-3.8 main" >> /etc/apt/sources.list +RUN wget -O - http://llvm.org/apt/llvm-snapshot.gpg.key | apt-key add - RUN apt-get update && apt-get -y install clang-format-3.8 ADD clang_format_all_the_things.sh / diff --git a/tools/dockerfile/interoptest/grpc_interop_csharp/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_csharp/Dockerfile index dbf58023c5..ea82476b65 100644 --- a/tools/dockerfile/interoptest/grpc_interop_csharp/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_csharp/Dockerfile @@ -60,7 +60,7 @@ RUN apt-get update && apt-get install -y \ python-pip # Install Python packages from PyPI -RUN pip install pip --upgrade +RUN pip install --upgrade pip==9.0.1 RUN pip install virtualenv RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 diff --git a/tools/dockerfile/interoptest/grpc_interop_csharpcoreclr/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_csharpcoreclr/Dockerfile index 4ccfbc43c3..56b8be89ec 100644 --- a/tools/dockerfile/interoptest/grpc_interop_csharpcoreclr/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_csharpcoreclr/Dockerfile @@ -59,7 +59,7 @@ RUN apt-get update && apt-get install -y \ python-pip # Install Python packages from PyPI -RUN pip install pip --upgrade +RUN pip install --upgrade pip==9.0.1 RUN pip install virtualenv RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 diff --git a/tools/dockerfile/interoptest/grpc_interop_cxx/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_cxx/Dockerfile index 7cfe98cbc8..38d377ce2f 100644 --- a/tools/dockerfile/interoptest/grpc_interop_cxx/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_cxx/Dockerfile @@ -60,7 +60,7 @@ RUN apt-get update && apt-get install -y \ python-pip # Install Python packages from PyPI -RUN pip install pip --upgrade +RUN pip install --upgrade pip==9.0.1 RUN pip install virtualenv RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 diff --git a/tools/dockerfile/interoptest/grpc_interop_go/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_go/Dockerfile index febe2fa251..73c41a4838 100644 --- a/tools/dockerfile/interoptest/grpc_interop_go/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_go/Dockerfile @@ -28,7 +28,7 @@ RUN apt-get update && apt-get install -y \ python-pip # Install Python packages from PyPI -RUN pip install pip --upgrade +RUN pip install --upgrade pip==9.0.1 RUN pip install virtualenv RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 diff --git a/tools/dockerfile/interoptest/grpc_interop_go1.7/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_go1.7/Dockerfile index 3a516cbb62..7c083de365 100644 --- a/tools/dockerfile/interoptest/grpc_interop_go1.7/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_go1.7/Dockerfile @@ -28,7 +28,7 @@ RUN apt-get update && apt-get install -y \ python-pip # Install Python packages from PyPI -RUN pip install pip --upgrade +RUN pip install --upgrade pip==9.0.1 RUN pip install virtualenv RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 diff --git a/tools/dockerfile/interoptest/grpc_interop_go1.8/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_go1.8/Dockerfile index acb640a81d..61efc18df9 100644 --- a/tools/dockerfile/interoptest/grpc_interop_go1.8/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_go1.8/Dockerfile @@ -28,7 +28,7 @@ RUN apt-get update && apt-get install -y \ python-pip # Install Python packages from PyPI -RUN pip install pip --upgrade +RUN pip install --upgrade pip==9.0.1 RUN pip install virtualenv RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 diff --git a/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile index 354b7bfdb5..278b09a666 100644 --- a/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile @@ -28,7 +28,7 @@ RUN apt-get update && apt-get install -y \ python-pip # Install Python packages from PyPI -RUN pip install pip --upgrade +RUN pip install --upgrade pip==9.0.1 RUN pip install virtualenv RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 diff --git a/tools/dockerfile/interoptest/grpc_interop_java/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_java/Dockerfile index 92a542ff76..d56632440f 100644 --- a/tools/dockerfile/interoptest/grpc_interop_java/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_java/Dockerfile @@ -43,7 +43,7 @@ RUN apt-get update && apt-get install -y \ python-pip # Install Python packages from PyPI -RUN pip install pip --upgrade +RUN pip install --upgrade pip==9.0.1 RUN pip install virtualenv RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 diff --git a/tools/dockerfile/interoptest/grpc_interop_java_oracle8/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_java_oracle8/Dockerfile index 92a542ff76..d56632440f 100644 --- a/tools/dockerfile/interoptest/grpc_interop_java_oracle8/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_java_oracle8/Dockerfile @@ -43,7 +43,7 @@ RUN apt-get update && apt-get install -y \ python-pip # Install Python packages from PyPI -RUN pip install pip --upgrade +RUN pip install --upgrade pip==9.0.1 RUN pip install virtualenv RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 diff --git a/tools/dockerfile/interoptest/grpc_interop_node/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_node/Dockerfile index 4343d56cb4..f4c3e4103e 100644 --- a/tools/dockerfile/interoptest/grpc_interop_node/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_node/Dockerfile @@ -60,7 +60,7 @@ RUN apt-get update && apt-get install -y \ python-pip # Install Python packages from PyPI -RUN pip install pip --upgrade +RUN pip install --upgrade pip==9.0.1 RUN pip install virtualenv RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 diff --git a/tools/dockerfile/interoptest/grpc_interop_python/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_python/Dockerfile index 271c6e75e9..d165307031 100644 --- a/tools/dockerfile/interoptest/grpc_interop_python/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_python/Dockerfile @@ -60,7 +60,7 @@ RUN apt-get update && apt-get install -y \ python-pip # Install Python packages from PyPI -RUN pip install pip --upgrade +RUN pip install --upgrade pip==9.0.1 RUN pip install virtualenv RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 diff --git a/tools/dockerfile/interoptest/grpc_interop_ruby/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_ruby/Dockerfile index 7bcada68e0..2217b10b8b 100644 --- a/tools/dockerfile/interoptest/grpc_interop_ruby/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_ruby/Dockerfile @@ -60,7 +60,7 @@ RUN apt-get update && apt-get install -y \ python-pip # Install Python packages from PyPI -RUN pip install pip --upgrade +RUN pip install --upgrade pip==9.0.1 RUN pip install virtualenv RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 diff --git a/tools/dockerfile/test/csharp_jessie_x64/Dockerfile b/tools/dockerfile/test/csharp_jessie_x64/Dockerfile index 40d46fcf58..3e31e67f5f 100644 --- a/tools/dockerfile/test/csharp_jessie_x64/Dockerfile +++ b/tools/dockerfile/test/csharp_jessie_x64/Dockerfile @@ -64,7 +64,7 @@ RUN apt-get update && apt-get install -y \ python-pip # Install Python packages from PyPI -RUN pip install pip --upgrade +RUN pip install --upgrade pip==9.0.1 RUN pip install virtualenv RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 diff --git a/tools/dockerfile/test/cxx_alpine_x64/Dockerfile b/tools/dockerfile/test/cxx_alpine_x64/Dockerfile index 1ae50c106f..af5e7d6f14 100644 --- a/tools/dockerfile/test/cxx_alpine_x64/Dockerfile +++ b/tools/dockerfile/test/cxx_alpine_x64/Dockerfile @@ -36,7 +36,7 @@ RUN apk update && apk add \ zip # Install Python packages from PyPI -RUN pip install pip --upgrade +RUN pip install --upgrade pip==9.0.1 RUN pip install virtualenv RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 diff --git a/tools/dockerfile/test/cxx_jessie_x64/Dockerfile b/tools/dockerfile/test/cxx_jessie_x64/Dockerfile index 888a37baca..3492dd72c5 100644 --- a/tools/dockerfile/test/cxx_jessie_x64/Dockerfile +++ b/tools/dockerfile/test/cxx_jessie_x64/Dockerfile @@ -64,7 +64,7 @@ RUN apt-get update && apt-get install -y \ python-pip # Install Python packages from PyPI -RUN pip install pip --upgrade +RUN pip install --upgrade pip==9.0.1 RUN pip install virtualenv RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 diff --git a/tools/dockerfile/test/cxx_jessie_x86/Dockerfile b/tools/dockerfile/test/cxx_jessie_x86/Dockerfile index 319f1e1889..f8cbf35d39 100644 --- a/tools/dockerfile/test/cxx_jessie_x86/Dockerfile +++ b/tools/dockerfile/test/cxx_jessie_x86/Dockerfile @@ -64,7 +64,7 @@ RUN apt-get update && apt-get install -y \ python-pip # Install Python packages from PyPI -RUN pip install pip --upgrade +RUN pip install --upgrade pip==9.0.1 RUN pip install virtualenv RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 diff --git a/tools/dockerfile/test/cxx_ubuntu1404_x64/Dockerfile b/tools/dockerfile/test/cxx_ubuntu1404_x64/Dockerfile index 61f005d9da..6966d6b8d3 100644 --- a/tools/dockerfile/test/cxx_ubuntu1404_x64/Dockerfile +++ b/tools/dockerfile/test/cxx_ubuntu1404_x64/Dockerfile @@ -64,7 +64,7 @@ RUN apt-get update && apt-get install -y \ python-pip # Install Python packages from PyPI -RUN pip install pip --upgrade +RUN pip install --upgrade pip==9.0.1 RUN pip install virtualenv RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 diff --git a/tools/dockerfile/test/cxx_ubuntu1604_x64/Dockerfile b/tools/dockerfile/test/cxx_ubuntu1604_x64/Dockerfile index f35247eccb..016034aa50 100644 --- a/tools/dockerfile/test/cxx_ubuntu1604_x64/Dockerfile +++ b/tools/dockerfile/test/cxx_ubuntu1604_x64/Dockerfile @@ -64,7 +64,7 @@ RUN apt-get update && apt-get install -y \ python-pip # Install Python packages from PyPI -RUN pip install pip --upgrade +RUN pip install --upgrade pip==9.0.1 RUN pip install virtualenv RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 diff --git a/tools/dockerfile/test/fuzzer/Dockerfile b/tools/dockerfile/test/fuzzer/Dockerfile index ce1badfeb1..50104ad7ae 100644 --- a/tools/dockerfile/test/fuzzer/Dockerfile +++ b/tools/dockerfile/test/fuzzer/Dockerfile @@ -64,7 +64,7 @@ RUN apt-get update && apt-get install -y \ python-pip # Install Python packages from PyPI -RUN pip install pip --upgrade +RUN pip install --upgrade pip==9.0.1 RUN pip install virtualenv RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 diff --git a/tools/dockerfile/test/multilang_jessie_x64/Dockerfile b/tools/dockerfile/test/multilang_jessie_x64/Dockerfile index 59fe4d8f93..1a4b681576 100644 --- a/tools/dockerfile/test/multilang_jessie_x64/Dockerfile +++ b/tools/dockerfile/test/multilang_jessie_x64/Dockerfile @@ -121,7 +121,7 @@ RUN apt-get update && apt-get install -y \ python-pip # Install Python packages from PyPI -RUN pip install pip --upgrade +RUN pip install --upgrade pip==9.0.1 RUN pip install virtualenv RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 diff --git a/tools/dockerfile/test/node_jessie_x64/Dockerfile b/tools/dockerfile/test/node_jessie_x64/Dockerfile index 103be8412b..4f18dbae79 100644 --- a/tools/dockerfile/test/node_jessie_x64/Dockerfile +++ b/tools/dockerfile/test/node_jessie_x64/Dockerfile @@ -75,7 +75,7 @@ RUN apt-get update && apt-get install -y \ python-pip # Install Python packages from PyPI -RUN pip install pip --upgrade +RUN pip install --upgrade pip==9.0.1 RUN pip install virtualenv RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 diff --git a/tools/dockerfile/test/php7_jessie_x64/Dockerfile b/tools/dockerfile/test/php7_jessie_x64/Dockerfile index f6d426bcd6..1399502840 100644 --- a/tools/dockerfile/test/php7_jessie_x64/Dockerfile +++ b/tools/dockerfile/test/php7_jessie_x64/Dockerfile @@ -75,7 +75,7 @@ RUN apt-get update && apt-get install -y \ python-pip # Install Python packages from PyPI -RUN pip install pip --upgrade +RUN pip install --upgrade pip==9.0.1 RUN pip install virtualenv RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 diff --git a/tools/dockerfile/test/php_jessie_x64/Dockerfile b/tools/dockerfile/test/php_jessie_x64/Dockerfile index ae82a8d99f..56dc6045e1 100644 --- a/tools/dockerfile/test/php_jessie_x64/Dockerfile +++ b/tools/dockerfile/test/php_jessie_x64/Dockerfile @@ -64,7 +64,7 @@ RUN apt-get update && apt-get install -y \ python-pip # Install Python packages from PyPI -RUN pip install pip --upgrade +RUN pip install --upgrade pip==9.0.1 RUN pip install virtualenv RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 diff --git a/tools/dockerfile/test/python_alpine_x64/Dockerfile b/tools/dockerfile/test/python_alpine_x64/Dockerfile index 7bd11d7407..7584ab8767 100644 --- a/tools/dockerfile/test/python_alpine_x64/Dockerfile +++ b/tools/dockerfile/test/python_alpine_x64/Dockerfile @@ -36,7 +36,7 @@ RUN apk update && apk add \ zip # Install Python packages from PyPI -RUN pip install pip --upgrade +RUN pip install --upgrade pip==9.0.1 RUN pip install virtualenv RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 diff --git a/tools/dockerfile/test/python_jessie_x64/Dockerfile b/tools/dockerfile/test/python_jessie_x64/Dockerfile index d5d781cd1a..8d89f5022f 100644 --- a/tools/dockerfile/test/python_jessie_x64/Dockerfile +++ b/tools/dockerfile/test/python_jessie_x64/Dockerfile @@ -64,7 +64,7 @@ RUN apt-get update && apt-get install -y \ python-pip # Install Python packages from PyPI -RUN pip install pip --upgrade +RUN pip install --upgrade pip==9.0.1 RUN pip install virtualenv RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 diff --git a/tools/dockerfile/test/python_pyenv_x64/Dockerfile b/tools/dockerfile/test/python_pyenv_x64/Dockerfile index 3b4ad12b6d..f8cbdc5726 100644 --- a/tools/dockerfile/test/python_pyenv_x64/Dockerfile +++ b/tools/dockerfile/test/python_pyenv_x64/Dockerfile @@ -64,7 +64,7 @@ RUN apt-get update && apt-get install -y \ python-pip # Install Python packages from PyPI -RUN pip install pip --upgrade +RUN pip install --upgrade pip==9.0.1 RUN pip install virtualenv RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 diff --git a/tools/dockerfile/test/ruby_jessie_x64/Dockerfile b/tools/dockerfile/test/ruby_jessie_x64/Dockerfile index 3d879bb0c9..5d7f80bf88 100644 --- a/tools/dockerfile/test/ruby_jessie_x64/Dockerfile +++ b/tools/dockerfile/test/ruby_jessie_x64/Dockerfile @@ -64,7 +64,7 @@ RUN apt-get update && apt-get install -y \ python-pip # Install Python packages from PyPI -RUN pip install pip --upgrade +RUN pip install --upgrade pip==9.0.1 RUN pip install virtualenv RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 diff --git a/tools/dockerfile/test/sanity/Dockerfile b/tools/dockerfile/test/sanity/Dockerfile index 44732a5ae4..487ce15e2e 100644 --- a/tools/dockerfile/test/sanity/Dockerfile +++ b/tools/dockerfile/test/sanity/Dockerfile @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM ubuntu:15.10 +FROM debian:jessie # Install Git and basic packages. RUN apt-get update && apt-get install -y \ @@ -64,7 +64,7 @@ RUN apt-get update && apt-get install -y \ python-pip # Install Python packages from PyPI -RUN pip install pip --upgrade +RUN pip install --upgrade pip==9.0.1 RUN pip install virtualenv RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 @@ -82,19 +82,28 @@ RUN pip install simplejson mako #====================================== # More sanity test dependencies (bazel) -RUN apt-get install -y openjdk-8-jdk -# Check out Bazel version 0.4.1 since this version allows running -# ./compile.sh without a local protoc dependency -# TODO(mattkwong): install dependencies to support latest Bazel version if newer -# version is needed -RUN git clone https://github.com/bazelbuild/bazel.git /bazel && \ - cd /bazel && git checkout tags/0.4.1 && ./compile.sh -RUN ln -s /bazel/output/bazel /bin/ +RUN echo "deb http://http.debian.net/debian jessie-backports main" >> /etc/apt/sources.list +RUN apt-get update +RUN apt-get install -y -t jessie-backports openjdk-8-jdk + +#======================== +# Bazel installation +RUN echo "deb [arch=amd64] http://storage.googleapis.com/bazel-apt stable jdk1.8" > /etc/apt/sources.list.d/bazel.list +RUN curl https://bazel.build/bazel-release.pub.gpg | apt-key add - +RUN apt-get -y update +RUN apt-get -y install bazel + +# Pin Bazel to 0.4.4 +# Installing Bazel via apt-get first is required before installing 0.4.4 to +# allow gRPC to build without errors. See https://github.com/grpc/grpc/issues/10553 +RUN curl -fSsL -O https://github.com/bazelbuild/bazel/releases/download/0.4.4/bazel-0.4.4-installer-linux-x86_64.sh +RUN chmod +x ./bazel-0.4.4-installer-linux-x86_64.sh +RUN ./bazel-0.4.4-installer-linux-x86_64.sh RUN apt-get update && apt-get -y install wget -RUN echo deb http://llvm.org/apt/wily/ llvm-toolchain-wily-3.8 main >> /etc/apt/sources.list -RUN echo deb-src http://llvm.org/apt/wily/ llvm-toolchain-wily-3.8 main >> /etc/apt/sources.list -RUN wget -O - http://llvm.org/apt/llvm-snapshot.gpg.key| apt-key add - +RUN echo "deb http://llvm.org/apt/jessie/ llvm-toolchain-jessie-3.8 main" >> /etc/apt/sources.list +RUN echo "deb-src http://llvm.org/apt/jessie/ llvm-toolchain-jessie-3.8 main" >> /etc/apt/sources.list +RUN wget -O - http://llvm.org/apt/llvm-snapshot.gpg.key | apt-key add - RUN apt-get update && apt-get -y install clang-format-3.8 # Prepare ccache diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal index 6b2764298c..d220365fc0 100644 --- a/tools/doxygen/Doxyfile.c++.internal +++ b/tools/doxygen/Doxyfile.c++.internal @@ -945,6 +945,8 @@ src/core/lib/channel/handshaker_registry.h \ src/core/lib/compression/algorithm_metadata.h \ src/core/lib/compression/message_compress.h \ src/core/lib/compression/stream_compression.h \ +src/core/lib/compression/stream_compression_gzip.h \ +src/core/lib/compression/stream_compression_identity.h \ src/core/lib/debug/stats.h \ src/core/lib/debug/stats_data.h \ src/core/lib/debug/trace.h \ diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index 364a4f7e2f..430566bfbe 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -1080,6 +1080,10 @@ src/core/lib/compression/message_compress.c \ src/core/lib/compression/message_compress.h \ src/core/lib/compression/stream_compression.c \ src/core/lib/compression/stream_compression.h \ +src/core/lib/compression/stream_compression_gzip.c \ +src/core/lib/compression/stream_compression_gzip.h \ +src/core/lib/compression/stream_compression_identity.c \ +src/core/lib/compression/stream_compression_identity.h \ src/core/lib/debug/stats.c \ src/core/lib/debug/stats.h \ src/core/lib/debug/stats_data.c \ diff --git a/tools/gce/linux_performance_worker_init.sh b/tools/gce/linux_performance_worker_init.sh index 88d8de7402..8d900f1d16 100755 --- a/tools/gce/linux_performance_worker_init.sh +++ b/tools/gce/linux_performance_worker_init.sh @@ -72,6 +72,7 @@ sudo apt-get install -y netperf sudo apt-get install -y libgflags-dev libgtest-dev libc++-dev clang # Python dependencies +sudo pip install --upgrade pip==9.0.1 sudo pip install tabulate sudo pip install google-api-python-client sudo pip install virtualenv diff --git a/tools/github/pr_latency.py b/tools/github/pr_latency.py new file mode 100644 index 0000000000..5d635835e5 --- /dev/null +++ b/tools/github/pr_latency.py @@ -0,0 +1,163 @@ +#!/usr/bin/env python +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Measure the time between PR creation and completion of all tests. + +You'll need a github API token to avoid being rate-limited. See +https://help.github.com/articles/creating-a-personal-access-token-for-the-command-line/ + +This script goes over the most recent 100 pull requests. For PRs with a single +commit, it uses the PR's creation as the initial time; othewise, it uses the +date of the last commit. This is somewhat fragile, and imposed by the fact that +GitHub reports a PR's updated timestamp for any event that modifies the PR (e.g. +comments), not just the addition of new commits. + +In addition, it ignores latencies greater than five hours, as that's likely due +to a manual re-run of tests. +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import json +import logging +import pprint +import urllib2 + +from datetime import datetime, timedelta + +logging.basicConfig(format='%(asctime)s %(message)s') + +PRS = 'https://api.github.com/repos/grpc/grpc/pulls?state=open&per_page=100' +COMMITS = 'https://api.github.com/repos/grpc/grpc/pulls/{pr_number}/commits' + + +def gh(url): + request = urllib2.Request(url) + if TOKEN: + request.add_header('Authorization', 'token {}'.format(TOKEN)) + response = urllib2.urlopen(request) + return response.read() + + +def print_csv_header(): + print('pr,base_time,test_time,latency_seconds,successes,failures,errors') + + +def output(pr, base_time, test_time, diff_time, successes, failures, errors, mode='human'): + if mode == 'human': + print("PR #{} base time: {} UTC, Tests completed at: {} UTC. Latency: {}." + "\n\tSuccesses: {}, Failures: {}, Errors: {}".format( + pr, base_time, test_time, diff_time, successes, failures, errors)) + elif mode == 'csv': + print(','.join([str(pr), str(base_time), + str(test_time), str(int((test_time-base_time).total_seconds())), + str(successes), str(failures), str(errors)])) + + +def parse_timestamp(datetime_str): + return datetime.strptime(datetime_str, '%Y-%m-%dT%H:%M:%SZ') + + +def to_posix_timestamp(dt): + return str((dt - datetime(1970, 1, 1)).total_seconds()) + + +def get_pr_data(): + latest_prs = json.loads(gh(PRS)) + res = [{'number': pr['number'], + 'created_at': parse_timestamp(pr['created_at']), + 'updated_at': parse_timestamp(pr['updated_at']), + 'statuses_url': pr['statuses_url']} + for pr in latest_prs] + return res + + +def get_commits_data(pr_number): + commits = json.loads(gh(COMMITS.format(pr_number=pr_number))) + return {'num_commits': len(commits), + 'most_recent_date': parse_timestamp(commits[-1]['commit']['author']['date'])} + + +def get_status_data(statuses_url, system): + status_url = statuses_url.replace('statuses', 'status') + statuses = json.loads(gh(status_url + '?per_page=100')) + successes = 0 + failures = 0 + errors = 0 + latest_datetime = None + if not statuses: return None + if system == 'kokoro': string_in_target_url = 'kokoro' + elif system == 'jenkins': string_in_target_url = 'grpc-testing' + for status in statuses['statuses']: + if not status['target_url'] or string_in_target_url not in status['target_url']: continue # Ignore jenkins + if status['state'] == 'pending': return None + elif status['state'] == 'success': successes += 1 + elif status['state'] == 'failure': failures += 1 + elif status['state'] == 'error': errors += 1 + if not latest_datetime: + latest_datetime = parse_timestamp(status['updated_at']) + else: + latest_datetime = max(latest_datetime, parse_timestamp(status['updated_at'])) + # First status is the most recent one. + if any([successes, failures, errors]) and sum([successes, failures, errors]) > 15: + return {'latest_datetime': latest_datetime, + 'successes': successes, + 'failures': failures, + 'errors': errors} + else: return None + + +def build_args_parser(): + import argparse + parser = argparse.ArgumentParser() + parser.add_argument('--format', type=str, choices=['human', 'csv'], + default='human', + help='Output format: are you a human or a machine?') + parser.add_argument('--system', type=str, choices=['jenkins', 'kokoro'], + required=True, help='Consider only the given CI system') + parser.add_argument('--token', type=str, default='', + help='GitHub token to use its API with a higher rate limit') + return parser + + +def main(): + import sys + global TOKEN + args_parser = build_args_parser() + args = args_parser.parse_args() + TOKEN = args.token + if args.format == 'csv': print_csv_header() + for pr_data in get_pr_data(): + commit_data = get_commits_data(pr_data['number']) + # PR with a single commit -> use the PRs creation time. + # else -> use the latest commit's date. + base_timestamp = pr_data['updated_at'] + if commit_data['num_commits'] > 1: + base_timestamp = commit_data['most_recent_date'] + else: + base_timestamp = pr_data['created_at'] + last_status = get_status_data(pr_data['statuses_url'], args.system) + if last_status: + diff = last_status['latest_datetime'] - base_timestamp + if diff < timedelta(hours=5): + output(pr_data['number'], base_timestamp, last_status['latest_datetime'], + diff, last_status['successes'], last_status['failures'], + last_status['errors'], mode=args.format) + + +if __name__ == '__main__': + main() diff --git a/tools/internal_ci/linux/grpc_performance_profile_daily.sh b/tools/internal_ci/linux/grpc_performance_profile_daily.sh index 25523e21b8..34d41bc04c 100755 --- a/tools/internal_ci/linux/grpc_performance_profile_daily.sh +++ b/tools/internal_ci/linux/grpc_performance_profile_daily.sh @@ -22,6 +22,8 @@ source tools/internal_ci/helper_scripts/prepare_build_linux_perf_rc CPUS=`python -c 'import multiprocessing; print multiprocessing.cpu_count()'` +./tools/run_tests/start_port_server.py || true + make CONFIG=opt memory_profile_test memory_profile_client memory_profile_server -j $CPUS bins/opt/memory_profile_test bq load microbenchmarks.memory memory_usage.csv diff --git a/tools/interop_matrix/README.md b/tools/interop_matrix/README.md index f92dc690e2..c2f354399f 100644 --- a/tools/interop_matrix/README.md +++ b/tools/interop_matrix/README.md @@ -5,6 +5,21 @@ This directory contains scripts that facilitate building and running gRPC tests The setup builds gRPC docker images for each language/runtime and upload it to Google Container Registry (GCR). These images, encapsulating gRPC stack from specific releases/tag, are used to test version compatiblity between gRPC release versions. +## Step-by-step instructions for adding a new release to compatibility test +We have continuous nightly test setup to test gRPC backward compatibility between old clients and latest server. When a gRPC developer creates a new gRPC release, s/he is also responsible to add the just-released gRPC client to the nightly test. The steps are: +- Add (or update) an entry in ./client_matrix.py file to reference the github tag for the release. +- Build new client docker image(s). For example, for java release `v1.9.9`, do + - `tools/interop_matrix/create_matrix_images.py --git_checkout --release=v1.9.9 --language=java` +- Verify that the new docker image was built successfully and uploaded to GCR. For example, + - `gcloud beta container images list-tags gcr.io/grpc-testing/grpc_interop_java_oracle8` + - should show an image entry with tag `v1.9.9`. +- Verify the just-created docker client image would pass backward compatibility test (it should). For example, + - `gcloud docker -- pull gcr.io/grpc-testing/grpc_interop_java_oracle8:v1.9.9` followed by + - `docker_image=gcr.io/grpc-testing/grpc_interop_java_oracle8:v1.9.9 ./testcases/java__master` +- git commit the change and merge it to upstream/master. +- (Optional) clean up the tmp directory to where grpc source is cloned at `/export/hda3/tmp/grpc_matrix/`. +For more details on each step, refer to sections below. + ## Instructions for creating GCR images - Edit `./client_matrix.py` to include desired gRPC release. - Run `tools/interop_matrix/create_matrix_images.py`. Useful options: @@ -45,3 +60,4 @@ For example: Note: - File path starting with `tools/` or `template/` are relative to the grpc repo root dir. File path starting with `./` are relative to current directory (`tools/interop_matrix`). +- Creating and referencing images in GCR require read and write permission to Google Container Registry path gcr.io/grpc-testing. diff --git a/tools/jenkins/run_full_performance.sh b/tools/jenkins/run_full_performance.sh index 0f101f6da9..a9661c7e26 100755 --- a/tools/jenkins/run_full_performance.sh +++ b/tools/jenkins/run_full_performance.sh @@ -21,7 +21,7 @@ cd $(dirname $0)/../.. # run 8core client vs 8core server tools/run_tests/run_performance_tests.py \ - -l c++ csharp node ruby java python go node_express \ + -l c++ csharp node ruby java python go node_express php \ --netperf \ --category scalable \ --bq_result_table performance_test.performance_experiment \ diff --git a/tools/jenkins/run_performance_profile_daily.sh b/tools/jenkins/run_performance_profile_daily.sh index 04a2464aee..48d82a9b7f 100755 --- a/tools/jenkins/run_performance_profile_daily.sh +++ b/tools/jenkins/run_performance_profile_daily.sh @@ -29,4 +29,6 @@ fi BENCHMARKS_TO_RUN="bm_fullstack_unary_ping_pong bm_fullstack_streaming_ping_pong bm_fullstack_streaming_pump bm_closure bm_cq bm_call_create bm_error bm_chttp2_hpack bm_chttp2_transport bm_pollset bm_metadata" +./tools/run_tests/start_port_server.py || true + $PYTHON tools/run_tests/run_microbenchmark.py --collect summary perf latency -b $BENCHMARKS_TO_RUN diff --git a/tools/profiling/microbenchmarks/bm_diff/bm_main.py b/tools/profiling/microbenchmarks/bm_diff/bm_main.py index 5aa11ac391..516d110b97 100755 --- a/tools/profiling/microbenchmarks/bm_diff/bm_main.py +++ b/tools/profiling/microbenchmarks/bm_diff/bm_main.py @@ -23,6 +23,7 @@ import bm_diff import sys import os +import random import argparse import multiprocessing import subprocess @@ -32,6 +33,12 @@ sys.path.append( os.path.dirname(sys.argv[0]), '..', '..', 'run_tests', 'python_utils')) import comment_on_pr +sys.path.append( + os.path.join( + os.path.dirname(sys.argv[0]), '..', '..', '..', 'run_tests', + 'python_utils')) +import jobset + def _args(): argp = argparse.ArgumentParser( @@ -125,8 +132,13 @@ def main(args): subprocess.check_call(['git', 'checkout', where_am_i]) subprocess.check_call(['git', 'submodule', 'update']) - bm_run.run('new', args.benchmarks, args.jobs, args.loops, args.regex, args.counters) - bm_run.run(old, args.benchmarks, args.jobs, args.loops, args.regex, args.counters) + jobs_list = [] + jobs_list += bm_run.create_jobs('new', args.benchmarks, args.loops, args.regex, args.counters) + jobs_list += bm_run.create_jobs(old, args.benchmarks, args.loops, args.regex, args.counters) + + # shuffle all jobs to eliminate noise from GCE CPU drift + random.shuffle(jobs_list, random.SystemRandom().random) + jobset.run(jobs_list, maxjobs=args.jobs) diff, note = bm_diff.diff(args.benchmarks, args.loops, args.regex, args.track, old, 'new', args.counters) diff --git a/tools/profiling/microbenchmarks/bm_diff/bm_run.py b/tools/profiling/microbenchmarks/bm_diff/bm_run.py index 206f7c5845..81db5a226a 100755 --- a/tools/profiling/microbenchmarks/bm_diff/bm_run.py +++ b/tools/profiling/microbenchmarks/bm_diff/bm_run.py @@ -95,11 +95,12 @@ def _collect_bm_data(bm, cfg, name, regex, idx, loops): shortname='%s %s %s %s %d/%d' % (bm, line, cfg, name, idx + 1, loops), verbose_success=True, + cpu_cost=2, timeout_seconds=60 * 60)) # one hour return jobs_list -def run(name, benchmarks, jobs, loops, regex, counters): +def create_jobs(name, benchmarks, loops, regex, counters): jobs_list = [] for loop in range(0, loops): for bm in benchmarks: @@ -108,9 +109,11 @@ def run(name, benchmarks, jobs, loops, regex, counters): jobs_list += _collect_bm_data(bm, 'counters', name, regex, loop, loops) random.shuffle(jobs_list, random.SystemRandom().random) - jobset.run(jobs_list, maxjobs=jobs) + return jobs_list if __name__ == '__main__': args = _args() - run(args.name, args.benchmarks, args.jobs, args.loops, args.regex, args.counters) + jobs_list = create_jobs(args.name, args.benchmarks, args.loops, + args.regex, args.counters) + jobset.run(jobs_list, maxjobs=args.jobs) diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json index 4e5e9f4a7f..5e825e2082 100644 --- a/tools/run_tests/generated/sources_and_headers.json +++ b/tools/run_tests/generated/sources_and_headers.json @@ -8007,6 +8007,8 @@ "src/core/lib/compression/compression.c", "src/core/lib/compression/message_compress.c", "src/core/lib/compression/stream_compression.c", + "src/core/lib/compression/stream_compression_gzip.c", + "src/core/lib/compression/stream_compression_identity.c", "src/core/lib/debug/stats.c", "src/core/lib/debug/stats_data.c", "src/core/lib/http/format_request.c", @@ -8160,6 +8162,8 @@ "src/core/lib/compression/algorithm_metadata.h", "src/core/lib/compression/message_compress.h", "src/core/lib/compression/stream_compression.h", + "src/core/lib/compression/stream_compression_gzip.h", + "src/core/lib/compression/stream_compression_identity.h", "src/core/lib/debug/stats.h", "src/core/lib/debug/stats_data.h", "src/core/lib/http/format_request.h", @@ -8292,6 +8296,8 @@ "src/core/lib/compression/algorithm_metadata.h", "src/core/lib/compression/message_compress.h", "src/core/lib/compression/stream_compression.h", + "src/core/lib/compression/stream_compression_gzip.h", + "src/core/lib/compression/stream_compression_identity.h", "src/core/lib/debug/stats.h", "src/core/lib/debug/stats_data.h", "src/core/lib/http/format_request.h", diff --git a/tools/run_tests/helper_scripts/build_python.sh b/tools/run_tests/helper_scripts/build_python.sh index be650553dc..e3620821f3 100755 --- a/tools/run_tests/helper_scripts/build_python.sh +++ b/tools/run_tests/helper_scripts/build_python.sh @@ -152,7 +152,7 @@ pip_install_dir() { cd $PWD } -$VENV_PYTHON -m pip install --upgrade pip +$VENV_PYTHON -m pip install --upgrade pip==9.0.1 $VENV_PYTHON -m pip install setuptools $VENV_PYTHON -m pip install cython $VENV_PYTHON -m pip install six enum34 protobuf futures diff --git a/tools/run_tests/helper_scripts/run_grpc-node.sh b/tools/run_tests/helper_scripts/run_grpc-node.sh new file mode 100755 index 0000000000..25f149f579 --- /dev/null +++ b/tools/run_tests/helper_scripts/run_grpc-node.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# Copyright 2015 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This script runs grpc/grpc-node tests with their grpc submodule updated +# to this reference + +# cd to gRPC root directory +cd $(dirname $0)/../../.. + +CURRENT_COMMIT=$(git rev-parse --verify HEAD) + +rm -rf ./../grpc-node +git clone --recursive https://github.com/grpc/grpc-node ./../grpc-node +cd ./../grpc-node + +./test-grpc-submodule.sh $CURRENT_COMMIT diff --git a/tools/run_tests/performance/massage_qps_stats.py b/tools/run_tests/performance/massage_qps_stats.py index a1a6aa4526..ebbfe6c26c 100644 --- a/tools/run_tests/performance/massage_qps_stats.py +++ b/tools/run_tests/performance/massage_qps_stats.py @@ -107,8 +107,6 @@ def massage_qps_stats(scenario_result): stats["core_executor_wakeup_initiated"] = massage_qps_stats_helpers.counter(core_stats, "executor_wakeup_initiated") stats["core_executor_queue_drained"] = massage_qps_stats_helpers.counter(core_stats, "executor_queue_drained") stats["core_executor_push_retries"] = massage_qps_stats_helpers.counter(core_stats, "executor_push_retries") - stats["core_executor_threads_created"] = massage_qps_stats_helpers.counter(core_stats, "executor_threads_created") - stats["core_executor_threads_used"] = massage_qps_stats_helpers.counter(core_stats, "executor_threads_used") stats["core_server_requested_calls"] = massage_qps_stats_helpers.counter(core_stats, "server_requested_calls") stats["core_server_slowpath_requests_queued"] = massage_qps_stats_helpers.counter(core_stats, "server_slowpath_requests_queued") h = massage_qps_stats_helpers.histogram(core_stats, "call_initial_size") @@ -183,12 +181,6 @@ def massage_qps_stats(scenario_result): stats["core_http2_send_flowctl_per_write_50p"] = massage_qps_stats_helpers.percentile(h.buckets, 50, h.boundaries) stats["core_http2_send_flowctl_per_write_95p"] = massage_qps_stats_helpers.percentile(h.buckets, 95, h.boundaries) stats["core_http2_send_flowctl_per_write_99p"] = massage_qps_stats_helpers.percentile(h.buckets, 99, h.boundaries) - h = massage_qps_stats_helpers.histogram(core_stats, "executor_closures_per_wakeup") - stats["core_executor_closures_per_wakeup"] = ",".join("%f" % x for x in h.buckets) - stats["core_executor_closures_per_wakeup_bkts"] = ",".join("%f" % x for x in h.boundaries) - stats["core_executor_closures_per_wakeup_50p"] = massage_qps_stats_helpers.percentile(h.buckets, 50, h.boundaries) - stats["core_executor_closures_per_wakeup_95p"] = massage_qps_stats_helpers.percentile(h.buckets, 95, h.boundaries) - stats["core_executor_closures_per_wakeup_99p"] = massage_qps_stats_helpers.percentile(h.buckets, 99, h.boundaries) h = massage_qps_stats_helpers.histogram(core_stats, "server_cqs_checked") stats["core_server_cqs_checked"] = ",".join("%f" % x for x in h.buckets) stats["core_server_cqs_checked_bkts"] = ",".join("%f" % x for x in h.boundaries) diff --git a/tools/run_tests/performance/scenario_result_schema.json b/tools/run_tests/performance/scenario_result_schema.json index be52166915..169221d18c 100644 --- a/tools/run_tests/performance/scenario_result_schema.json +++ b/tools/run_tests/performance/scenario_result_schema.json @@ -547,16 +547,6 @@ }, { "mode": "NULLABLE", - "name": "core_executor_threads_created", - "type": "INTEGER" - }, - { - "mode": "NULLABLE", - "name": "core_executor_threads_used", - "type": "INTEGER" - }, - { - "mode": "NULLABLE", "name": "core_server_requested_calls", "type": "INTEGER" }, @@ -867,31 +857,6 @@ }, { "mode": "NULLABLE", - "name": "core_executor_closures_per_wakeup", - "type": "STRING" - }, - { - "mode": "NULLABLE", - "name": "core_executor_closures_per_wakeup_bkts", - "type": "STRING" - }, - { - "mode": "NULLABLE", - "name": "core_executor_closures_per_wakeup_50p", - "type": "FLOAT" - }, - { - "mode": "NULLABLE", - "name": "core_executor_closures_per_wakeup_95p", - "type": "FLOAT" - }, - { - "mode": "NULLABLE", - "name": "core_executor_closures_per_wakeup_99p", - "type": "FLOAT" - }, - { - "mode": "NULLABLE", "name": "core_server_cqs_checked", "type": "STRING" }, @@ -1379,16 +1344,6 @@ }, { "mode": "NULLABLE", - "name": "core_executor_threads_created", - "type": "INTEGER" - }, - { - "mode": "NULLABLE", - "name": "core_executor_threads_used", - "type": "INTEGER" - }, - { - "mode": "NULLABLE", "name": "core_server_requested_calls", "type": "INTEGER" }, @@ -1699,31 +1654,6 @@ }, { "mode": "NULLABLE", - "name": "core_executor_closures_per_wakeup", - "type": "STRING" - }, - { - "mode": "NULLABLE", - "name": "core_executor_closures_per_wakeup_bkts", - "type": "STRING" - }, - { - "mode": "NULLABLE", - "name": "core_executor_closures_per_wakeup_50p", - "type": "FLOAT" - }, - { - "mode": "NULLABLE", - "name": "core_executor_closures_per_wakeup_95p", - "type": "FLOAT" - }, - { - "mode": "NULLABLE", - "name": "core_executor_closures_per_wakeup_99p", - "type": "FLOAT" - }, - { - "mode": "NULLABLE", "name": "core_server_cqs_checked", "type": "STRING" }, diff --git a/tools/run_tests/python_utils/filter_pull_request_tests.py b/tools/run_tests/python_utils/filter_pull_request_tests.py index f99143fdd7..393aef8662 100644 --- a/tools/run_tests/python_utils/filter_pull_request_tests.py +++ b/tools/run_tests/python_utils/filter_pull_request_tests.py @@ -47,7 +47,7 @@ class TestSuite: _CORE_TEST_SUITE = TestSuite(['c']) _CPP_TEST_SUITE = TestSuite(['c++']) _CSHARP_TEST_SUITE = TestSuite(['csharp']) -_NODE_TEST_SUITE = TestSuite(['node']) +_NODE_TEST_SUITE = TestSuite(['node', 'grpc-node']) _OBJC_TEST_SUITE = TestSuite(['objc']) _PHP_TEST_SUITE = TestSuite(['php', 'php7']) _PYTHON_TEST_SUITE = TestSuite(['python']) diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index f661aed4e2..5750b90619 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -444,6 +444,67 @@ class CLanguage(object): return self.make_target +# This tests Node on grpc/grpc-node and will become the standard for Node testing +class RemoteNodeLanguage(object): + + def __init__(self): + self.platform = platform_string() + + def configure(self, config, args): + self.config = config + self.args = args + # Note: electron ABI only depends on major and minor version, so that's all + # we should specify in the compiler argument + _check_compiler(self.args.compiler, ['default', 'node0.12', + 'node4', 'node5', 'node6', + 'node7', 'node8', + 'electron1.3', 'electron1.6']) + if self.args.compiler == 'default': + self.runtime = 'node' + self.node_version = '8' + else: + if self.args.compiler.startswith('electron'): + self.runtime = 'electron' + self.node_version = self.args.compiler[8:] + else: + self.runtime = 'node' + # Take off the word "node" + self.node_version = self.args.compiler[4:] + + # TODO: update with Windows/electron scripts when available for grpc/grpc-node + def test_specs(self): + if self.platform == 'windows': + return [self.config.job_spec(['tools\\run_tests\\helper_scripts\\run_node.bat'])] + else: + return [self.config.job_spec(['tools/run_tests/helper_scripts/run_grpc-node.sh'], + None, + environ=_FORCE_ENVIRON_FOR_WRAPPERS)] + + def pre_build_steps(self): + return [] + + def make_targets(self): + return [] + + def make_options(self): + return [] + + def build_steps(self): + return [] + + def post_tests_steps(self): + return [] + + def makefile_name(self): + return 'Makefile' + + def dockerfile_dir(self): + return 'tools/dockerfile/test/node_jessie_%s' % _docker_arch_suffix(self.args.arch) + + def __str__(self): + return 'grpc-node' + + class NodeLanguage(object): def __init__(self): @@ -717,6 +778,9 @@ class PythonLanguage(object): return (pypy32_config,) elif args.compiler == 'python_alpine': return (python27_config,) + elif args.compiler == 'all_the_cpythons': + return (python27_config, python34_config, python35_config, + python36_config,) else: raise Exception('Compiler %s not supported.' % args.compiler) @@ -1064,6 +1128,7 @@ with open('tools/run_tests/generated/configs.json') as f: _LANGUAGES = { 'c++': CLanguage('cxx', 'c++'), 'c': CLanguage('c', 'c'), + 'grpc-node': RemoteNodeLanguage(), 'node': NodeLanguage(), 'node_express': NodeExpressLanguage(), 'php': PhpLanguage(), @@ -1214,7 +1279,7 @@ argp.add_argument('--compiler', choices=['default', 'gcc4.4', 'gcc4.6', 'gcc4.8', 'gcc4.9', 'gcc5.3', 'gcc_musl', 'clang3.4', 'clang3.5', 'clang3.6', 'clang3.7', - 'python2.7', 'python3.4', 'python3.5', 'python3.6', 'pypy', 'pypy3', 'python_alpine', + 'python2.7', 'python3.4', 'python3.5', 'python3.6', 'pypy', 'pypy3', 'python_alpine', 'all_the_cpythons', 'node0.12', 'node4', 'node5', 'node6', 'node7', 'node8', 'electron1.3', 'electron1.6', 'coreclr', diff --git a/tools/run_tests/run_tests_matrix.py b/tools/run_tests/run_tests_matrix.py index 2417c5ae55..34d839e0b8 100755 --- a/tools/run_tests/run_tests_matrix.py +++ b/tools/run_tests/run_tests_matrix.py @@ -169,7 +169,7 @@ def _create_test_jobs(extra_args=[], inner_jobs=_DEFAULT_INNER_JOBS): inner_jobs=inner_jobs, timeout_seconds=_CPP_RUNTESTS_TIMEOUT) - test_jobs += _generate_jobs(languages=['ruby', 'php'], + test_jobs += _generate_jobs(languages=['grpc-node', 'ruby', 'php'], configs=['dbg', 'opt'], platforms=['linux', 'macos'], labels=['basictests', 'multilang'], diff --git a/tools/run_tests/sanity/check_test_filtering.py b/tools/run_tests/sanity/check_test_filtering.py index bea8125fb9..3ebb9389f7 100755 --- a/tools/run_tests/sanity/check_test_filtering.py +++ b/tools/run_tests/sanity/check_test_filtering.py @@ -25,7 +25,7 @@ sys.path.insert(0, os.path.abspath('tools/run_tests/')) from run_tests_matrix import _create_test_jobs, _create_portability_test_jobs import python_utils.filter_pull_request_tests as filter_pull_request_tests -_LIST_OF_LANGUAGE_LABELS = ['c', 'c++', 'csharp', 'node', 'objc', 'php', 'php7', 'python', 'ruby'] +_LIST_OF_LANGUAGE_LABELS = ['c', 'c++', 'csharp', 'grpc-node', 'node', 'objc', 'php', 'php7', 'python', 'ruby'] _LIST_OF_PLATFORM_LABELS = ['linux', 'macos', 'windows'] class TestFilteringTest(unittest.TestCase): |