diff options
Diffstat (limited to 'tools')
20 files changed, 208 insertions, 38 deletions
diff --git a/tools/distrib/pylint_code.sh b/tools/distrib/pylint_code.sh index 6369e605d5..b1e305c56d 100755 --- a/tools/distrib/pylint_code.sh +++ b/tools/distrib/pylint_code.sh @@ -31,18 +31,22 @@ set -ex # change to root directory -cd $(dirname $0)/../.. +cd "$(dirname "$0")/../.." -DIRS=src/python/grpcio/grpc +DIRS=( + 'src/python/grpcio/grpc' + 'src/python/grpcio_reflection/grpc_reflection' + 'src/python/grpcio_health_checking/grpc_health' +) VIRTUALENV=python_pylint_venv virtualenv $VIRTUALENV -PYTHON=`realpath $VIRTUALENV/bin/python` +PYTHON=$(realpath $VIRTUALENV/bin/python) $PYTHON -m pip install pylint==1.6.5 -for dir in $DIRS; do - $PYTHON -m pylint --rcfile=.pylintrc -rn $dir || exit $? +for dir in "${DIRS[@]}"; do + $PYTHON -m pylint --rcfile=.pylintrc -rn "$dir" || exit $? done exit 0 diff --git a/tools/distrib/python/grpcio_tools/grpc_version.py b/tools/distrib/python/grpcio_tools/grpc_version.py index 1f2aa81c85..417722a4d2 100644 --- a/tools/distrib/python/grpcio_tools/grpc_version.py +++ b/tools/distrib/python/grpcio_tools/grpc_version.py @@ -29,4 +29,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/tools/distrib/python/grpcio_tools/grpc_version.py.template`!!! -VERSION='1.4.0.dev0' +VERSION='1.5.0.dev0' diff --git a/tools/dockerfile/test/cxx_alpine_x64/Dockerfile b/tools/dockerfile/test/cxx_alpine_x64/Dockerfile index b13157f280..fc62d230fb 100644 --- a/tools/dockerfile/test/cxx_alpine_x64/Dockerfile +++ b/tools/dockerfile/test/cxx_alpine_x64/Dockerfile @@ -55,6 +55,9 @@ RUN pip install pip --upgrade RUN pip install virtualenv RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 +# Google Cloud platform API libraries +RUN pip install --upgrade google-api-python-client + # Prepare ccache RUN ln -s /usr/bin/ccache /usr/local/bin/gcc RUN ln -s /usr/bin/ccache /usr/local/bin/g++ diff --git a/tools/dockerfile/test/python_alpine_x64/Dockerfile b/tools/dockerfile/test/python_alpine_x64/Dockerfile index bdffbd3598..5d25ab0ebe 100644 --- a/tools/dockerfile/test/python_alpine_x64/Dockerfile +++ b/tools/dockerfile/test/python_alpine_x64/Dockerfile @@ -55,6 +55,9 @@ RUN pip install pip --upgrade RUN pip install virtualenv RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 +# Google Cloud platform API libraries +RUN pip install --upgrade google-api-python-client + # Prepare ccache RUN ln -s /usr/bin/ccache /usr/local/bin/gcc RUN ln -s /usr/bin/ccache /usr/local/bin/g++ diff --git a/tools/dockerfile/test/python_jessie_x64/Dockerfile b/tools/dockerfile/test/python_jessie_x64/Dockerfile index cc69f4b5cd..f470bc2487 100644 --- a/tools/dockerfile/test/python_jessie_x64/Dockerfile +++ b/tools/dockerfile/test/python_jessie_x64/Dockerfile @@ -75,7 +75,8 @@ RUN pip install --upgrade google-api-python-client RUN apt-get update && apt-get install -y \ python-all-dev \ python3-all-dev \ - python-pip + python-pip \ + python3-pip # Install Python packages from PyPI RUN pip install pip --upgrade diff --git a/tools/doxygen/Doxyfile.c++ b/tools/doxygen/Doxyfile.c++ index 3861bdb85a..23746de1aa 100644 --- a/tools/doxygen/Doxyfile.c++ +++ b/tools/doxygen/Doxyfile.c++ @@ -40,7 +40,7 @@ PROJECT_NAME = "GRPC C++" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 1.4.0-dev +PROJECT_NUMBER = 1.5.0-dev # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal index 5bab66c7d6..52e722f56c 100644 --- a/tools/doxygen/Doxyfile.c++.internal +++ b/tools/doxygen/Doxyfile.c++.internal @@ -40,7 +40,7 @@ PROJECT_NAME = "GRPC C++" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 1.4.0-dev +PROJECT_NUMBER = 1.5.0-dev # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/tools/internal_ci/helper_scripts/prepare_build_macos_rc b/tools/internal_ci/helper_scripts/prepare_build_macos_rc new file mode 100644 index 0000000000..3eccff80b4 --- /dev/null +++ b/tools/internal_ci/helper_scripts/prepare_build_macos_rc @@ -0,0 +1,72 @@ +#!/bin/bash +# Copyright 2017, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# Source this rc script to prepare the environment for macos builds + +# TODO(jtattermusch): remove all deps once installed on MacOS workers + +# brew and C++ deps +yes | ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" +brew install autoconf automake libtool ccache cmake gflags gpg wget + +# TODO(jtattermusch): install rvm & ruby +# TODO(jtattermusch): install cocoapods + +# python +wget https://bootstrap.pypa.io/get-pip.py +sudo python get-pip.py +sudo pip install virtualenv + +# TODO(jtattermusch): install python3 + +# mono +wget https://download.mono-project.com/archive/5.0.1/macos-10-universal/MonoFramework-MDK-5.0.1.1.macos10.xamarin.universal.pkg +sudo installer -pkg MonoFramework-MDK-5.0.1.1.macos10.xamarin.universal.pkg -target / + +# dotnet SDK +wget https://go.microsoft.com/fwlink/?linkid=843444 -O dotnet-dev-osx-x64.1.0.1.pkg +sudo installer -pkg dotnet-dev-osx-x64.1.0.1.pkg -target / +ln -s /usr/local/share/dotnet/dotnet /usr/local/bin/dotnet +dotnet --version # bootstrap dotnet SDK + +# nvm +wget -qO- https://raw.githubusercontent.com/creationix/nvm/v0.30.2/install.sh | bash +# bootstrap nvm silently & without terminating this script +set +ex +source ~/.nvm/nvm.sh +set -ex + +# node +nvm install 4 +nvm alias default 4 +npm update npm -g +npm install -g node-pre-gyp + +git submodule update --init diff --git a/tools/internal_ci/linux/grpc_build_artifacts.sh b/tools/internal_ci/linux/grpc_build_artifacts.sh index 1b12be143e..16c2a748a6 100755 --- a/tools/internal_ci/linux/grpc_build_artifacts.sh +++ b/tools/internal_ci/linux/grpc_build_artifacts.sh @@ -35,4 +35,8 @@ cd $(dirname $0)/../../.. source tools/internal_ci/helper_scripts/prepare_build_linux_rc +# TODO(jtattermusch): install ruby on the internal_ci worker +gpg --keyserver hkp://keys.gnupg.net --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 +curl -sSL https://get.rvm.io | bash -s stable --ruby + tools/run_tests/task_runner.py -f artifact linux diff --git a/tools/internal_ci/linux/grpc_sanity.cfg b/tools/internal_ci/linux/grpc_sanity.cfg new file mode 100644 index 0000000000..e2f320494b --- /dev/null +++ b/tools/internal_ci/linux/grpc_sanity.cfg @@ -0,0 +1,39 @@ +# Copyright 2017, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/linux/grpc_sanity.sh" +timeout_mins: 20 +action { + define_artifacts { + regex: "**/*sponge_log.xml" + } +} diff --git a/tools/internal_ci/linux/grpc_sanity.sh b/tools/internal_ci/linux/grpc_sanity.sh index 7166ce7d24..432a42c449 100755 --- a/tools/internal_ci/linux/grpc_sanity.sh +++ b/tools/internal_ci/linux/grpc_sanity.sh @@ -35,4 +35,4 @@ cd $(dirname $0)/../../.. source tools/internal_ci/helper_scripts/prepare_build_linux_rc -tools/run_tests/run_tests.py -l sanity -c opt -t -x sponge_log.xml --use_docker --report_suite_name sanity_linux_opt +tools/run_tests/run_tests_matrix.py -f basictests linux sanity opt --inner_jobs 16 -j 1 --internal_ci diff --git a/tools/internal_ci/macos/grpc_master.sh b/tools/internal_ci/macos/grpc_master.sh index 4ce1af73a5..4eeac4e1e6 100755 --- a/tools/internal_ci/macos/grpc_master.sh +++ b/tools/internal_ci/macos/grpc_master.sh @@ -33,7 +33,7 @@ set -ex # change to grpc repo root cd $(dirname $0)/../../.. -git submodule update --init +source tools/internal_ci/helper_scripts/prepare_build_macos_rc tools/run_tests/run_tests_matrix.py -f basictests macos --internal_ci || FAILED="true" diff --git a/tools/profiling/microbenchmarks/bm_json.py b/tools/profiling/microbenchmarks/bm_json.py index f4d628e11f..49a3707220 100644 --- a/tools/profiling/microbenchmarks/bm_json.py +++ b/tools/profiling/microbenchmarks/bm_json.py @@ -56,7 +56,7 @@ _BM_SPECS = { }, 'BM_PumpUnbalancedUnary_Trickle': { 'tpl': [], - 'dyn': ['request_size', 'bandwidth_kilobits'], + 'dyn': ['cli_req_size', 'svr_req_size', 'bandwidth_kilobits'], }, 'BM_ErrorStringOnNewError': { 'tpl': ['fixture'], diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json index f3a3c5a0f5..ccb3d6abca 100644 --- a/tools/run_tests/generated/sources_and_headers.json +++ b/tools/run_tests/generated/sources_and_headers.json @@ -1765,6 +1765,23 @@ "headers": [], "is_filegroup": false, "language": "c", + "name": "num_external_connectivity_watchers_test", + "src": [ + "test/core/surface/num_external_connectivity_watchers_test.c" + ], + "third_party": false, + "type": "target" + }, + { + "deps": [ + "gpr", + "gpr_test_util", + "grpc", + "grpc_test_util" + ], + "headers": [], + "is_filegroup": false, + "language": "c", "name": "parse_address_test", "src": [ "test/core/client_channel/parse_address_test.c" diff --git a/tools/run_tests/generated/tests.json b/tools/run_tests/generated/tests.json index 82db441a1a..1242025fc2 100644 --- a/tools/run_tests/generated/tests.json +++ b/tools/run_tests/generated/tests.json @@ -1839,6 +1839,30 @@ ], "cpu_cost": 1.0, "exclude_configs": [], + "exclude_iomgrs": [ + "uv" + ], + "flaky": false, + "gtest": false, + "language": "c", + "name": "num_external_connectivity_watchers_test", + "platforms": [ + "linux", + "mac", + "posix", + "windows" + ] + }, + { + "args": [], + "ci_platforms": [ + "linux", + "mac", + "posix", + "windows" + ], + "cpu_cost": 1.0, + "exclude_configs": [], "exclude_iomgrs": [], "flaky": false, "gtest": false, diff --git a/tools/run_tests/helper_scripts/build_python.sh b/tools/run_tests/helper_scripts/build_python.sh index 6ad285cdb9..bd4b40b508 100755 --- a/tools/run_tests/helper_scripts/build_python.sh +++ b/tools/run_tests/helper_scripts/build_python.sh @@ -146,6 +146,7 @@ fi ############################ # Perform build operations # ############################ +$PYTHON -m pip install virtualenv # Instnatiate the virtualenv, preferring to do so from the relevant python # version. Even if these commands fail (e.g. on Windows due to name conflicts) diff --git a/tools/run_tests/helper_scripts/run_ruby_end2end_tests.sh b/tools/run_tests/helper_scripts/run_ruby_end2end_tests.sh index 6688025260..ab882d62bc 100755 --- a/tools/run_tests/helper_scripts/run_ruby_end2end_tests.sh +++ b/tools/run_tests/helper_scripts/run_ruby_end2end_tests.sh @@ -41,4 +41,5 @@ ruby src/ruby/end2end/sig_int_during_channel_watch_driver.rb || EXIT_CODE=1 ruby src/ruby/end2end/killed_client_thread_driver.rb || EXIT_CODE=1 ruby src/ruby/end2end/forking_client_driver.rb || EXIT_CODE=1 ruby src/ruby/end2end/grpc_class_init_driver.rb || EXIT_CODE=1 +ruby src/ruby/end2end/multiple_killed_watching_threads_driver.rb || EXIT_CODE=1 exit $EXIT_CODE diff --git a/tools/run_tests/performance/scenario_config.py b/tools/run_tests/performance/scenario_config.py index c2ffd67dbf..e18fd69cc4 100644 --- a/tools/run_tests/performance/scenario_config.py +++ b/tools/run_tests/performance/scenario_config.py @@ -523,15 +523,14 @@ class NodeLanguage: def scenarios(self): # TODO(jtattermusch): make this scenario work - #yield _ping_pong_scenario( - # 'node_generic_async_streaming_ping_pong', rpc_type='STREAMING', - # client_type='ASYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER', - # use_generic_payload=True) + yield _ping_pong_scenario( + 'node_generic_streaming_ping_pong', rpc_type='STREAMING', + client_type='ASYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER', + use_generic_payload=True) - # TODO(jtattermusch): make this scenario work - #yield _ping_pong_scenario( - # 'node_protobuf_async_streaming_ping_pong', rpc_type='STREAMING', - # client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER') + yield _ping_pong_scenario( + 'node_protobuf_streaming_ping_pong', rpc_type='STREAMING', + client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER') yield _ping_pong_scenario( 'node_protobuf_unary_ping_pong', rpc_type='UNARY', @@ -564,29 +563,26 @@ class NodeLanguage: secure=secure, categories=[SCALABLE]) - # TODO(murgatroid99): fix bugs with this scenario and re-enable it - # yield _ping_pong_scenario( - # 'node_protobuf_async_unary_qps_unconstrained', rpc_type='UNARY', - # client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', - # unconstrained_client='async', - # categories=[SCALABLE, SMOKETEST]) + yield _ping_pong_scenario( + 'node_protobuf_unary_qps_unconstrained', rpc_type='UNARY', + client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', + unconstrained_client='async', + categories=[SCALABLE, SMOKETEST]) - # TODO(jtattermusch): make this scenario work - #yield _ping_pong_scenario( - # 'node_protobuf_async_streaming_qps_unconstrained', rpc_type='STREAMING', - # client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', - # unconstrained_client='async') + yield _ping_pong_scenario( + 'node_protobuf_streaming_qps_unconstrained', rpc_type='STREAMING', + client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', + unconstrained_client='async') yield _ping_pong_scenario( 'node_to_cpp_protobuf_async_unary_ping_pong', rpc_type='UNARY', client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', server_language='c++', async_server_threads=1) - # TODO(jtattermusch): make this scenario work - #yield _ping_pong_scenario( - # 'node_to_cpp_protobuf_async_streaming_ping_pong', rpc_type='STREAMING', - # client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', - # server_language='c++', async_server_threads=1) + yield _ping_pong_scenario( + 'node_to_cpp_protobuf_async_streaming_ping_pong', rpc_type='STREAMING', + client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', + server_language='c++', async_server_threads=1) def __str__(self): return 'node' diff --git a/tools/run_tests/python_utils/jobset.py b/tools/run_tests/python_utils/jobset.py index 3ff773300f..def612881e 100755 --- a/tools/run_tests/python_utils/jobset.py +++ b/tools/run_tests/python_utils/jobset.py @@ -276,8 +276,13 @@ class Job(object): env = sanitized_environment(env) self._start = time.time() cmdline = self._spec.cmdline - if measure_cpu_costs: + # The Unix time command is finicky when used with MSBuild, so we don't use it + # with jobs that run MSBuild. + global measure_cpu_costs + if measure_cpu_costs and not 'vsprojects\\build' in cmdline[0]: cmdline = ['time', '-p'] + cmdline + else: + measure_cpu_costs = False try_start = lambda: subprocess.Popen(args=cmdline, stderr=subprocess.STDOUT, stdout=self._tempfile, diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index 568774cece..204ed5c397 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -672,7 +672,7 @@ class PythonLanguage(object): if args.compiler == 'default': if os.name == 'nt': - return (python27_config,) + return (python35_config,) else: return (python27_config, python34_config,) elif args.compiler == 'python2.7': |