diff options
author | Craig Tiller <ctiller@google.com> | 2017-02-23 10:12:35 -0800 |
---|---|---|
committer | Craig Tiller <ctiller@google.com> | 2017-02-23 10:12:35 -0800 |
commit | b92d5a2f70b750310bd3ffe584dcf145ca609587 (patch) | |
tree | 9277b5c8c1ac76b7628c8cdb0eb558af31437d16 /tools | |
parent | b8e2bca4eb71c109943ea3ed59dca74dd30cbd68 (diff) | |
parent | bc34a08bf5e53264e1e88b99824807e990de2121 (diff) |
Merge branch 'bm_error' into bm_meta
Diffstat (limited to 'tools')
41 files changed, 582 insertions, 229 deletions
diff --git a/tools/codegen/core/gen_static_metadata.py b/tools/codegen/core/gen_static_metadata.py index 4703708aaa..109701f740 100755 --- a/tools/codegen/core/gen_static_metadata.py +++ b/tools/codegen/core/gen_static_metadata.py @@ -123,7 +123,6 @@ CONFIG = [ ('if-unmodified-since', ''), ('last-modified', ''), ('lb-token', ''), - ('lb-cost-bin', ''), ('link', ''), ('location', ''), ('max-forwards', ''), @@ -160,7 +159,6 @@ METADATA_BATCH_CALLOUTS = [ 'user-agent', 'host', 'lb-token', - 'lb-cost-bin', ] COMPRESSION_ALGORITHMS = [ diff --git a/tools/distrib/check_copyright.py b/tools/distrib/check_copyright.py index c993407eb2..611a9f40ae 100755 --- a/tools/distrib/check_copyright.py +++ b/tools/distrib/check_copyright.py @@ -106,6 +106,8 @@ _EXEMPT = frozenset(( 'src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h', 'src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.c', + 'src/cpp/server/health/health.pb.h', + 'src/cpp/server/health/health.pb.c', # An older file originally from outside gRPC. 'src/php/tests/bootstrap.php', diff --git a/tools/doxygen/Doxyfile.c++ b/tools/doxygen/Doxyfile.c++ index 0706cdca0f..551ba46b3b 100644 --- a/tools/doxygen/Doxyfile.c++ +++ b/tools/doxygen/Doxyfile.c++ @@ -795,9 +795,11 @@ include/grpc++/client_context.h \ include/grpc++/completion_queue.h \ include/grpc++/create_channel.h \ include/grpc++/create_channel_posix.h \ +include/grpc++/ext/health_check_service_server_builder_option.h \ include/grpc++/generic/async_generic_service.h \ include/grpc++/generic/generic_stub.h \ include/grpc++/grpc++.h \ +include/grpc++/health_check_service_interface.h \ include/grpc++/impl/call.h \ include/grpc++/impl/client_unary_call.h \ include/grpc++/impl/codegen/async_stream.h \ diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal index cb0a3e55f8..0d7ddfc6d6 100644 --- a/tools/doxygen/Doxyfile.c++.internal +++ b/tools/doxygen/Doxyfile.c++.internal @@ -795,9 +795,11 @@ include/grpc++/client_context.h \ include/grpc++/completion_queue.h \ include/grpc++/create_channel.h \ include/grpc++/create_channel_posix.h \ +include/grpc++/ext/health_check_service_server_builder_option.h \ include/grpc++/generic/async_generic_service.h \ include/grpc++/generic/generic_stub.h \ include/grpc++/grpc++.h \ +include/grpc++/health_check_service_interface.h \ include/grpc++/impl/call.h \ include/grpc++/impl/client_unary_call.h \ include/grpc++/impl/codegen/async_stream.h \ @@ -913,6 +915,12 @@ src/cpp/server/async_generic_service.cc \ src/cpp/server/create_default_thread_pool.cc \ src/cpp/server/dynamic_thread_pool.cc \ src/cpp/server/dynamic_thread_pool.h \ +src/cpp/server/health/default_health_check_service.cc \ +src/cpp/server/health/default_health_check_service.h \ +src/cpp/server/health/health.pb.c \ +src/cpp/server/health/health.pb.h \ +src/cpp/server/health/health_check_service.cc \ +src/cpp/server/health/health_check_service_server_builder_option.cc \ src/cpp/server/insecure_server_credentials.cc \ src/cpp/server/secure_server_credentials.cc \ src/cpp/server/secure_server_credentials.h \ diff --git a/tools/doxygen/Doxyfile.core b/tools/doxygen/Doxyfile.core index 060e09869d..896ee0213b 100644 --- a/tools/doxygen/Doxyfile.core +++ b/tools/doxygen/Doxyfile.core @@ -827,6 +827,7 @@ include/grpc/impl/codegen/sync_posix.h \ include/grpc/impl/codegen/sync_posix.h \ include/grpc/impl/codegen/sync_windows.h \ include/grpc/impl/codegen/sync_windows.h \ +include/grpc/load_reporting.h \ include/grpc/slice.h \ include/grpc/slice_buffer.h \ include/grpc/status.h \ diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index ce7e103647..10801254ef 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -827,6 +827,7 @@ include/grpc/impl/codegen/sync_posix.h \ include/grpc/impl/codegen/sync_posix.h \ include/grpc/impl/codegen/sync_windows.h \ include/grpc/impl/codegen/sync_windows.h \ +include/grpc/load_reporting.h \ include/grpc/slice.h \ include/grpc/slice_buffer.h \ include/grpc/status.h \ diff --git a/tools/internal_ci/linux/grpc_fuzzer_api.cfg b/tools/internal_ci/linux/grpc_fuzzer_api.cfg index a34fb9d47e..5c2592e933 100644 --- a/tools/internal_ci/linux/grpc_fuzzer_api.cfg +++ b/tools/internal_ci/linux/grpc_fuzzer_api.cfg @@ -1,5 +1,4 @@ -#!/bin/bash -# Copyright 2016, Google Inc. +# Copyright 2017, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without diff --git a/tools/internal_ci/linux/grpc_fuzzer_api.sh b/tools/internal_ci/linux/grpc_fuzzer_api.sh index c3cf1109de..b1f792e765 100755 --- a/tools/internal_ci/linux/grpc_fuzzer_api.sh +++ b/tools/internal_ci/linux/grpc_fuzzer_api.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2016, Google Inc. +# Copyright 2017, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without diff --git a/tools/internal_ci/linux/grpc_fuzzer_client.cfg b/tools/internal_ci/linux/grpc_fuzzer_client.cfg index b1bce02282..1e8f688576 100644 --- a/tools/internal_ci/linux/grpc_fuzzer_client.cfg +++ b/tools/internal_ci/linux/grpc_fuzzer_client.cfg @@ -1,5 +1,4 @@ -#!/bin/bash -# Copyright 2016, Google Inc. +# Copyright 2017, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without diff --git a/tools/internal_ci/linux/grpc_fuzzer_client.sh b/tools/internal_ci/linux/grpc_fuzzer_client.sh index f9ff13d303..c03f92559c 100755 --- a/tools/internal_ci/linux/grpc_fuzzer_client.sh +++ b/tools/internal_ci/linux/grpc_fuzzer_client.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2016, Google Inc. +# Copyright 2017, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without diff --git a/tools/internal_ci/linux/grpc_fuzzer_hpack_parser.cfg b/tools/internal_ci/linux/grpc_fuzzer_hpack_parser.cfg index 215ce2bf9c..72482b62e3 100644 --- a/tools/internal_ci/linux/grpc_fuzzer_hpack_parser.cfg +++ b/tools/internal_ci/linux/grpc_fuzzer_hpack_parser.cfg @@ -1,5 +1,4 @@ -#!/bin/bash -# Copyright 2016, Google Inc. +# Copyright 2017, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without diff --git a/tools/internal_ci/linux/grpc_fuzzer_hpack_parser.sh b/tools/internal_ci/linux/grpc_fuzzer_hpack_parser.sh index d9a73a622b..40179aa547 100755 --- a/tools/internal_ci/linux/grpc_fuzzer_hpack_parser.sh +++ b/tools/internal_ci/linux/grpc_fuzzer_hpack_parser.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2016, Google Inc. +# Copyright 2017, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without diff --git a/tools/internal_ci/linux/grpc_fuzzer_http_request.cfg b/tools/internal_ci/linux/grpc_fuzzer_http_request.cfg index 120e8f8f76..a4a0e8922e 100644 --- a/tools/internal_ci/linux/grpc_fuzzer_http_request.cfg +++ b/tools/internal_ci/linux/grpc_fuzzer_http_request.cfg @@ -1,5 +1,4 @@ -#!/bin/bash -# Copyright 2016, Google Inc. +# Copyright 2017, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without diff --git a/tools/internal_ci/linux/grpc_fuzzer_http_request.sh b/tools/internal_ci/linux/grpc_fuzzer_http_request.sh index d412d921ba..6dac4ed9ab 100755 --- a/tools/internal_ci/linux/grpc_fuzzer_http_request.sh +++ b/tools/internal_ci/linux/grpc_fuzzer_http_request.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2016, Google Inc. +# Copyright 2017, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without diff --git a/tools/internal_ci/linux/grpc_fuzzer_json.cfg b/tools/internal_ci/linux/grpc_fuzzer_json.cfg index cab4f293ed..d22da2d705 100644 --- a/tools/internal_ci/linux/grpc_fuzzer_json.cfg +++ b/tools/internal_ci/linux/grpc_fuzzer_json.cfg @@ -1,5 +1,4 @@ -#!/bin/bash -# Copyright 2016, Google Inc. +# Copyright 2017, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without diff --git a/tools/internal_ci/linux/grpc_fuzzer_json.sh b/tools/internal_ci/linux/grpc_fuzzer_json.sh index d9869f6c30..b002ecb262 100755 --- a/tools/internal_ci/linux/grpc_fuzzer_json.sh +++ b/tools/internal_ci/linux/grpc_fuzzer_json.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2016, Google Inc. +# Copyright 2017, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without diff --git a/tools/internal_ci/linux/grpc_fuzzer_nanopb_response.cfg b/tools/internal_ci/linux/grpc_fuzzer_nanopb_response.cfg index c73aa819ee..cbf44ba29e 100644 --- a/tools/internal_ci/linux/grpc_fuzzer_nanopb_response.cfg +++ b/tools/internal_ci/linux/grpc_fuzzer_nanopb_response.cfg @@ -1,5 +1,4 @@ -#!/bin/bash -# Copyright 2016, Google Inc. +# Copyright 2017, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without diff --git a/tools/internal_ci/linux/grpc_fuzzer_nanopb_response.sh b/tools/internal_ci/linux/grpc_fuzzer_nanopb_response.sh index 0a7187f8bf..51a8eb58bb 100755 --- a/tools/internal_ci/linux/grpc_fuzzer_nanopb_response.sh +++ b/tools/internal_ci/linux/grpc_fuzzer_nanopb_response.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2016, Google Inc. +# Copyright 2017, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without diff --git a/tools/internal_ci/linux/grpc_fuzzer_server.cfg b/tools/internal_ci/linux/grpc_fuzzer_server.cfg index a1931cb891..7877d51792 100644 --- a/tools/internal_ci/linux/grpc_fuzzer_server.cfg +++ b/tools/internal_ci/linux/grpc_fuzzer_server.cfg @@ -1,5 +1,4 @@ -#!/bin/bash -# Copyright 2016, Google Inc. +# Copyright 2017, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without diff --git a/tools/internal_ci/linux/grpc_fuzzer_server.sh b/tools/internal_ci/linux/grpc_fuzzer_server.sh index e00e940382..b6648db34d 100755 --- a/tools/internal_ci/linux/grpc_fuzzer_server.sh +++ b/tools/internal_ci/linux/grpc_fuzzer_server.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2016, Google Inc. +# Copyright 2017, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without diff --git a/tools/internal_ci/linux/grpc_fuzzer_uri.cfg b/tools/internal_ci/linux/grpc_fuzzer_uri.cfg index c312ae0464..134b3d06d6 100644 --- a/tools/internal_ci/linux/grpc_fuzzer_uri.cfg +++ b/tools/internal_ci/linux/grpc_fuzzer_uri.cfg @@ -1,5 +1,4 @@ -#!/bin/bash -# Copyright 2016, Google Inc. +# Copyright 2017, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without diff --git a/tools/internal_ci/linux/grpc_fuzzer_uri.sh b/tools/internal_ci/linux/grpc_fuzzer_uri.sh index 4137f8061c..e3e46515e2 100755 --- a/tools/internal_ci/linux/grpc_fuzzer_uri.sh +++ b/tools/internal_ci/linux/grpc_fuzzer_uri.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2016, Google Inc. +# Copyright 2017, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without diff --git a/tools/internal_ci/linux/grpc_interop.cfg b/tools/internal_ci/linux/grpc_interop_badserver_java.cfg index 9259faf34d..e521b085c5 100644 --- a/tools/internal_ci/linux/grpc_interop.cfg +++ b/tools/internal_ci/linux/grpc_interop_badserver_java.cfg @@ -1,4 +1,3 @@ -#!/bin/bash # Copyright 2017, Google Inc. # All rights reserved. # @@ -31,11 +30,11 @@ # Config file for the internal CI (in protobuf text format) # Location of the continuous shell script in repository. -build_file: "grpc/tools/internal_ci/linux/grpc_interop.sh" +build_file: "grpc/tools/internal_ci/linux/grpc_interop_badserver_java.sh" # grpc_interop tests can take 6+ hours to complete. timeout_mins: 480 action { define_artifacts { - regex: "**/sponge_log.xml" + regex: "**/report.xml" } } diff --git a/tools/internal_ci/linux/grpc_interop.sh b/tools/internal_ci/linux/grpc_interop_badserver_java.sh index 68bb4199e3..0985e657c6 100755 --- a/tools/internal_ci/linux/grpc_interop.sh +++ b/tools/internal_ci/linux/grpc_interop_badserver_java.sh @@ -37,6 +37,5 @@ cd $(dirname $0)/../../.. git submodule update --init -tools/run_tests/run_interop_tests.py -l all -s all --cloud_to_prod --cloud_to_prod_auth --use_docker --http2_interop -t -j 12 $@ || FAILED="true" -tools/run_tests/run_interop_tests.py -l java --use_docker --http2_badserver_interop $@ || FAILED="true" -tools/run_tests/run_interop_tests.py -l python --use_docker --http2_badserver_interop $@ || FAILED="true" +tools/run_tests/run_interop_tests.py -l java --use_docker --http2_badserver_interop $@ + diff --git a/tools/internal_ci/linux/grpc_interop_badserver_python.cfg b/tools/internal_ci/linux/grpc_interop_badserver_python.cfg new file mode 100644 index 0000000000..940f760e97 --- /dev/null +++ b/tools/internal_ci/linux/grpc_interop_badserver_python.cfg @@ -0,0 +1,40 @@ +# Copyright 2017, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/linux/grpc_interop_badserver_python.sh" +# grpc_interop tests can take 6+ hours to complete. +timeout_mins: 480 +action { + define_artifacts { + regex: "**/report.xml" + } +} diff --git a/tools/internal_ci/linux/grpc_interop_badserver_python.sh b/tools/internal_ci/linux/grpc_interop_badserver_python.sh new file mode 100755 index 0000000000..3fff537d2b --- /dev/null +++ b/tools/internal_ci/linux/grpc_interop_badserver_python.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +# Copyright 2017, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +set -ex + +export LANG=en_US.UTF-8 + +# Enter the gRPC repo root +cd $(dirname $0)/../../.. + +git submodule update --init + +tools/run_tests/run_interop_tests.py -l python --use_docker --http2_badserver_interop $@ + diff --git a/tools/internal_ci/linux/grpc_interop_tocloud.cfg b/tools/internal_ci/linux/grpc_interop_tocloud.cfg new file mode 100644 index 0000000000..2b536446c1 --- /dev/null +++ b/tools/internal_ci/linux/grpc_interop_tocloud.cfg @@ -0,0 +1,40 @@ +# Copyright 2017, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/linux/grpc_interop_tocloud.sh" +# grpc_interop tests can take 6+ hours to complete. +timeout_mins: 480 +action { + define_artifacts { + regex: "**/report.xml" + } +} diff --git a/tools/internal_ci/linux/grpc_interop_tocloud.sh b/tools/internal_ci/linux/grpc_interop_tocloud.sh new file mode 100755 index 0000000000..572001d944 --- /dev/null +++ b/tools/internal_ci/linux/grpc_interop_tocloud.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +# Copyright 2017, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +set -ex + +export LANG=en_US.UTF-8 + +# Enter the gRPC repo root +cd $(dirname $0)/../../.. + +git submodule update --init + +tools/run_tests/run_interop_tests.py -l all -s all --use_docker --http2_interop -t -j 12 $@ diff --git a/tools/internal_ci/linux/grpc_master.cfg b/tools/internal_ci/linux/grpc_master.cfg index 8ce2ef11a2..7536a91a67 100644 --- a/tools/internal_ci/linux/grpc_master.cfg +++ b/tools/internal_ci/linux/grpc_master.cfg @@ -1,5 +1,4 @@ -#!/bin/bash -# Copyright 2016, Google Inc. +# Copyright 2017, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -32,7 +31,7 @@ # Location of the continuous shell script in repository. build_file: "grpc/tools/internal_ci/linux/grpc_master.sh" -timeout_mins: 60 +timeout_mins: 240 action { define_artifacts { regex: "**/sponge_log.xml" diff --git a/tools/internal_ci/linux/grpc_master.sh b/tools/internal_ci/linux/grpc_master.sh index ea77d11305..397e1bd713 100755 --- a/tools/internal_ci/linux/grpc_master.sh +++ b/tools/internal_ci/linux/grpc_master.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2016, Google Inc. +# Copyright 2017, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without diff --git a/tools/internal_ci/linux/grpc_portability_build_only.cfg b/tools/internal_ci/linux/grpc_portability_build_only.cfg new file mode 100644 index 0000000000..ce5be5abe9 --- /dev/null +++ b/tools/internal_ci/linux/grpc_portability_build_only.cfg @@ -0,0 +1,39 @@ +# Copyright 2017, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/linux/grpc_portability_build_only.sh" +timeout_mins: 180 +action { + define_artifacts { + regex: "**report**.xml" + } +} diff --git a/tools/internal_ci/linux/grpc_portability_build_only.sh b/tools/internal_ci/linux/grpc_portability_build_only.sh new file mode 100644 index 0000000000..9fac5bcac0 --- /dev/null +++ b/tools/internal_ci/linux/grpc_portability_build_only.sh @@ -0,0 +1,38 @@ +#!/bin/bash +# Copyright 2017, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +set -ex + +# change to grpc repo root +cd $(dirname $0)/../../.. + +git submodule update --init + +tools/jenkins/run_jenkins_matrix.sh -f portability linux --build_only diff --git a/tools/internal_ci/windows/grpc_master.bat b/tools/internal_ci/windows/grpc_master.bat new file mode 100644 index 0000000000..4041c50313 --- /dev/null +++ b/tools/internal_ci/windows/grpc_master.bat @@ -0,0 +1,44 @@ +@rem Copyright 2017, Google Inc. +@rem All rights reserved. +@rem +@rem Redistribution and use in source and binary forms, with or without +@rem modification, are permitted provided that the following conditions are +@rem met: +@rem +@rem * Redistributions of source code must retain the above copyright +@rem notice, this list of conditions and the following disclaimer. +@rem * Redistributions in binary form must reproduce the above +@rem copyright notice, this list of conditions and the following disclaimer +@rem in the documentation and/or other materials provided with the +@rem distribution. +@rem * Neither the name of Google Inc. nor the names of its +@rem contributors may be used to endorse or promote products derived from +@rem this software without specific prior written permission. +@rem +@rem THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +@rem "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +@rem LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +@rem A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +@rem OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +@rem SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +@rem LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +@rem DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +@rem THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +@rem (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +@rem OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +setlocal + +@rem enter repo root +cd /d %~dp0\..\..\.. + +git submodule update --init + +sh tools\run_tests\helper_scripts\run_tests_in_workspace.sh -t -j 4 -x c_windows_dbg_sponge_log.xml --report_suite_name c_windows_dbg -l c -c dbg +sh tools\run_tests\helper_scripts\run_tests_in_workspace.sh -t -j 4 -x c_windows_opt_sponge_log.xml --report_suite_name c_windows_opt -l c -c opt +sh tools\run_tests\helper_scripts\run_tests_in_workspace.sh -t -j 4 -x csharp_windows_dbg_sponge_log.xml --report_suite_name csharp_windows_dbg -l csharp -c dbg +sh tools\run_tests\helper_scripts\run_tests_in_workspace.sh -t -j 4 -x csharp_windows_opt_sponge_log.xml --report_suite_name csharp_windows_opt -l csharp -c opt +sh tools\run_tests\helper_scripts\run_tests_in_workspace.sh -t -j 4 -x node_windows_dbg_sponge_log.xml --report_suite_name node_windows_dbg -l node -c dbg +sh tools\run_tests\helper_scripts\run_tests_in_workspace.sh -t -j 4 -x node_windows_opt_sponge_log.xml --report_suite_name node_windows_opt -l node -c opt +sh tools\run_tests\helper_scripts\run_tests_in_workspace.sh -t -j 4 -x python_windows_dbg_sponge_log.xml --report_suite_name python_windows_dbg -l python -c dbg +sh tools\run_tests\helper_scripts\run_tests_in_workspace.sh -t -j 4 -x python_windows_opt_sponge_log.xml --report_suite_name python_windows_opt -l python -c opt diff --git a/tools/internal_ci/windows/grpc_master.cfg b/tools/internal_ci/windows/grpc_master.cfg new file mode 100644 index 0000000000..f90af11308 --- /dev/null +++ b/tools/internal_ci/windows/grpc_master.cfg @@ -0,0 +1,39 @@ +# Copyright 2017, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/windows/grpc_master.bat" +timeout_mins: 360 +action { + define_artifacts { + regex: "**sponge_log.xml" + } +} diff --git a/tools/profiling/microbenchmarks/bm2bq.py b/tools/profiling/microbenchmarks/bm2bq.py index 8f728b5c3f..3aa700388e 100755 --- a/tools/profiling/microbenchmarks/bm2bq.py +++ b/tools/profiling/microbenchmarks/bm2bq.py @@ -66,6 +66,8 @@ columns = [ ('cli_stream_stalls_per_iteration', 'float'), ('svr_transport_stalls_per_iteration', 'float'), ('svr_stream_stalls_per_iteration', 'float'), + ('atm_cas_per_iteration', 'float'), + ('atm_add_per_iteration', 'float') ] if sys.argv[1] == '--schema': @@ -126,6 +128,10 @@ bm_specs = { 'tpl': ['fixture'], 'dyn': [], }, + 'BM_IsolatedFilter' : { + 'tpl': ['fixture', 'client_mutator'], + 'dyn': [], + } } def numericalize(s): @@ -182,7 +188,7 @@ def parse_name(name): for bm in js['benchmarks']: context = js['context'] if 'label' in bm: - labels_list = [s.split(':') for s in bm['label'].split(' ')] + labels_list = [s.split(':') for s in bm['label'].strip().split(' ') if len(s) and s[0] != '#'] for el in labels_list: el[0] = el[0].replace('/iter', '_per_iteration') labels = dict(labels_list) diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json index cb1833dfb5..3808b5cf13 100644 --- a/tools/run_tests/generated/sources_and_headers.json +++ b/tools/run_tests/generated/sources_and_headers.json @@ -1299,57 +1299,6 @@ "headers": [], "is_filegroup": false, "language": "c", - "name": "internal_api_canary_iomgr_test", - "src": [ - "test/core/internal_api_canaries/iomgr.c" - ], - "third_party": false, - "type": "target" - }, - { - "deps": [ - "gpr", - "gpr_test_util", - "grpc", - "grpc_test_util" - ], - "headers": [], - "is_filegroup": false, - "language": "c", - "name": "internal_api_canary_support_test", - "src": [ - "test/core/internal_api_canaries/iomgr.c" - ], - "third_party": false, - "type": "target" - }, - { - "deps": [ - "gpr", - "gpr_test_util", - "grpc", - "grpc_test_util" - ], - "headers": [], - "is_filegroup": false, - "language": "c", - "name": "internal_api_canary_transport_test", - "src": [ - "test/core/internal_api_canaries/iomgr.c" - ], - "third_party": false, - "type": "target" - }, - { - "deps": [ - "gpr", - "gpr_test_util", - "grpc", - "grpc_test_util" - ], - "headers": [], - "is_filegroup": false, - "language": "c", "name": "invalid_call_argument_test", "src": [ "test/core/end2end/invalid_call_argument_test.c" @@ -2954,6 +2903,25 @@ }, { "deps": [ + "gpr", + "gpr_test_util", + "grpc", + "grpc++", + "grpc++_test_util", + "grpc_test_util" + ], + "headers": [], + "is_filegroup": false, + "language": "c++", + "name": "health_service_end2end_test", + "src": [ + "test/cpp/end2end/health_service_end2end_test.cc" + ], + "third_party": false, + "type": "target" + }, + { + "deps": [ "grpc", "grpc++", "grpc++_test_config", @@ -5552,23 +5520,6 @@ "type": "lib" }, { - "deps": [ - "grpc++" - ], - "headers": [ - "include/grpc++/test/server_context_test_spouse.h" - ], - "is_filegroup": false, - "language": "c++", - "name": "grpc++_test", - "src": [ - "include/grpc++/test/server_context_test_spouse.h", - "src/cpp/test/server_context_test_spouse.cc" - ], - "third_party": false, - "type": "lib" - }, - { "deps": [], "headers": [ "test/cpp/util/test_config.h" @@ -5594,6 +5545,8 @@ "thrift_util" ], "headers": [ + "src/proto/grpc/health/v1/health.grpc.pb.h", + "src/proto/grpc/health/v1/health.pb.h", "src/proto/grpc/testing/duplicate/echo_duplicate.grpc.pb.h", "src/proto/grpc/testing/duplicate/echo_duplicate.pb.h", "src/proto/grpc/testing/echo.grpc.pb.h", @@ -7160,6 +7113,7 @@ "include/grpc/grpc.h", "include/grpc/grpc_posix.h", "include/grpc/grpc_security_constants.h", + "include/grpc/load_reporting.h", "include/grpc/slice.h", "include/grpc/slice_buffer.h", "include/grpc/status.h", @@ -7280,6 +7234,7 @@ "include/grpc/grpc.h", "include/grpc/grpc_posix.h", "include/grpc/grpc_security_constants.h", + "include/grpc/load_reporting.h", "include/grpc/slice.h", "include/grpc/slice_buffer.h", "include/grpc/status.h", @@ -8181,9 +8136,11 @@ "include/grpc++/completion_queue.h", "include/grpc++/create_channel.h", "include/grpc++/create_channel_posix.h", + "include/grpc++/ext/health_check_service_server_builder_option.h", "include/grpc++/generic/async_generic_service.h", "include/grpc++/generic/generic_stub.h", "include/grpc++/grpc++.h", + "include/grpc++/health_check_service_interface.h", "include/grpc++/impl/call.h", "include/grpc++/impl/client_unary_call.h", "include/grpc++/impl/codegen/core_codegen.h", @@ -8220,6 +8177,8 @@ "src/cpp/client/create_channel_internal.h", "src/cpp/common/channel_filter.h", "src/cpp/server/dynamic_thread_pool.h", + "src/cpp/server/health/default_health_check_service.h", + "src/cpp/server/health/health.pb.h", "src/cpp/server/thread_pool_interface.h", "src/cpp/thread_manager/thread_manager.h" ], @@ -8233,9 +8192,11 @@ "include/grpc++/completion_queue.h", "include/grpc++/create_channel.h", "include/grpc++/create_channel_posix.h", + "include/grpc++/ext/health_check_service_server_builder_option.h", "include/grpc++/generic/async_generic_service.h", "include/grpc++/generic/generic_stub.h", "include/grpc++/grpc++.h", + "include/grpc++/health_check_service_interface.h", "include/grpc++/impl/call.h", "include/grpc++/impl/client_unary_call.h", "include/grpc++/impl/codegen/core_codegen.h", @@ -8289,6 +8250,12 @@ "src/cpp/server/create_default_thread_pool.cc", "src/cpp/server/dynamic_thread_pool.cc", "src/cpp/server/dynamic_thread_pool.h", + "src/cpp/server/health/default_health_check_service.cc", + "src/cpp/server/health/default_health_check_service.h", + "src/cpp/server/health/health.pb.c", + "src/cpp/server/health/health.pb.h", + "src/cpp/server/health/health_check_service.cc", + "src/cpp/server/health/health_check_service_server_builder_option.cc", "src/cpp/server/server_builder.cc", "src/cpp/server/server_cc.cc", "src/cpp/server/server_context.cc", @@ -8440,6 +8407,22 @@ }, { "deps": [ + "grpc++" + ], + "headers": [ + "include/grpc++/test/server_context_test_spouse.h" + ], + "is_filegroup": true, + "language": "c++", + "name": "grpc++_test", + "src": [ + "include/grpc++/test/server_context_test_spouse.h" + ], + "third_party": false, + "type": "filegroup" + }, + { + "deps": [ "grpc++_codegen_base" ], "headers": [ diff --git a/tools/run_tests/generated/tests.json b/tools/run_tests/generated/tests.json index 7486f0989a..ddc7424555 100644 --- a/tools/run_tests/generated/tests.json +++ b/tools/run_tests/generated/tests.json @@ -1529,28 +1529,6 @@ "posix", "windows" ], - "cpu_cost": 0.1, - "exclude_configs": [], - "exclude_iomgrs": [], - "flaky": true, - "gtest": false, - "language": "c", - "name": "lb_policies_test", - "platforms": [ - "linux", - "mac", - "posix", - "windows" - ] - }, - { - "args": [], - "ci_platforms": [ - "linux", - "mac", - "posix", - "windows" - ], "cpu_cost": 1.0, "exclude_configs": [], "exclude_iomgrs": [], @@ -2984,6 +2962,28 @@ "cpu_cost": 1.0, "exclude_configs": [], "exclude_iomgrs": [], + "flaky": false, + "gtest": true, + "language": "c++", + "name": "health_service_end2end_test", + "platforms": [ + "linux", + "mac", + "posix", + "windows" + ] + }, + { + "args": [], + "ci_platforms": [ + "linux", + "mac", + "posix", + "windows" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [], "flaky": true, "gtest": false, "language": "c++", diff --git a/tools/run_tests/python_utils/start_port_server.py b/tools/run_tests/python_utils/start_port_server.py index 8ee7080168..4c9f6aac63 100644 --- a/tools/run_tests/python_utils/start_port_server.py +++ b/tools/run_tests/python_utils/start_port_server.py @@ -30,101 +30,109 @@ from __future__ import print_function from six.moves import urllib +import jobset +import logging import os +import socket import subprocess -import tempfile import sys +import tempfile import time -import jobset -import socket + def start_port_server(port_server_port): - # check if a compatible port server is running - # if incompatible (version mismatch) ==> start a new one - # if not running ==> start a new one - # otherwise, leave it up - try: - version = int(urllib.request.urlopen( - 'http://localhost:%d/version_number' % port_server_port, - timeout=10).read()) - print('detected port server running version %d' % version) - running = True - except Exception as e: - print('failed to detect port server: %s' % sys.exc_info()[0]) - print(e.strerror) - running = False - if running: - current_version = int(subprocess.check_output( - [sys.executable, os.path.abspath('tools/run_tests/python_utils/port_server.py'), - 'dump_version'])) - print('my port server is version %d' % current_version) - running = (version >= current_version) + # check if a compatible port server is running + # if incompatible (version mismatch) ==> start a new one + # if not running ==> start a new one + # otherwise, leave it up + try: + version = int( + urllib.request.urlopen( + 'http://localhost:%d/version_number' % port_server_port, + timeout=10).read()) + logging.info('detected port server running version %d', version) + running = True + except Exception as e: + logging.exception('failed to detect port server') + running = False + if running: + current_version = int( + subprocess.check_output([ + sys.executable, os.path.abspath( + 'tools/run_tests/python_utils/port_server.py'), + 'dump_version' + ])) + logging.info('my port server is version %d', current_version) + running = (version >= current_version) + if not running: + logging.info('port_server version mismatch: killing the old one') + urllib.request.urlopen('http://localhost:%d/quitquitquit' % + port_server_port).read() + time.sleep(1) if not running: - print('port_server version mismatch: killing the old one') - urllib.request.urlopen('http://localhost:%d/quitquitquit' % port_server_port).read() - time.sleep(1) - if not running: - fd, logfile = tempfile.mkstemp() - os.close(fd) - print('starting port_server, with log file %s' % logfile) - args = [sys.executable, os.path.abspath('tools/run_tests/python_utils/port_server.py'), - '-p', '%d' % port_server_port, '-l', logfile] - env = dict(os.environ) - env['BUILD_ID'] = 'pleaseDontKillMeJenkins' - if jobset.platform_string() == 'windows': - # Working directory of port server needs to be outside of Jenkins - # workspace to prevent file lock issues. - tempdir = tempfile.mkdtemp() - port_server = subprocess.Popen( - args, - env=env, - cwd=tempdir, - creationflags = 0x00000008, # detached process - close_fds=True) - else: - port_server = subprocess.Popen( - args, - env=env, - preexec_fn=os.setsid, - close_fds=True) - time.sleep(1) - # ensure port server is up - waits = 0 - while True: - if waits > 10: - print('killing port server due to excessive start up waits') - port_server.kill() - if port_server.poll() is not None: - print('port_server failed to start') - # try one final time: maybe another build managed to start one + fd, logfile = tempfile.mkstemp() + os.close(fd) + logging.info('starting port_server, with log file %s', logfile) + args = [ + sys.executable, + os.path.abspath('tools/run_tests/python_utils/port_server.py'), + '-p', '%d' % port_server_port, '-l', logfile + ] + env = dict(os.environ) + env['BUILD_ID'] = 'pleaseDontKillMeJenkins' + if jobset.platform_string() == 'windows': + # Working directory of port server needs to be outside of Jenkins + # workspace to prevent file lock issues. + tempdir = tempfile.mkdtemp() + port_server = subprocess.Popen( + args, + env=env, + cwd=tempdir, + creationflags=0x00000008, # detached process + close_fds=True) + else: + port_server = subprocess.Popen( + args, env=env, preexec_fn=os.setsid, close_fds=True) time.sleep(1) - try: - urllib.request.urlopen('http://localhost:%d/get' % port_server_port, - timeout=1).read() - print('last ditch attempt to contact port server succeeded') - break - except: - traceback.print_exc() - port_log = open(logfile, 'r').read() - print(port_log) - sys.exit(1) - try: - urllib.request.urlopen('http://localhost:%d/get' % port_server_port, + # ensure port server is up + waits = 0 + while True: + if waits > 10: + logging.warning( + 'killing port server due to excessive start up waits') + port_server.kill() + if port_server.poll() is not None: + logging.error('port_server failed to start') + # try one final time: maybe another build managed to start one + time.sleep(1) + try: + urllib.request.urlopen( + 'http://localhost:%d/get' % port_server_port, timeout=1).read() - print('port server is up and ready') - break - except socket.timeout: - print('waiting for port_server: timeout') - traceback.print_exc(); - time.sleep(1) - waits += 1 - except urllib.error.URLError: - print('waiting for port_server: urlerror') - traceback.print_exc(); - time.sleep(1) - waits += 1 - except: - traceback.print_exc() - port_server.kill() - raise - + logging.info( + 'last ditch attempt to contact port server succeeded') + break + except: + logging.exception( + 'final attempt to contact port server failed') + port_log = open(logfile, 'r').read() + print(port_log) + sys.exit(1) + try: + port_server_url = 'http://localhost:%d/get' % port_server_port + urllib.request.urlopen(port_server_url, timeout=1).read() + logging.info('port server is up and ready') + break + except socket.timeout: + logging.exception('while waiting for port_server') + time.sleep(1) + waits += 1 + except urllib.error.URLError: + logging.exception('while waiting for port_server') + time.sleep(1) + waits += 1 + except: + logging.exception('error while contacting port server at "%s".' + 'Will try killing it.', port_server_url) + port_server.kill() + raise diff --git a/tools/run_tests/run_microbenchmark.py b/tools/run_tests/run_microbenchmark.py index 2bf4bc1d4d..261766a881 100755 --- a/tools/run_tests/run_microbenchmark.py +++ b/tools/run_tests/run_microbenchmark.py @@ -28,6 +28,7 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +import cgi import multiprocessing import os import subprocess @@ -71,11 +72,12 @@ def heading(name): def link(txt, tgt): global index_html - index_html += "<p><a href=\"%s\">%s</a></p>\n" % (tgt, txt) + index_html += "<p><a href=\"%s\">%s</a></p>\n" % ( + cgi.escape(tgt, quote=True), cgi.escape(txt)) def text(txt): global index_html - index_html += "<p><pre>%s</pre></p>\n" % txt + index_html += "<p><pre>%s</pre></p>\n" % cgi.escape(txt) def collect_latency(bm_name, args): """generate latency profiles""" @@ -91,7 +93,9 @@ def collect_latency(bm_name, args): '--benchmark_list_tests']).splitlines(): link(line, '%s.txt' % fnize(line)) benchmarks.append( - jobset.JobSpec(['bins/basicprof/%s' % bm_name, '--benchmark_filter=^%s$' % line], + jobset.JobSpec(['bins/basicprof/%s' % bm_name, + '--benchmark_filter=^%s$' % line, + '--benchmark_min_time=0.05'], environ={'LATENCY_TRACE': '%s.trace' % fnize(line)})) profile_analysis.append( jobset.JobSpec([sys.executable, @@ -103,7 +107,7 @@ def collect_latency(bm_name, args): # consume upwards of five gigabytes of ram in some cases, and so analysing # hundreds of them at once is impractical -- but we want at least some # concurrency or the work takes too long - if len(benchmarks) >= min(4, multiprocessing.cpu_count()): + if len(benchmarks) >= min(16, multiprocessing.cpu_count()): # run up to half the cpu count: each benchmark can use up to two cores # (one for the microbenchmark, one for the data flush) jobset.run(benchmarks, maxjobs=max(1, multiprocessing.cpu_count()/2), diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index 8b76193cbe..2d9eb29e7f 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -38,6 +38,7 @@ import collections import glob import itertools import json +import logging import multiprocessing import os import os.path @@ -80,6 +81,13 @@ def platform_string(): _DEFAULT_TIMEOUT_SECONDS = 5 * 60 +def run_shell_command(cmd, env=None, cwd=None): + try: + subprocess.check_output(cmd, shell=True, env=env, cwd=cwd) + except subprocess.CalledProcessError as e: + logging.exception("Error while running command '%s'. Exit status %d. Output:\n%s", + e.cmd, e.returncode, e.output) + raise # SimpleConfig: just compile with CONFIG=config, and run the binary to test class Config(object): @@ -1092,6 +1100,18 @@ def runs_per_test_type(arg_str): raise argparse.ArgumentTypeError(msg) +def percent_type(arg_str): + pct = float(arg_str) + if pct > 100 or pct < 0: + raise argparse.ArgumentTypeError( + "'%f' is not a valid percentage in the [0, 100] range" % pct) + return pct + +# This is math.isclose in python >= 3.5 +def isclose(a, b, rel_tol=1e-09, abs_tol=0.0): + return abs(a-b) <= max(rel_tol * max(abs(a), abs(b)), abs_tol) + + # parse command line argp = argparse.ArgumentParser(description='Run grpc tests.') argp.add_argument('-c', '--config', @@ -1104,6 +1124,8 @@ argp.add_argument('-r', '--regex', default='.*', type=str) argp.add_argument('--regex_exclude', default='', type=str) argp.add_argument('-j', '--jobs', default=multiprocessing.cpu_count(), type=int) argp.add_argument('-s', '--slowdown', default=1.0, type=float) +argp.add_argument('-p', '--sample_percent', default=100.0, type=percent_type, + help='Run a random sample with that percentage of tests') argp.add_argument('-f', '--forever', default=False, action='store_const', @@ -1199,7 +1221,7 @@ for spec in args.update_submodules: cwd = 'third_party/%s' % submodule def git(cmd, cwd=cwd): print('in %s: git %s' % (cwd, cmd)) - subprocess.check_call('git %s' % cmd, cwd=cwd, shell=True) + run_shell_command('git %s' % cmd, cwd=cwd) git('fetch') git('checkout %s' % branch) git('pull origin %s' % branch) @@ -1207,7 +1229,7 @@ for spec in args.update_submodules: need_to_regenerate_projects = True if need_to_regenerate_projects: if jobset.platform_string() == 'linux': - subprocess.check_call('tools/buildgen/generate_projects.sh', shell=True) + run_shell_command('tools/buildgen/generate_projects.sh') else: print('WARNING: may need to regenerate projects, but since we are not on') print(' Linux this step is being skipped. Compilation MAY fail.') @@ -1276,9 +1298,7 @@ if args.use_docker: if not args.travis: env['TTY_FLAG'] = '-t' # enables Ctrl-C when not on Jenkins. - subprocess.check_call(['tools/run_tests/dockerize/build_docker_and_run_tests.sh'], - shell=True, - env=env) + run_shell_command('tools/run_tests/dockerize/build_docker_and_run_tests.sh', env=env) sys.exit(0) _check_arch_option(args.arch) @@ -1438,8 +1458,18 @@ def _build_and_run( else: # whereas otherwise, we want to shuffle things up to give all tests a # chance to run. - massaged_one_run = list(one_run) # random.shuffle needs an indexable seq. - random.shuffle(massaged_one_run) # which it modifies in-place. + massaged_one_run = list(one_run) # random.sample needs an indexable seq. + num_jobs = len(massaged_one_run) + # for a random sample, get as many as indicated by the 'sample_percent' + # argument. By default this arg is 100, resulting in a shuffle of all + # jobs. + sample_size = int(num_jobs * args.sample_percent/100.0) + massaged_one_run = random.sample(massaged_one_run, sample_size) + if not isclose(args.sample_percent, 100.0): + print("Running %d tests out of %d (~%d%%)" % + (sample_size, num_jobs, args.sample_percent)) + else: + assert args.runs_per_test == 1, "Can't do sampling (-p) over multiple runs (-n)." if infinite_runs: assert len(massaged_one_run) > 0, 'Must have at least one test for a -n inf run' runs_sequence = (itertools.repeat(massaged_one_run) if infinite_runs @@ -1450,7 +1480,7 @@ def _build_and_run( jobset.message('START', 'Running tests quietly, only failing tests will be reported', do_newline=True) num_test_failures, resultset = jobset.run( all_runs, check_cancelled, newline_on_success=newline_on_success, - travis=args.travis, infinite_runs=infinite_runs, maxjobs=args.jobs, + travis=args.travis, maxjobs=args.jobs, stop_on_failure=args.stop_on_failure, add_env={'GRPC_TEST_PORT_SERVER': 'localhost:%d' % port_server_port}, quiet_success=args.quiet_success) diff --git a/tools/run_tests/sanity/check_submodules.sh b/tools/run_tests/sanity/check_submodules.sh index 0b68319d29..cfe4e2731c 100755 --- a/tools/run_tests/sanity/check_submodules.sh +++ b/tools/run_tests/sanity/check_submodules.sh @@ -44,7 +44,7 @@ cat << EOF | awk '{ print $1 }' | sort > $want_submodules 44c25c892a6229b20db7cd9dc05584ea865896de third_party/benchmark (v0.1.0-343-g44c25c8) 78684e5b222645828ca302e56b40b9daff2b2d27 third_party/boringssl (78684e5) 886e7d75368e3f4fab3f4d0d3584e4abfc557755 third_party/boringssl-with-bazel (version_for_cocoapods_7.0-857-g886e7d7) - f8a0efe03aa69b3336d8e228b37d4ccb17324b88 third_party/gflags (v2.2.0) + 30dbc81fb5ffdc98ea9b14b1918bfe4e8779b26e third_party/gflags (v2.2.0) c99458533a9b4c743ed51537e25989ea55944908 third_party/googletest (release-1.7.0) a428e42072765993ff674fda72863c9f1aa2d268 third_party/protobuf (v3.1.0-alpha-1) bcad91771b7f0bff28a1cac1981d7ef2b9bcef3c third_party/thrift (bcad917) |