diff options
Diffstat (limited to 'tools')
28 files changed, 529 insertions, 453 deletions
diff --git a/tools/buildgen/plugins/grpc_shadow_boringssl.py b/tools/buildgen/plugins/grpc_shadow_boringssl.py new file mode 100644 index 0000000000..da4d8c12af --- /dev/null +++ b/tools/buildgen/plugins/grpc_shadow_boringssl.py @@ -0,0 +1,32 @@ +# Copyright 2018 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Buldigen generate grpc_shadow_boringssl headers +This script takes the list of symbols from +src/objective-c/grpc_shadow_boringssl_symbols and populate them in +settings.grpc_shadow_boringssl_symbols +""" + + +def mako_plugin(dictionary): + with open('src/objective-c/grpc_shadow_boringssl_symbol_list') as f: + symbols = f.readlines() + # Remove trailing '\n' + symbols = [s.strip() for s in symbols] + # Remove comments + symbols = [s for s in symbols if s[0] != '#'] + # Remove the commit number + del symbols[0] + + settings = dictionary['settings'] + settings['grpc_shadow_boringssl_symbols'] = symbols diff --git a/tools/distrib/check_nanopb_output.sh b/tools/distrib/check_nanopb_output.sh index 6b98619c32..1c2ef9b768 100755 --- a/tools/distrib/check_nanopb_output.sh +++ b/tools/distrib/check_nanopb_output.sh @@ -16,6 +16,7 @@ set -ex readonly NANOPB_ALTS_TMP_OUTPUT="$(mktemp -d)" +readonly NANOPB_HEALTH_TMP_OUTPUT="$(mktemp -d)" readonly NANOPB_TMP_OUTPUT="$(mktemp -d)" readonly PROTOBUF_INSTALL_PREFIX="$(mktemp -d)" @@ -68,6 +69,23 @@ if ! diff -r "$NANOPB_TMP_OUTPUT" src/core/ext/filters/client_channel/lb_policy/ fi # +# checks for health.proto +# +readonly HEALTH_GRPC_OUTPUT_PATH='src/cpp/server/health' +# nanopb-compile the proto to a temp location +./tools/codegen/core/gen_nano_proto.sh \ + src/proto/grpc/health/v1/health.proto \ + "$NANOPB_HEALTH_TMP_OUTPUT" \ + "$HEALTH_GRPC_OUTPUT_PATH" +# compare outputs to checked compiled code +for NANOPB_OUTPUT_FILE in $NANOPB_HEALTH_TMP_OUTPUT/*.pb.*; do + if ! diff "$NANOPB_OUTPUT_FILE" "src/cpp/server/health/$(basename $NANOPB_OUTPUT_FILE)"; then + echo "Outputs differ: $NANOPB_HEALTH_TMP_OUTPUT vs $HEALTH_GRPC_OUTPUT_PATH" + exit 2 + fi +done + +# # Checks for handshaker.proto and transport_security_common.proto # readonly HANDSHAKER_GRPC_OUTPUT_PATH='src/core/tsi/alts/handshaker' diff --git a/tools/distrib/check_shadow_boringssl_symbol_list.sh b/tools/distrib/check_shadow_boringssl_symbol_list.sh new file mode 100755 index 0000000000..34ba09e07d --- /dev/null +++ b/tools/distrib/check_shadow_boringssl_symbol_list.sh @@ -0,0 +1,32 @@ +#!/bin/bash +# Copyright 2018 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# Check if the commit version of BoringSSL podspec, BoringSSL submodule, and +# the shadowed symbol list are all based on the same BoringSSL commit. +set -e + +cd $(dirname $0) + +boringssl_podspec_original="../../src/objective-c/BoringSSL-GRPC.podspec" +symbol_list="../../src/objective-c/grpc_shadow_boringssl_symbol_list" + +# Check BoringSSL version matches +ver1=$(git submodule |grep "boringssl " | awk '{print $1}' | head -n 1) +ver2=$(cat $boringssl_podspec_original | grep ':commit =>' | sed -E 's/.*"(.*)".*/\1/g') +ver3=$(cat $symbol_list | sed -n '2 p') +[ $ver1 == $ver2 ] && [ $ver1 == $ver3 ] || { echo "BoringSSL podspec (src/objective-c/BoringSSL.podspec), BoringSSL submodule (third_party/boringssl), and BoringSSL symbol list (src/objective-c/grpc_shadow_boringssl_symbol_list) commit do not match." ; echo "BoringSSL podspec: $ver1" ; echo "BoringSSL submodule: $ver2" ; echo "BoringSSL symbol list: $ver3" ; exit 1 ; } + +exit 0 diff --git a/tools/distrib/generate_grpc_shadow_boringssl_symbol_list.sh b/tools/distrib/generate_grpc_shadow_boringssl_symbol_list.sh new file mode 100755 index 0000000000..2e5bb44548 --- /dev/null +++ b/tools/distrib/generate_grpc_shadow_boringssl_symbol_list.sh @@ -0,0 +1,45 @@ +#!/bin/bash +# Copyright 2018 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Generate the list of boringssl symbols that need to be shadowed based on the +# current boringssl submodule. Requires local toolchain to build boringssl. +set -e + +cd $(dirname $0) + +symbol_list="../../src/objective-c/grpc_shadow_boringssl_symbol_list" + +ssl_lib='../../third_party/boringssl/build/ssl/libssl.a' +crypto_lib='../../third_party/boringssl/build/crypto/libcrypto.a' + +# Generate boringssl archives +( cd ../../third_party/boringssl ; mkdir -p build ; cd build ; cmake .. ; make ) + +# Generate shadow_boringssl.h +outputs="$(nm -C $ssl_lib)"$'\n'"$(nm -C $crypto_lib)" +symbols=$(echo "$outputs" | + grep '^[0-9a-f]* [A-Z] ' | # Only public symbols + grep -v ' bssl::' | # Filter BoringSSL symbols since they are already namespaced + sed 's/(.*//g' | # Remove parenthesis from C++ symbols + grep '^[0-9a-f]* [A-Z] _' | # Filter symbols that is not prefixed with '_' + sed 's/[0-9a-f]* [A-Z] _\(.*\)/\1/g') # Extract the symbol names + +commit=$(git submodule | grep "boringssl " | awk '{print $1}' | head -n 1) + +echo "# Automatically generated by tools/distrib/generate_grpc_shadow_boringssl_symbol_list.sh" > $symbol_list +echo $commit >> $symbol_list +echo "$symbols" >> $symbol_list + +exit 0 diff --git a/tools/distrib/python/grpcio_tools/grpc_tools/protobuf_generated_well_known_types_embed.cc b/tools/distrib/python/grpcio_tools/grpc_tools/protobuf_generated_well_known_types_embed.cc deleted file mode 100644 index ba93621e4f..0000000000 --- a/tools/distrib/python/grpcio_tools/grpc_tools/protobuf_generated_well_known_types_embed.cc +++ /dev/null @@ -1,328 +0,0 @@ -// Copyright 2017 gRPC authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// HACK: Embed the generated well_known_types_js.cc to make -// grpc-tools python package compilation easy. -#include <google/protobuf/compiler/js/well_known_types_embed.h> -struct FileToc well_known_types_js[] = { -{"any.js", - "// Protocol Buffers - Google's data interchange format\n" - "// Copyright 2008 Google Inc. All rights reserved.\n" - "// https://developers.google.com/protocol-buffers/\n" - "//\n" - "// Redistribution and use in source and binary forms, with or without\n" - "// modification, are permitted provided that the following conditions are\n" - "// met:\n" - "//\n" - "// * Redistributions of source code must retain the above copyright\n" - "// notice, this list of conditions and the following disclaimer.\n" - "// * Redistributions in binary form must reproduce the above\n" - "// copyright notice, this list of conditions and the following disclaimer\n" - "// in the documentation and/or other materials provided with the\n" - "// distribution.\n" - "// * Neither the name of Google Inc. nor the names of its\n" - "// contributors may be used to endorse or promote products derived from\n" - "// this software without specific prior written permission.\n" - "//\n" - "// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS\n" - "// \"AS IS\" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT\n" - "// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR\n" - "// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT\n" - "// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,\n" - "// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT\n" - "// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,\n" - "// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY\n" - "// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n" - "// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n" - "// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n" - "\n" - "/* This code will be inserted into generated code for\n" - " * google/protobuf/any.proto. */\n" - "\n" - "/**\n" - " * Returns the type name contained in this instance, if any.\n" - " * @return {string|undefined}\n" - " */\n" - "proto.google.protobuf.Any.prototype.getTypeName = function() {\n" - " return this.getTypeUrl().split('/').pop();\n" - "};\n" - "\n" - "\n" - "/**\n" - " * Packs the given message instance into this Any.\n" - " * @param {!Uint8Array} serialized The serialized data to pack.\n" - " * @param {string} name The type name of this message object.\n" - " * @param {string=} opt_typeUrlPrefix the type URL prefix.\n" - " */\n" - "proto.google.protobuf.Any.prototype.pack = function(serialized, name,\n" - " opt_typeUrlPrefix) {\n" - " if (!opt_typeUrlPrefix) {\n" - " opt_typeUrlPrefix = 'type.googleapis.com/';\n" - " }\n" - "\n" - " if (opt_typeUrlPrefix.substr(-1) != '/') {\n" - " this.setTypeUrl(opt_typeUrlPrefix + '/' + name);\n" - " } else {\n" - " this.setTypeUrl(opt_typeUrlPrefix + name);\n" - " }\n" - "\n" - " this.setValue(serialized);\n" - "};\n" - "\n" - "\n" - "/**\n" - " * @template T\n" - " * Unpacks this Any into the given message object.\n" - " * @param {function(Uint8Array):T} deserialize Function that will deserialize\n" - " * the binary data properly.\n" - " * @param {string} name The expected type name of this message object.\n" - " * @return {?T} If the name matched the expected name, returns the deserialized\n" - " * object, otherwise returns undefined.\n" - " */\n" - "proto.google.protobuf.Any.prototype.unpack = function(deserialize, name) {\n" - " if (this.getTypeName() == name) {\n" - " return deserialize(this.getValue_asU8());\n" - " } else {\n" - " return null;\n" - " }\n" - "};\n" -}, -{"struct.js", - "// Protocol Buffers - Google's data interchange format\n" - "// Copyright 2008 Google Inc. All rights reserved.\n" - "// https://developers.google.com/protocol-buffers/\n" - "//\n" - "// Redistribution and use in source and binary forms, with or without\n" - "// modification, are permitted provided that the following conditions are\n" - "// met:\n" - "//\n" - "// * Redistributions of source code must retain the above copyright\n" - "// notice, this list of conditions and the following disclaimer.\n" - "// * Redistributions in binary form must reproduce the above\n" - "// copyright notice, this list of conditions and the following disclaimer\n" - "// in the documentation and/or other materials provided with the\n" - "// distribution.\n" - "// * Neither the name of Google Inc. nor the names of its\n" - "// contributors may be used to endorse or promote products derived from\n" - "// this software without specific prior written permission.\n" - "//\n" - "// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS\n" - "// \"AS IS\" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT\n" - "// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR\n" - "// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT\n" - "// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,\n" - "// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT\n" - "// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,\n" - "// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY\n" - "// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n" - "// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n" - "// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n" - "\n" - "/* This code will be inserted into generated code for\n" - " * google/protobuf/struct.proto. */\n" - "\n" - "/**\n" - " * Typedef representing plain JavaScript values that can go into a\n" - " * Struct.\n" - " * @typedef {null|number|string|boolean|Array|Object}\n" - " */\n" - "proto.google.protobuf.JavaScriptValue;\n" - "\n" - "\n" - "/**\n" - " * Converts this Value object to a plain JavaScript value.\n" - " * @return {?proto.google.protobuf.JavaScriptValue} a plain JavaScript\n" - " * value representing this Struct.\n" - " */\n" - "proto.google.protobuf.Value.prototype.toJavaScript = function() {\n" - " var kindCase = proto.google.protobuf.Value.KindCase;\n" - " switch (this.getKindCase()) {\n" - " case kindCase.NULL_VALUE:\n" - " return null;\n" - " case kindCase.NUMBER_VALUE:\n" - " return this.getNumberValue();\n" - " case kindCase.STRING_VALUE:\n" - " return this.getStringValue();\n" - " case kindCase.BOOL_VALUE:\n" - " return this.getBoolValue();\n" - " case kindCase.STRUCT_VALUE:\n" - " return this.getStructValue().toJavaScript();\n" - " case kindCase.LIST_VALUE:\n" - " return this.getListValue().toJavaScript();\n" - " default:\n" - " throw new Error('Unexpected struct type');\n" - " }\n" - "};\n" - "\n" - "\n" - "/**\n" - " * Converts this JavaScript value to a new Value proto.\n" - " * @param {!proto.google.protobuf.JavaScriptValue} value The value to\n" - " * convert.\n" - " * @return {!proto.google.protobuf.Value} The newly constructed value.\n" - " */\n" - "proto.google.protobuf.Value.fromJavaScript = function(value) {\n" - " var ret = new proto.google.protobuf.Value();\n" - " switch (goog.typeOf(value)) {\n" - " case 'string':\n" - " ret.setStringValue(/** @type {string} */ (value));\n" - " break;\n" - " case 'number':\n" - " ret.setNumberValue(/** @type {number} */ (value));\n" - " break;\n" - " case 'boolean':\n" - " ret.setBoolValue(/** @type {boolean} */ (value));\n" - " break;\n" - " case 'null':\n" - " ret.setNullValue(proto.google.protobuf.NullValue.NULL_VALUE);\n" - " break;\n" - " case 'array':\n" - " ret.setListValue(proto.google.protobuf.ListValue.fromJavaScript(\n" - " /** @type{!Array} */ (value)));\n" - " break;\n" - " case 'object':\n" - " ret.setStructValue(proto.google.protobuf.Struct.fromJavaScript(\n" - " /** @type{!Object} */ (value)));\n" - " break;\n" - " default:\n" - " throw new Error('Unexpected struct type.');\n" - " }\n" - "\n" - " return ret;\n" - "};\n" - "\n" - "\n" - "/**\n" - " * Converts this ListValue object to a plain JavaScript array.\n" - " * @return {!Array} a plain JavaScript array representing this List.\n" - " */\n" - "proto.google.protobuf.ListValue.prototype.toJavaScript = function() {\n" - " var ret = [];\n" - " var values = this.getValuesList();\n" - "\n" - " for (var i = 0; i < values.length; i++) {\n" - " ret[i] = values[i].toJavaScript();\n" - " }\n" - "\n" - " return ret;\n" - "};\n" - "\n" - "\n" - "/**\n" - " * Constructs a ListValue protobuf from this plain JavaScript array.\n" - " * @param {!Array} array a plain JavaScript array\n" - " * @return {proto.google.protobuf.ListValue} a new ListValue object\n" - " */\n" - "proto.google.protobuf.ListValue.fromJavaScript = function(array) {\n" - " var ret = new proto.google.protobuf.ListValue();\n" - "\n" - " for (var i = 0; i < array.length; i++) {\n" - " ret.addValues(proto.google.protobuf.Value.fromJavaScript(array[i]));\n" - " }\n" - "\n" - " return ret;\n" - "};\n" - "\n" - "\n" - "/**\n" - " * Converts this Struct object to a plain JavaScript object.\n" - " * @return {!Object<string, !proto.google.protobuf.JavaScriptValue>} a plain\n" - " * JavaScript object representing this Struct.\n" - " */\n" - "proto.google.protobuf.Struct.prototype.toJavaScript = function() {\n" - " var ret = {};\n" - "\n" - " this.getFieldsMap().forEach(function(value, key) {\n" - " ret[key] = value.toJavaScript();\n" - " });\n" - "\n" - " return ret;\n" - "};\n" - "\n" - "\n" - "/**\n" - " * Constructs a Struct protobuf from this plain JavaScript object.\n" - " * @param {!Object} obj a plain JavaScript object\n" - " * @return {proto.google.protobuf.Struct} a new Struct object\n" - " */\n" - "proto.google.protobuf.Struct.fromJavaScript = function(obj) {\n" - " var ret = new proto.google.protobuf.Struct();\n" - " var map = ret.getFieldsMap();\n" - "\n" - " for (var property in obj) {\n" - " var val = obj[property];\n" - " map.set(property, proto.google.protobuf.Value.fromJavaScript(val));\n" - " }\n" - "\n" - " return ret;\n" - "};\n" -}, -{"timestamp.js", - "// Protocol Buffers - Google's data interchange format\n" - "// Copyright 2008 Google Inc. All rights reserved.\n" - "// https://developers.google.com/protocol-buffers/\n" - "//\n" - "// Redistribution and use in source and binary forms, with or without\n" - "// modification, are permitted provided that the following conditions are\n" - "// met:\n" - "//\n" - "// * Redistributions of source code must retain the above copyright\n" - "// notice, this list of conditions and the following disclaimer.\n" - "// * Redistributions in binary form must reproduce the above\n" - "// copyright notice, this list of conditions and the following disclaimer\n" - "// in the documentation and/or other materials provided with the\n" - "// distribution.\n" - "// * Neither the name of Google Inc. nor the names of its\n" - "// contributors may be used to endorse or promote products derived from\n" - "// this software without specific prior written permission.\n" - "//\n" - "// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS\n" - "// \"AS IS\" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT\n" - "// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR\n" - "// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT\n" - "// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,\n" - "// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT\n" - "// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,\n" - "// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY\n" - "// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n" - "// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n" - "// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n" - "\n" - "/* This code will be inserted into generated code for\n" - " * google/protobuf/timestamp.proto. */\n" - "\n" - "/**\n" - " * Returns a JavaScript 'Date' object corresponding to this Timestamp.\n" - " * @return {!Date}\n" - " */\n" - "proto.google.protobuf.Timestamp.prototype.toDate = function() {\n" - " var seconds = this.getSeconds();\n" - " var nanos = this.getNanos();\n" - "\n" - " return new Date((seconds * 1000) + (nanos / 1000000));\n" - "};\n" - "\n" - "\n" - "/**\n" - " * Sets the value of this Timestamp object to be the given Date.\n" - " * @param {!Date} value The value to set.\n" - " */\n" - "proto.google.protobuf.Timestamp.prototype.fromDate = function(value) {\n" - " var millis = value.getTime();\n" - " this.setSeconds(Math.floor(value.getTime() / 1000));\n" - " this.setNanos(value.getMilliseconds() * 1000000);\n" - "};\n" -}, - {NULL, NULL} // Terminate the list. -}; diff --git a/tools/distrib/python/grpcio_tools/grpc_version.py b/tools/distrib/python/grpcio_tools/grpc_version.py index ccb69a8ebc..581dab3b4e 100644 --- a/tools/distrib/python/grpcio_tools/grpc_version.py +++ b/tools/distrib/python/grpcio_tools/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/tools/distrib/python/grpcio_tools/grpc_version.py.template`!!! -VERSION = '1.15.0.dev0' +VERSION = '1.16.0.dev0' diff --git a/tools/distrib/python/grpcio_tools/setup.py b/tools/distrib/python/grpcio_tools/setup.py index c6bcee497f..c13dfe9ade 100644 --- a/tools/distrib/python/grpcio_tools/setup.py +++ b/tools/distrib/python/grpcio_tools/setup.py @@ -160,29 +160,7 @@ def extension_modules(): plugin_sources += [ os.path.join('grpc_tools', 'main.cc'), os.path.join('grpc_root', 'src', 'compiler', 'python_generator.cc') - ] - - #HACK: Substitute the embed.cc, which is a JS to C++ - # preprocessor with the generated code. - # The generated code should not be material - # to the parts of protoc we use (it affects - # the JavaScript code generator, supposedly), - # but we need to be cautious about it. - cc_files_clone = list(CC_FILES) - embed_cc_file = os.path.normpath('google/protobuf/compiler/js/embed.cc') - well_known_types_file = os.path.normpath( - 'google/protobuf/compiler/js/well_known_types_embed.cc') - if embed_cc_file in cc_files_clone: - cc_files_clone.remove(embed_cc_file) - if well_known_types_file in cc_files_clone: - cc_files_clone.remove(well_known_types_file) - plugin_sources += [ - os.path.join('grpc_tools', - 'protobuf_generated_well_known_types_embed.cc') - ] - plugin_sources += [ - os.path.join(CC_INCLUDE, cc_file) for cc_file in cc_files_clone - ] + ] + [os.path.join(CC_INCLUDE, cc_file) for cc_file in CC_FILES] plugin_ext = extension.Extension( name='grpc_tools._protoc_compiler', diff --git a/tools/dockerfile/grpc_clang_format/clang_format_all_the_things.sh b/tools/dockerfile/grpc_clang_format/clang_format_all_the_things.sh index 3b901ae4bf..0c8ecc21a0 100755 --- a/tools/dockerfile/grpc_clang_format/clang_format_all_the_things.sh +++ b/tools/dockerfile/grpc_clang_format/clang_format_all_the_things.sh @@ -29,7 +29,7 @@ for dir in $DIRS do for glob in $GLOB do - files="$files `find ${CLANG_FORMAT_ROOT}/$dir -name $glob -and -not -name '*.generated.*' -and -not -name '*.pb.h' -and -not -name '*.pb.c' -and -not -name '*.pb.cc' -and -not -name '*.pbobjc.h' -and -not -name '*.pbobjc.m' -and -not -name '*.pbrpc.h' -and -not -name '*.pbrpc.m' -and -not -name end2end_tests.cc -and -not -name end2end_nosec_tests.cc -and -not -name public_headers_must_be_c89.c`" + files="$files `find ${CLANG_FORMAT_ROOT}/$dir -name $glob -and -not -name '*.generated.*' -and -not -name '*.pb.h' -and -not -name '*.pb.c' -and -not -name '*.pb.cc' -and -not -name '*.pbobjc.h' -and -not -name '*.pbobjc.m' -and -not -name '*.pbrpc.h' -and -not -name '*.pbrpc.m' -and -not -name end2end_tests.cc -and -not -name end2end_nosec_tests.cc -and -not -name public_headers_must_be_c89.c -and -not -name grpc_shadow_boringssl.h`" done done diff --git a/tools/doxygen/Doxyfile.c++ b/tools/doxygen/Doxyfile.c++ index 688c271fea..9a97ee84f2 100644 --- a/tools/doxygen/Doxyfile.c++ +++ b/tools/doxygen/Doxyfile.c++ @@ -40,7 +40,7 @@ PROJECT_NAME = "GRPC C++" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 1.15.0-dev +PROJECT_NUMBER = 1.16.0-dev # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal index 592b0b20e6..0cd4cfd647 100644 --- a/tools/doxygen/Doxyfile.c++.internal +++ b/tools/doxygen/Doxyfile.c++.internal @@ -40,7 +40,7 @@ PROJECT_NAME = "GRPC C++" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 1.15.0-dev +PROJECT_NUMBER = 1.16.0-dev # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a @@ -1058,6 +1058,7 @@ src/core/lib/gprpp/fork.h \ src/core/lib/gprpp/inlined_vector.h \ src/core/lib/gprpp/manual_constructor.h \ src/core/lib/gprpp/memory.h \ +src/core/lib/gprpp/mutex_lock.h \ src/core/lib/gprpp/orphanable.h \ src/core/lib/gprpp/ref_counted.h \ src/core/lib/gprpp/ref_counted_ptr.h \ @@ -1066,6 +1067,7 @@ src/core/lib/http/format_request.h \ src/core/lib/http/httpcli.h \ src/core/lib/http/parser.h \ src/core/lib/iomgr/block_annotate.h \ +src/core/lib/iomgr/buffer_list.h \ src/core/lib/iomgr/call_combiner.h \ src/core/lib/iomgr/closure.h \ src/core/lib/iomgr/combiner.h \ @@ -1081,6 +1083,7 @@ src/core/lib/iomgr/ev_posix.h \ src/core/lib/iomgr/exec_ctx.h \ src/core/lib/iomgr/executor.h \ src/core/lib/iomgr/gethostname.h \ +src/core/lib/iomgr/internal_errqueue.h \ src/core/lib/iomgr/iocp_windows.h \ src/core/lib/iomgr/iomgr.h \ src/core/lib/iomgr/iomgr_custom.h \ diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index fa2ad93a45..7cd1dc7bf3 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -868,6 +868,7 @@ include/grpc/support/time.h \ include/grpc/support/workaround_list.h \ src/core/README.md \ src/core/ext/README.md \ +src/core/ext/filters/census/grpc_context.cc \ src/core/ext/filters/client_channel/README.md \ src/core/ext/filters/client_channel/backup_poller.cc \ src/core/ext/filters/client_channel/backup_poller.h \ @@ -1142,6 +1143,7 @@ src/core/lib/gprpp/fork.h \ src/core/lib/gprpp/inlined_vector.h \ src/core/lib/gprpp/manual_constructor.h \ src/core/lib/gprpp/memory.h \ +src/core/lib/gprpp/mutex_lock.h \ src/core/lib/gprpp/orphanable.h \ src/core/lib/gprpp/ref_counted.h \ src/core/lib/gprpp/ref_counted_ptr.h \ @@ -1157,6 +1159,8 @@ src/core/lib/http/parser.cc \ src/core/lib/http/parser.h \ src/core/lib/iomgr/README.md \ src/core/lib/iomgr/block_annotate.h \ +src/core/lib/iomgr/buffer_list.cc \ +src/core/lib/iomgr/buffer_list.h \ src/core/lib/iomgr/call_combiner.cc \ src/core/lib/iomgr/call_combiner.h \ src/core/lib/iomgr/closure.h \ @@ -1192,6 +1196,8 @@ src/core/lib/iomgr/gethostname.h \ src/core/lib/iomgr/gethostname_fallback.cc \ src/core/lib/iomgr/gethostname_host_name_max.cc \ src/core/lib/iomgr/gethostname_sysconf.cc \ +src/core/lib/iomgr/internal_errqueue.cc \ +src/core/lib/iomgr/internal_errqueue.h \ src/core/lib/iomgr/iocp_windows.cc \ src/core/lib/iomgr/iocp_windows.h \ src/core/lib/iomgr/iomgr.cc \ @@ -1506,6 +1512,7 @@ src/core/tsi/alts_transport_security.cc \ src/core/tsi/alts_transport_security.h \ src/core/tsi/fake_transport_security.cc \ src/core/tsi/fake_transport_security.h \ +src/core/tsi/grpc_shadow_boringssl.h \ src/core/tsi/local_transport_security.cc \ src/core/tsi/local_transport_security.h \ src/core/tsi/ssl/session_cache/ssl_session.h \ @@ -1521,7 +1528,6 @@ src/core/tsi/transport_security.h \ src/core/tsi/transport_security_grpc.cc \ src/core/tsi/transport_security_grpc.h \ src/core/tsi/transport_security_interface.h \ -src/cpp/ext/filters/census/grpc_context.cc \ third_party/nanopb/pb.h \ third_party/nanopb/pb_common.c \ third_party/nanopb/pb_common.h \ diff --git a/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh b/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh index 8f9658f78a..3d4d90317c 100755 --- a/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh +++ b/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh @@ -22,8 +22,8 @@ mkdir -p ${KOKORO_KEYSTORE_DIR} cp ${KOKORO_GFILE_DIR}/GrpcTesting-d0eeee2db331.json ${KOKORO_KEYSTORE_DIR}/4321_grpc-testing-service temp_dir=$(mktemp -d) -ln -f "${KOKORO_GFILE_DIR}/bazel-rc-0.14.0rc5" ${temp_dir}/bazel -chmod 755 "${KOKORO_GFILE_DIR}/bazel-rc-0.14.0rc5" +ln -f "${KOKORO_GFILE_DIR}/bazel-latest-release" ${temp_dir}/bazel +chmod 755 "${KOKORO_GFILE_DIR}/bazel-latest-release" export PATH="${temp_dir}:${PATH}" # This should show ${temp_dir}/bazel which bazel @@ -49,13 +49,15 @@ source tools/internal_ci/helper_scripts/prepare_build_linux_rc --strategy=Closure=remote \ --genrule_strategy=remote \ --experimental_strict_action_env=true \ - --crosstool_top=@com_github_bazelbuild_bazeltoolchains//configs/ubuntu16_04_clang/1.0/bazel_0.13.0/default:toolchain \ + --crosstool_top=@com_github_bazelbuild_bazeltoolchains//configs/ubuntu16_04_clang/1.0/bazel_0.16.1/default:toolchain \ --define GRPC_PORT_ISOLATED_RUNTIME=1 \ --action_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1 \ - --extra_toolchains=@com_github_bazelbuild_bazeltoolchains//configs/ubuntu16_04_clang/1.0/bazel_0.13.0/cpp:cc-toolchain-clang-x86_64-default \ + --extra_toolchains=@com_github_bazelbuild_bazeltoolchains//configs/ubuntu16_04_clang/1.0/bazel_0.16.1/cpp:cc-toolchain-clang-x86_64-default \ --extra_execution_platforms=//third_party/toolchains:rbe_ubuntu1604 \ --host_platform=//third_party/toolchains:rbe_ubuntu1604 \ --platforms=//third_party/toolchains:rbe_ubuntu1604 \ + --remote_instance_name=grpc-testing/instances/default_instance \ + --test_env=GRPC_VERBOSITY=debug \ $1 \ -- //test/... || FAILED="true" diff --git a/tools/internal_ci/linux/grpc_full_performance_master.sh b/tools/internal_ci/linux/grpc_full_performance_master.sh index 4eddc18731..24ee71edd1 100755 --- a/tools/internal_ci/linux/grpc_full_performance_master.sh +++ b/tools/internal_ci/linux/grpc_full_performance_master.sh @@ -21,7 +21,7 @@ source tools/internal_ci/helper_scripts/prepare_build_linux_perf_multilang_rc # run 8core client vs 8core server tools/run_tests/run_performance_tests.py \ - -l c++ csharp ruby java python go php7 php7_protobuf_c \ + -l c++ csharp ruby java python go php7 php7_protobuf_c node node_purejs \ --netperf \ --category scalable \ --remote_worker_host grpc-kokoro-performance-server-8core grpc-kokoro-performance-client-8core grpc-kokoro-performance-client2-8core \ diff --git a/tools/internal_ci/linux/grpc_msan_on_foundry.sh b/tools/internal_ci/linux/grpc_msan_on_foundry.sh index aa1e613d78..19cfc1739a 100644 --- a/tools/internal_ci/linux/grpc_msan_on_foundry.sh +++ b/tools/internal_ci/linux/grpc_msan_on_foundry.sh @@ -23,8 +23,8 @@ mkdir -p ${KOKORO_KEYSTORE_DIR} cp ${KOKORO_GFILE_DIR}/GrpcTesting-d0eeee2db331.json ${KOKORO_KEYSTORE_DIR}/4321_grpc-testing-service temp_dir=$(mktemp -d) -ln -f "${KOKORO_GFILE_DIR}/bazel-rc-0.14.0rc5" ${temp_dir}/bazel -chmod 755 "${KOKORO_GFILE_DIR}/bazel-rc-0.14.0rc5" +ln -f "${KOKORO_GFILE_DIR}/bazel-latest-release" ${temp_dir}/bazel +chmod 755 "${KOKORO_GFILE_DIR}/bazel-latest-release" export PATH="${temp_dir}:${PATH}" # This should show ${temp_dir}/bazel which bazel @@ -58,13 +58,15 @@ source tools/internal_ci/helper_scripts/prepare_build_linux_rc --linkopt=-fsanitize=memory \ --copt=-fsanitize-memory-track-origins \ --action_env=LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH \ - --host_crosstool_top=@com_github_bazelbuild_bazeltoolchains//configs/ubuntu16_04_clang/1.0/bazel_0.13.0/default:toolchain \ - --crosstool_top=@com_github_bazelbuild_bazeltoolchains//configs/ubuntu16_04_clang/1.0/bazel_0.13.0/msan:toolchain \ + --host_crosstool_top=@com_github_bazelbuild_bazeltoolchains//configs/ubuntu16_04_clang/1.0/bazel_0.16.1/default:toolchain \ + --crosstool_top=@com_github_bazelbuild_bazeltoolchains//configs/ubuntu16_04_clang/1.0/bazel_0.16.1/msan:toolchain \ --action_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1 \ - --extra_toolchains=@com_github_bazelbuild_bazeltoolchains//configs/ubuntu16_04_clang/1.0/bazel_0.13.0/cpp:cc-toolchain-clang-x86_64-default \ + --extra_toolchains=@com_github_bazelbuild_bazeltoolchains//configs/ubuntu16_04_clang/1.0/bazel_0.16.1/cpp:cc-toolchain-clang-x86_64-default \ --extra_execution_platforms=//third_party/toolchains:rbe_ubuntu1604 \ --host_platform=//third_party/toolchains:rbe_ubuntu1604 \ --platforms=//third_party/toolchains:rbe_ubuntu1604 \ + --remote_instance_name=grpc-testing/instances/default_instance \ + --test_env=GRPC_VERBOSITY=debug \ -- //test/... || FAILED="true" # Sleep to let ResultStore finish writing results before querying diff --git a/tools/internal_ci/linux/grpc_ubsan_on_foundry.sh b/tools/internal_ci/linux/grpc_ubsan_on_foundry.sh index 338b1b6a0d..edb2069fb2 100644 --- a/tools/internal_ci/linux/grpc_ubsan_on_foundry.sh +++ b/tools/internal_ci/linux/grpc_ubsan_on_foundry.sh @@ -23,8 +23,8 @@ mkdir -p ${KOKORO_KEYSTORE_DIR} cp ${KOKORO_GFILE_DIR}/GrpcTesting-d0eeee2db331.json ${KOKORO_KEYSTORE_DIR}/4321_grpc-testing-service temp_dir=$(mktemp -d) -ln -f "${KOKORO_GFILE_DIR}/bazel-rc-0.14.0rc5" ${temp_dir}/bazel -chmod 755 "${KOKORO_GFILE_DIR}/bazel-rc-0.14.0rc5" +ln -f "${KOKORO_GFILE_DIR}/bazel-latest-release" ${temp_dir}/bazel +chmod 755 "${KOKORO_GFILE_DIR}/bazel-latest-release" export PATH="${temp_dir}:${PATH}" # This should show ${temp_dir}/bazel which bazel @@ -55,13 +55,15 @@ source tools/internal_ci/helper_scripts/prepare_build_linux_rc --strip=never \ --copt=-fsanitize=undefined \ --linkopt=-fsanitize=undefined \ - --crosstool_top=@com_github_bazelbuild_bazeltoolchains//configs/experimental/ubuntu16_04_clang/1.0/bazel_0.13.0/ubsan:toolchain \ + --crosstool_top=@com_github_bazelbuild_bazeltoolchains//configs/experimental/ubuntu16_04_clang/1.0/bazel_0.15.0/ubsan:toolchain \ --action_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1 \ - --extra_toolchains=@com_github_bazelbuild_bazeltoolchains//configs/ubuntu16_04_clang/1.0/bazel_0.13.0/cpp:cc-toolchain-clang-x86_64-default \ + --extra_toolchains=@com_github_bazelbuild_bazeltoolchains//configs/ubuntu16_04_clang/1.0/bazel_0.16.1/cpp:cc-toolchain-clang-x86_64-default \ --extra_execution_platforms=//third_party/toolchains:rbe_ubuntu1604 \ --host_platform=//third_party/toolchains:rbe_ubuntu1604 \ --platforms=//third_party/toolchains:rbe_ubuntu1604 \ --cache_test_results=no \ + --remote_instance_name=grpc-testing/instances/default_instance \ + --test_env=GRPC_VERBOSITY=debug \ -- //test/... || FAILED="true" # Sleep to let ResultStore finish writing results before querying diff --git a/tools/internal_ci/linux/pull_request/grpc_ubsan_on_foundry.sh b/tools/internal_ci/linux/pull_request/grpc_ubsan_on_foundry.sh index c7ab5b104a..f8c9314236 100644 --- a/tools/internal_ci/linux/pull_request/grpc_ubsan_on_foundry.sh +++ b/tools/internal_ci/linux/pull_request/grpc_ubsan_on_foundry.sh @@ -23,8 +23,8 @@ mkdir -p ${KOKORO_KEYSTORE_DIR} cp ${KOKORO_GFILE_DIR}/GrpcTesting-d0eeee2db331.json ${KOKORO_KEYSTORE_DIR}/4321_grpc-testing-service temp_dir=$(mktemp -d) -ln -f "${KOKORO_GFILE_DIR}/bazel-rc-0.14.0rc5" ${temp_dir}/bazel -chmod 755 "${KOKORO_GFILE_DIR}/bazel-rc-0.14.0rc5" +ln -f "${KOKORO_GFILE_DIR}/bazel-latest-release" ${temp_dir}/bazel +chmod 755 "${KOKORO_GFILE_DIR}/bazel-latest-release" export PATH="${temp_dir}:${PATH}" # This should show ${temp_dir}/bazel which bazel @@ -55,12 +55,14 @@ source tools/internal_ci/helper_scripts/prepare_build_linux_rc --strip=never \ --copt=-fsanitize=undefined \ --linkopt=-fsanitize=undefined \ - --crosstool_top=@com_github_bazelbuild_bazeltoolchains//configs/experimental/ubuntu16_04_clang/1.0/bazel_0.13.0/ubsan:toolchain \ + --crosstool_top=@com_github_bazelbuild_bazeltoolchains//configs/experimental/ubuntu16_04_clang/1.0/bazel_0.15.0/ubsan:toolchain \ --action_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1 \ - --extra_toolchains=@com_github_bazelbuild_bazeltoolchains//configs/ubuntu16_04_clang/1.0/bazel_0.13.0/cpp:cc-toolchain-clang-x86_64-default \ + --extra_toolchains=@com_github_bazelbuild_bazeltoolchains//configs/ubuntu16_04_clang/1.0/bazel_0.16.1/cpp:cc-toolchain-clang-x86_64-default \ --extra_execution_platforms=//third_party/toolchains:rbe_ubuntu1604 \ --host_platform=//third_party/toolchains:rbe_ubuntu1604 \ --platforms=//third_party/toolchains:rbe_ubuntu1604 \ + --remote_instance_name=grpc-testing/instances/default_instance \ + --test_env=GRPC_VERBOSITY=debug \ -- //test/... || FAILED="true" if [ "$FAILED" != "" ] diff --git a/tools/package_hosting/build-201807.xsl b/tools/package_hosting/build-201807.xsl index 69a190446f..f7571d178e 100644 --- a/tools/package_hosting/build-201807.xsl +++ b/tools/package_hosting/build-201807.xsl @@ -1,75 +1,74 @@ -<?xml version="1.0"?> +<?xml version="1.0"?> <xsl:stylesheet version="1.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform"> <xsl:template match="//build"> - <html> - <head> - <title>Artifacts for gRPC Build <xsl:value-of select="@id"/> </title> - <link rel="stylesheet" type="text/css" href="/web-assets/style.css" /> - <link rel="apple-touch-icon" href="/web-assets/favicons/apple-touch-icon.png" sizes="180x180" /> - <link rel="icon" type="image/png" href="/web-assets/favicons/android-chrome-192x192.png" sizes="192x192" /> - <link rel="icon" type="image/png" href="/web-assets/favicons/favicon-32x32.png" sizes="32x32" /> - <link rel="icon" type="image/png" href="/web-assets/favicons/favicon-16x16.png" sizes="16x16" /> - <link rel="manifest" href="/web-assets/favicons/manifest.json" /> - <link rel="mask-icon" href="/web-assets/favicons/safari-pinned-tab.svg" color="#2DA6B0" /> - <meta name="msapplication-TileColor" content="#ffffff" /> - <meta name="msapplication-TileImage" content="/web-assets/favicons/mstile-150x150.png" /> +<html> + <head> + <title>Artifacts for gRPC Build <xsl:value-of select="@id"/></title> + <link rel="stylesheet" type="text/css" href="/web-assets/style.css" /> + <link rel="apple-touch-icon" href="/web-assets/favicons/apple-touch-icon.png" sizes="180x180" /> + <link rel="icon" type="image/png" href="/web-assets/favicons/android-chrome-192x192.png" sizes="192x192" /> + <link rel="icon" type="image/png" href="/web-assets/favicons/favicon-32x32.png" sizes="32x32" /> + <link rel="icon" type="image/png" href="/web-assets/favicons/favicon-16x16.png" sizes="16x16" /> + <link rel="manifest" href="/web-assets/favicons/manifest.json" /> + <link rel="mask-icon" href="/web-assets/favicons/safari-pinned-tab.svg" color="#2DA6B0" /> + <meta name="msapplication-TileColor" content="#ffffff" /> + <meta name="msapplication-TileImage" content="/web-assets/favicons/mstile-150x150.png" /> + <meta name="og:title" content="gRPC Package Build"/> + <meta name="og:image" content="https://grpc.io/img/grpc_square_reverse_4x.png"/> + <meta name="og:description" content="gRPC Package Build"/> + </head> + <body bgcolor="#ffffff"> + <div id="topbar"> + <span class="title">Artifacts for gRPC Build <xsl:value-of select="@id"/></span> + </div> + <div id="main"> + <div id="metadata"> + <span class="fieldname">Build: </span> <a href='#'><xsl:value-of select="@id"/></a> + [<a href="https://source.cloud.google.com/results/invocations/{@id}">invocation</a>]<br /> + <span class="fieldname">Timestamp: </span> + <xsl:value-of select="@timestamp"/> <br /> + <span class="fieldname">Branch: </span> + <a href="https://github.com/grpc/grpc/tree/{./metadata/branch[text()]}"> + <xsl:value-of select="./metadata/branch[text()]" /> + </a><br /> + <span class="fieldname">Commit: </span> + <a href="https://github.com/grpc/grpc/tree/{./metadata/commit[text()]}"> + <xsl:value-of select="./metadata/commit[text()]" /><br /></a> + </div> + <xsl:apply-templates select="artifacts" /> + <br /> + <br /> - <meta name="og:title" content="gRPC Package Build"/> - <meta name="og:image" content="https://grpc.io/img/grpc_square_reverse_4x.png"/> - <meta name="og:description" content="gRPC Package Build"/> - </head> - <body bgcolor="#ffffff"> - <div id="topbar"> - <span class="title">Artifacts for gRPC Build <xsl:value-of select="@id"/></span> - </div> - <div id="main"> - <div id="metadata"> - <span class="fieldname">Build: </span> <a href='#'><xsl:value-of select="@id"/></a> - [<a href="https://source.cloud.google.com/results/invocations/{@id}">invocation</a>]<br /> - <span class="fieldname">Timestamp: </span> - <xsl:value-of select="@timestamp"/> <br /> - <span class="fieldname">Branch: </span> - <a href="https://github.com/grpc/grpc/tree/{./metadata/branch[text()]}"> - <xsl:value-of select="./metadata/branch[text()]" /> - </a><br /> - <span class="fieldname">Commit: </span> - <a href="https://github.com/grpc/grpc/tree/{./metadata/commit[text()]}"> - <xsl:value-of select="./metadata/commit[text()]" /><br /></a> - </div> - <xsl:apply-templates select="artifacts" /> - <br /> - <br /> - - <p class="description"><a href="https://grpc.io">gRPC</a> is a <a href="https://www.cncf.io" class="external">Cloud Native Computing Foundation</a> project. <a href="https://policies.google.com/privacy" class="external">Privacy Policy</a>.</p> - <p class="description"> - Copyright © <xsl:value-of select="substring(@timestamp, 1, 4)" /> <a href="https://github.com/grpc/grpc/blob/{./metadata/commit[text()]}/AUTHORS">The gRPC Authors</a></p> - <br /> - <br /> - </div> - </body> - </html> + <p class="description"><a href="https://grpc.io">gRPC</a> is a <a href="https://www.cncf.io" class="external">Cloud Native Computing Foundation</a> project. <a href="https://policies.google.com/privacy" class="external">Privacy Policy</a>.</p> + <p class="description"> + Copyright © <xsl:value-of select="substring(@timestamp, 1, 4)" /> <a href="https://github.com/grpc/grpc/blob/{./metadata/commit[text()]}/AUTHORS">The gRPC Authors</a></p> + <br /> + <br /> + </div> + </body> +</html> </xsl:template> <xsl:template match="artifacts"> <h2> gRPC <code>protoc</code> Plugins </h2> <table> -<xsl:apply-templates select="artifact[@type='protoc']"> - <xsl:sort select="artifact/@name" /> + <xsl:apply-templates select="artifact[@type='protoc']"> + <xsl:sort select="@name" /> </xsl:apply-templates> </table> <h2> C# </h2> <table> -<xsl:apply-templates select="artifact[@type='csharp']"> - <xsl:sort select="artifact/@name" /> + <xsl:apply-templates select="artifact[@type='csharp']"> + <xsl:sort select="@name" /> </xsl:apply-templates> </table> <h2> PHP </h2> <table> -<xsl:apply-templates select="artifact[@type='php']"> - <xsl:sort select="artifact/@name" /> + <xsl:apply-templates select="artifact[@type='php']"> + <xsl:sort select="@name" /> </xsl:apply-templates> </table> @@ -87,24 +86,23 @@ document.write("<p><code>" + </script> <table> <xsl:apply-templates select="artifact[@type='python']"> - <xsl:sort select="artifact/@name" /> + <xsl:sort select="@name" /> </xsl:apply-templates> </table> <h2> Ruby </h2> <table> -<xsl:apply-templates select="artifact[@type='ruby']"> - <xsl:sort select="artifact/@name" /> + <xsl:apply-templates select="artifact[@type='ruby']"> + <xsl:sort select="@name" /> </xsl:apply-templates> </table> - </xsl:template> <xsl:template match="artifact"> <tr> -<td class="name"> <a href="{@path}"><xsl:value-of select="@name" /></a> </td> -<td class="hash"> <xsl:value-of select="@sha256"/> </td> + <td class="name"><a href="{@path}"><xsl:value-of select="@name" /></a></td> + <td class="hash"><xsl:value-of select="@sha256"/></td> </tr> </xsl:template> diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json index a9327bded1..8ea5126fde 100644 --- a/tools/run_tests/generated/sources_and_headers.json +++ b/tools/run_tests/generated/sources_and_headers.json @@ -173,6 +173,23 @@ "headers": [], "is_filegroup": false, "language": "c", + "name": "buffer_list_test", + "src": [ + "test/core/iomgr/buffer_list_test.cc" + ], + "third_party": false, + "type": "target" + }, + { + "deps": [ + "gpr", + "gpr_test_util", + "grpc", + "grpc_test_util" + ], + "headers": [], + "is_filegroup": false, + "language": "c", "name": "channel_create_test", "src": [ "test/core/surface/channel_create_test.cc" @@ -1417,6 +1434,26 @@ "grpc", "grpc_test_util" ], + "headers": [ + "test/core/end2end/end2end_tests.h" + ], + "is_filegroup": false, + "language": "c", + "name": "inproc_callback_test", + "src": [ + "test/core/end2end/end2end_tests.h", + "test/core/end2end/inproc_callback_test.cc" + ], + "third_party": false, + "type": "target" + }, + { + "deps": [ + "gpr", + "gpr_test_util", + "grpc", + "grpc_test_util" + ], "headers": [], "is_filegroup": false, "language": "c", @@ -9068,6 +9105,7 @@ "alts_util", "gpr", "grpc_base", + "grpc_shadow_boringssl", "grpc_transport_chttp2_client_insecure", "tsi", "tsi_interface" @@ -9187,7 +9225,7 @@ "name": "census", "src": [ "include/grpc/census.h", - "src/cpp/ext/filters/census/grpc_context.cc" + "src/core/ext/filters/census/grpc_context.cc" ], "third_party": false, "type": "filegroup" @@ -9306,6 +9344,7 @@ "src/core/lib/gprpp/fork.h", "src/core/lib/gprpp/manual_constructor.h", "src/core/lib/gprpp/memory.h", + "src/core/lib/gprpp/mutex_lock.h", "src/core/lib/gprpp/thd.h", "src/core/lib/profiling/timers.h" ], @@ -9353,6 +9392,7 @@ "src/core/lib/gprpp/fork.h", "src/core/lib/gprpp/manual_constructor.h", "src/core/lib/gprpp/memory.h", + "src/core/lib/gprpp/mutex_lock.h", "src/core/lib/gprpp/thd.h", "src/core/lib/profiling/timers.h" ], @@ -9465,6 +9505,7 @@ "src/core/lib/http/format_request.cc", "src/core/lib/http/httpcli.cc", "src/core/lib/http/parser.cc", + "src/core/lib/iomgr/buffer_list.cc", "src/core/lib/iomgr/call_combiner.cc", "src/core/lib/iomgr/combiner.cc", "src/core/lib/iomgr/endpoint.cc", @@ -9485,6 +9526,7 @@ "src/core/lib/iomgr/gethostname_fallback.cc", "src/core/lib/iomgr/gethostname_host_name_max.cc", "src/core/lib/iomgr/gethostname_sysconf.cc", + "src/core/lib/iomgr/internal_errqueue.cc", "src/core/lib/iomgr/iocp_windows.cc", "src/core/lib/iomgr/iomgr.cc", "src/core/lib/iomgr/iomgr_custom.cc", @@ -9644,6 +9686,7 @@ "src/core/lib/http/httpcli.h", "src/core/lib/http/parser.h", "src/core/lib/iomgr/block_annotate.h", + "src/core/lib/iomgr/buffer_list.h", "src/core/lib/iomgr/call_combiner.h", "src/core/lib/iomgr/closure.h", "src/core/lib/iomgr/combiner.h", @@ -9659,6 +9702,7 @@ "src/core/lib/iomgr/exec_ctx.h", "src/core/lib/iomgr/executor.h", "src/core/lib/iomgr/gethostname.h", + "src/core/lib/iomgr/internal_errqueue.h", "src/core/lib/iomgr/iocp_windows.h", "src/core/lib/iomgr/iomgr.h", "src/core/lib/iomgr/iomgr_custom.h", @@ -9794,6 +9838,7 @@ "src/core/lib/http/httpcli.h", "src/core/lib/http/parser.h", "src/core/lib/iomgr/block_annotate.h", + "src/core/lib/iomgr/buffer_list.h", "src/core/lib/iomgr/call_combiner.h", "src/core/lib/iomgr/closure.h", "src/core/lib/iomgr/combiner.h", @@ -9809,6 +9854,7 @@ "src/core/lib/iomgr/exec_ctx.h", "src/core/lib/iomgr/executor.h", "src/core/lib/iomgr/gethostname.h", + "src/core/lib/iomgr/internal_errqueue.h", "src/core/lib/iomgr/iocp_windows.h", "src/core/lib/iomgr/iomgr.h", "src/core/lib/iomgr/iomgr_custom.h", @@ -10355,6 +10401,7 @@ "alts_tsi", "gpr", "grpc_base", + "grpc_shadow_boringssl", "grpc_transport_chttp2_alpn", "tsi" ], @@ -10471,6 +10518,20 @@ "type": "filegroup" }, { + "deps": [], + "headers": [ + "src/core/tsi/grpc_shadow_boringssl.h" + ], + "is_filegroup": true, + "language": "c", + "name": "grpc_shadow_boringssl", + "src": [ + "src/core/tsi/grpc_shadow_boringssl.h" + ], + "third_party": false, + "type": "filegroup" + }, + { "deps": [ "cmdline", "gpr", @@ -10921,6 +10982,7 @@ "deps": [ "gpr", "grpc_base", + "grpc_shadow_boringssl", "grpc_trace", "tsi_interface" ], diff --git a/tools/run_tests/generated/tests.json b/tools/run_tests/generated/tests.json index de64097ec1..fba76d69d1 100644 --- a/tools/run_tests/generated/tests.json +++ b/tools/run_tests/generated/tests.json @@ -199,6 +199,26 @@ "args": [], "benchmark": false, "ci_platforms": [ + "linux" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [ + "uv" + ], + "flaky": false, + "gtest": false, + "language": "c", + "name": "buffer_list_test", + "platforms": [ + "linux" + ], + "uses_polling": true + }, + { + "args": [], + "benchmark": false, + "ci_platforms": [ "linux", "mac", "posix", @@ -1706,6 +1726,30 @@ "posix", "windows" ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "gtest": false, + "language": "c", + "name": "inproc_callback_test", + "platforms": [ + "linux", + "mac", + "posix", + "windows" + ], + "uses_polling": false + }, + { + "args": [], + "benchmark": false, + "ci_platforms": [ + "linux", + "mac", + "posix", + "windows" + ], "cpu_cost": 0.1, "exclude_configs": [], "exclude_iomgrs": [], @@ -3483,7 +3527,7 @@ "mac", "posix" ], - "uses_polling": true + "uses_polling": false }, { "args": [], diff --git a/tools/run_tests/performance/bq_upload_result.py b/tools/run_tests/performance/bq_upload_result.py index 6702587557..b442f0cf83 100755 --- a/tools/run_tests/performance/bq_upload_result.py +++ b/tools/run_tests/performance/bq_upload_result.py @@ -128,14 +128,16 @@ def _flatten_result_inplace(scenario_result): def _populate_metadata_inplace(scenario_result): """Populates metadata based on environment variables set by Jenkins.""" - # NOTE: Grabbing the Jenkins environment variables will only work if the - # driver is running locally on the same machine where Jenkins has started + # NOTE: Grabbing the Kokoro environment variables will only work if the + # driver is running locally on the same machine where Kokoro has started # the job. For our setup, this is currently the case, so just assume that. - build_number = os.getenv('BUILD_NUMBER') - build_url = os.getenv('BUILD_URL') - job_name = os.getenv('JOB_NAME') - git_commit = os.getenv('GIT_COMMIT') + build_number = os.getenv('KOKORO_BUILD_NUMBER') + build_url = 'https://source.cloud.google.com/results/invocations/%s' % os.getenv( + 'KOKORO_BUILD_ID') + job_name = os.getenv('KOKORO_JOB_NAME') + git_commit = os.getenv('KOKORO_GIT_COMMIT') # actual commit is the actual head of PR that is getting tested + # TODO(jtattermusch): unclear how to obtain on Kokoro git_actual_commit = os.getenv('ghprbActualCommit') utc_timestamp = str(calendar.timegm(time.gmtime())) diff --git a/tools/run_tests/performance/build_performance.sh b/tools/run_tests/performance/build_performance.sh index 22e0ca9fa0..35d9e90598 100755 --- a/tools/run_tests/performance/build_performance.sh +++ b/tools/run_tests/performance/build_performance.sh @@ -55,6 +55,9 @@ do "csharp") python tools/run_tests/run_tests.py -l "$language" -c "$CONFIG" --build_only -j 8 --compiler coreclr ;; + "node"|"node_purejs") + tools/run_tests/performance/build_performance_node.sh + ;; *) python tools/run_tests/run_tests.py -l "$language" -c "$CONFIG" --build_only -j 8 ;; diff --git a/tools/run_tests/performance/build_performance_node.sh b/tools/run_tests/performance/build_performance_node.sh new file mode 100755 index 0000000000..12e0872264 --- /dev/null +++ b/tools/run_tests/performance/build_performance_node.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# Copyright 2018 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set +ex + +. "$HOME/.nvm/nvm.sh" + +nvm install 10 + +set -ex + +cd "$(dirname "$0")/../../../../grpc-node" + +npm install + +./node_modules/.bin/gulp setup diff --git a/tools/run_tests/performance/run_worker_node.sh b/tools/run_tests/performance/run_worker_node.sh new file mode 100755 index 0000000000..3e5dd18edb --- /dev/null +++ b/tools/run_tests/performance/run_worker_node.sh @@ -0,0 +1,32 @@ +#!/bin/bash +# Copyright 2018 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +. "$HOME/.nvm/nvm.sh" + +nvm use 10 + +set -ex + +fixture=$1 + +shift + +# Enter repo root +cd "$(dirname "$0")/../../.." + +# Enter the grpc-node repo root (expected to be next to grpc repo root) +cd ../grpc-node + +node -r "./test/fixtures/$fixture" test/performance/worker.js "$@" diff --git a/tools/run_tests/performance/scenario_config.py b/tools/run_tests/performance/scenario_config.py index f05753154e..2e78bd07fb 100644 --- a/tools/run_tests/performance/scenario_config.py +++ b/tools/run_tests/performance/scenario_config.py @@ -1151,6 +1151,106 @@ class GoLanguage: return 'go' +class NodeLanguage: + + def __init__(self, node_purejs=False): + pass + self.node_purejs = node_purejs + self.safename = str(self) + + def worker_cmdline(self): + fixture = 'native_js' if self.node_purejs else 'native_native' + return [ + 'tools/run_tests/performance/run_worker_node.sh', fixture, + '--benchmark_impl=grpc' + ] + + def worker_port_offset(self): + if self.node_purejs: + return 1100 + return 1000 + + def scenarios(self): + node_implementation = 'node_purejs' if self.node_purejs else 'node' + for secure in [True, False]: + secstr = 'secure' if secure else 'insecure' + smoketest_categories = ([SMOKETEST] if secure else []) + [SCALABLE] + + yield _ping_pong_scenario( + '%s_to_node_generic_async_streaming_ping_pong_%s' % + (node_implementation, secstr), + rpc_type='STREAMING', + client_type='ASYNC_CLIENT', + server_type='ASYNC_GENERIC_SERVER', + server_language='node', + use_generic_payload=True, + async_server_threads=1, + secure=secure, + categories=smoketest_categories) + + yield _ping_pong_scenario( + '%s_to_node_protobuf_async_streaming_ping_pong_%s' % + (node_implementation, secstr), + rpc_type='STREAMING', + client_type='ASYNC_CLIENT', + server_type='ASYNC_SERVER', + server_language='node', + async_server_threads=1, + secure=secure) + + yield _ping_pong_scenario( + '%s_to_node_protobuf_async_unary_ping_pong_%s' % + (node_implementation, secstr), + rpc_type='UNARY', + client_type='ASYNC_CLIENT', + server_type='ASYNC_SERVER', + server_language='node', + async_server_threads=1, + secure=secure, + categories=smoketest_categories) + + yield _ping_pong_scenario( + '%s_to_node_protobuf_async_unary_qps_unconstrained_%s' % + (node_implementation, secstr), + rpc_type='UNARY', + client_type='ASYNC_CLIENT', + server_type='ASYNC_SERVER', + server_language='node', + unconstrained_client='async', + secure=secure, + categories=smoketest_categories + [SCALABLE]) + + yield _ping_pong_scenario( + '%s_to_node_protobuf_async_streaming_qps_unconstrained_%s' % + (node_implementation, secstr), + rpc_type='STREAMING', + client_type='ASYNC_CLIENT', + server_type='ASYNC_SERVER', + server_language='node', + unconstrained_client='async', + secure=secure, + categories=[SCALABLE]) + + yield _ping_pong_scenario( + '%s_to_node_generic_async_streaming_qps_unconstrained_%s' % + (node_implementation, secstr), + rpc_type='STREAMING', + client_type='ASYNC_CLIENT', + server_type='ASYNC_GENERIC_SERVER', + server_language='node', + unconstrained_client='async', + use_generic_payload=True, + secure=secure, + categories=[SCALABLE]) + + # TODO(murgatroid99): add scenarios node vs C++ + + def __str__(self): + if self.node_purejs: + return 'node_purejs' + return 'node' + + LANGUAGES = { 'c++': CXXLanguage(), 'csharp': CSharpLanguage(), @@ -1160,4 +1260,6 @@ LANGUAGES = { 'java': JavaLanguage(), 'python': PythonLanguage(), 'go': GoLanguage(), + 'node': NodeLanguage(), + 'node_purejs': NodeLanguage(node_purejs=True) } diff --git a/tools/run_tests/python_utils/upload_test_results.py b/tools/run_tests/python_utils/upload_test_results.py index cbb4c32a2a..9d99703725 100644 --- a/tools/run_tests/python_utils/upload_test_results.py +++ b/tools/run_tests/python_utils/upload_test_results.py @@ -68,15 +68,13 @@ _INTEROP_RESULTS_SCHEMA = [ def _get_build_metadata(test_results): - """Add Jenkins/Kokoro build metadata to test_results based on environment - variables set by Jenkins/Kokoro. + """Add Kokoro build metadata to test_results based on environment + variables set by Kokoro. """ - build_id = os.getenv('BUILD_ID') or os.getenv('KOKORO_BUILD_NUMBER') - build_url = os.getenv('BUILD_URL') - if os.getenv('KOKORO_BUILD_ID'): - build_url = 'https://source.cloud.google.com/results/invocations/%s' % os.getenv( - 'KOKORO_BUILD_ID') - job_name = os.getenv('JOB_BASE_NAME') or os.getenv('KOKORO_JOB_NAME') + build_id = os.getenv('KOKORO_BUILD_NUMBER') + build_url = 'https://source.cloud.google.com/results/invocations/%s' % os.getenv( + 'KOKORO_BUILD_ID') + job_name = os.getenv('KOKORO_JOB_NAME') if build_id: test_results['build_id'] = build_id diff --git a/tools/run_tests/run_performance_tests.py b/tools/run_tests/run_performance_tests.py index 9a9f74e9e5..04e706fa5b 100755 --- a/tools/run_tests/run_performance_tests.py +++ b/tools/run_tests/run_performance_tests.py @@ -189,11 +189,19 @@ def create_netperf_jobspec(server_host='localhost', def archive_repo(languages): """Archives local version of repo including submodules.""" - cmdline = ['tar', '-cf', '../grpc.tar', '../grpc/'] + # Directory contains symlinks that can't be correctly untarred on Windows + # so we just skip them as a workaround. + # See https://github.com/grpc/grpc/issues/16334 + bad_symlinks_dir = '../grpc/third_party/libcxx/test/std/experimental/filesystem/Inputs/static_test_env' + cmdline = [ + 'tar', '--exclude', bad_symlinks_dir, '-cf', '../grpc.tar', '../grpc/' + ] if 'java' in languages: cmdline.append('../grpc-java') if 'go' in languages: cmdline.append('../grpc-go') + if 'node' in languages or 'node_purejs' in languages: + cmdline.append('../grpc-node') archive_job = jobset.JobSpec( cmdline=cmdline, shortname='archive_repo', timeout_seconds=3 * 60) @@ -247,9 +255,9 @@ def build_on_remote_hosts(hosts, languages=scenario_config.LANGUAGES.keys(), build_local=False): """Builds performance worker on remote hosts (and maybe also locally).""" - build_timeout = 15 * 60 + build_timeout = 45 * 60 # Kokoro VMs (which are local only) do not have caching, so they need more time to build - local_build_timeout = 30 * 60 + local_build_timeout = 60 * 60 build_jobs = [] for host in hosts: user_at_host = '%s@%s' % (_REMOTE_HOST_USERNAME, host) diff --git a/tools/run_tests/sanity/check_unsecure.sh b/tools/run_tests/sanity/check_bad_dependencies.sh index cca1235479..5ae0e02c81 100755 --- a/tools/run_tests/sanity/check_unsecure.sh +++ b/tools/run_tests/sanity/check_bad_dependencies.sh @@ -23,5 +23,9 @@ test "$(bazel query 'somepath("//:grpc++_unsecure", "//external:libssl")' 2>/dev test "$(bazel query 'somepath("//:grpc++_codegen_proto", "//external:libssl")' 2>/dev/null | wc -l)" -eq 0 || exit 1 test "$(bazel query 'somepath("//test/cpp/microbenchmarks:helpers", "//external:libssl")' 2>/dev/null | wc -l)" -eq 0 || exit 1 +# Make sure that core doesn't depend on anything in C++ library + +test "$(bazel query 'deps("//:grpc")' 2>/dev/null | egrep 'src/cpp|include/grpcpp' | wc -l)" -eq 0 || exit 1 + exit 0 diff --git a/tools/run_tests/sanity/sanity_tests.yaml b/tools/run_tests/sanity/sanity_tests.yaml index ac0d4c70e5..fd9b34a198 100644 --- a/tools/run_tests/sanity/sanity_tests.yaml +++ b/tools/run_tests/sanity/sanity_tests.yaml @@ -1,4 +1,5 @@ # a set of tests that are run in parallel for sanity tests +- script: tools/run_tests/sanity/check_bad_dependencies.sh - script: tools/run_tests/sanity/check_bazel_workspace.py - script: tools/run_tests/sanity/check_cache_mk.sh - script: tools/run_tests/sanity/check_owners.sh @@ -6,7 +7,6 @@ - script: tools/run_tests/sanity/check_submodules.sh - script: tools/run_tests/sanity/check_test_filtering.py - script: tools/run_tests/sanity/check_tracer_sanity.py -- script: tools/run_tests/sanity/check_unsecure.sh - script: tools/run_tests/sanity/core_banned_functions.py - script: tools/run_tests/sanity/core_untyped_structs.sh - script: tools/run_tests/sanity/check_deprecated_grpc++.py @@ -22,4 +22,5 @@ - script: tools/distrib/pylint_code.sh - script: tools/distrib/yapf_code.sh - script: tools/distrib/python/check_grpcio_tools.py +- script: tools/distrib/check_shadow_boringssl_symbol_list.sh cpu_cost: 1000 |