diff options
Diffstat (limited to 'tools')
63 files changed, 980 insertions, 174 deletions
diff --git a/tools/distrib/python/bazel_deps.sh b/tools/distrib/python/bazel_deps.sh index de3ee07970..f6d42d29eb 100755 --- a/tools/distrib/python/bazel_deps.sh +++ b/tools/distrib/python/bazel_deps.sh @@ -33,14 +33,14 @@ cd $(dirname $0)/../../../ # First check if bazel is installed on the machine. If it is, then we don't need # to invoke the docker bazel. -if [ "bazel version" ] +if [ -x "$(command -v bazel)" ] then cd third_party/protobuf bazel query 'deps('$1')' else - docker build -t bazel `realpath ./tools/dockerfile/bazel/` - docker run -v "`realpath .`:/src/grpc/" \ - -w /src/grpc/third_party/protobuf \ - bazel \ + docker build -t bazel_local_img tools/dockerfile/test/sanity + docker run -v "$(realpath .):/src/grpc/:ro" \ + -w /src/grpc/third_party/protobuf \ + bazel_local_img \ bazel query 'deps('$1')' fi diff --git a/tools/distrib/python/grpcio_tools/grpc_tools/protobuf_generated_well_known_types_embed.cc b/tools/distrib/python/grpcio_tools/grpc_tools/protobuf_generated_well_known_types_embed.cc new file mode 100644 index 0000000000..682837a27f --- /dev/null +++ b/tools/distrib/python/grpcio_tools/grpc_tools/protobuf_generated_well_known_types_embed.cc @@ -0,0 +1,343 @@ +// Copyright 2017, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// HACK: Embed the generated well_known_types_js.cc to make +// grpc-tools python package compilation easy. +#include <google/protobuf/compiler/js/well_known_types_embed.h> +struct FileToc well_known_types_js[] = { +{"any.js", + "// Protocol Buffers - Google's data interchange format\n" + "// Copyright 2008 Google Inc. All rights reserved.\n" + "// https://developers.google.com/protocol-buffers/\n" + "//\n" + "// Redistribution and use in source and binary forms, with or without\n" + "// modification, are permitted provided that the following conditions are\n" + "// met:\n" + "//\n" + "// * Redistributions of source code must retain the above copyright\n" + "// notice, this list of conditions and the following disclaimer.\n" + "// * Redistributions in binary form must reproduce the above\n" + "// copyright notice, this list of conditions and the following disclaimer\n" + "// in the documentation and/or other materials provided with the\n" + "// distribution.\n" + "// * Neither the name of Google Inc. nor the names of its\n" + "// contributors may be used to endorse or promote products derived from\n" + "// this software without specific prior written permission.\n" + "//\n" + "// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS\n" + "// \"AS IS\" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT\n" + "// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR\n" + "// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT\n" + "// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,\n" + "// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT\n" + "// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,\n" + "// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY\n" + "// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n" + "// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n" + "// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n" + "\n" + "/* This code will be inserted into generated code for\n" + " * google/protobuf/any.proto. */\n" + "\n" + "/**\n" + " * Returns the type name contained in this instance, if any.\n" + " * @return {string|undefined}\n" + " */\n" + "proto.google.protobuf.Any.prototype.getTypeName = function() {\n" + " return this.getTypeUrl().split('/').pop();\n" + "};\n" + "\n" + "\n" + "/**\n" + " * Packs the given message instance into this Any.\n" + " * @param {!Uint8Array} serialized The serialized data to pack.\n" + " * @param {string} name The type name of this message object.\n" + " * @param {string=} opt_typeUrlPrefix the type URL prefix.\n" + " */\n" + "proto.google.protobuf.Any.prototype.pack = function(serialized, name,\n" + " opt_typeUrlPrefix) {\n" + " if (!opt_typeUrlPrefix) {\n" + " opt_typeUrlPrefix = 'type.googleapis.com/';\n" + " }\n" + "\n" + " if (opt_typeUrlPrefix.substr(-1) != '/') {\n" + " this.setTypeUrl(opt_typeUrlPrefix + '/' + name);\n" + " } else {\n" + " this.setTypeUrl(opt_typeUrlPrefix + name);\n" + " }\n" + "\n" + " this.setValue(serialized);\n" + "};\n" + "\n" + "\n" + "/**\n" + " * @template T\n" + " * Unpacks this Any into the given message object.\n" + " * @param {function(Uint8Array):T} deserialize Function that will deserialize\n" + " * the binary data properly.\n" + " * @param {string} name The expected type name of this message object.\n" + " * @return {?T} If the name matched the expected name, returns the deserialized\n" + " * object, otherwise returns undefined.\n" + " */\n" + "proto.google.protobuf.Any.prototype.unpack = function(deserialize, name) {\n" + " if (this.getTypeName() == name) {\n" + " return deserialize(this.getValue_asU8());\n" + " } else {\n" + " return null;\n" + " }\n" + "};\n" +}, +{"struct.js", + "// Protocol Buffers - Google's data interchange format\n" + "// Copyright 2008 Google Inc. All rights reserved.\n" + "// https://developers.google.com/protocol-buffers/\n" + "//\n" + "// Redistribution and use in source and binary forms, with or without\n" + "// modification, are permitted provided that the following conditions are\n" + "// met:\n" + "//\n" + "// * Redistributions of source code must retain the above copyright\n" + "// notice, this list of conditions and the following disclaimer.\n" + "// * Redistributions in binary form must reproduce the above\n" + "// copyright notice, this list of conditions and the following disclaimer\n" + "// in the documentation and/or other materials provided with the\n" + "// distribution.\n" + "// * Neither the name of Google Inc. nor the names of its\n" + "// contributors may be used to endorse or promote products derived from\n" + "// this software without specific prior written permission.\n" + "//\n" + "// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS\n" + "// \"AS IS\" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT\n" + "// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR\n" + "// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT\n" + "// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,\n" + "// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT\n" + "// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,\n" + "// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY\n" + "// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n" + "// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n" + "// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n" + "\n" + "/* This code will be inserted into generated code for\n" + " * google/protobuf/struct.proto. */\n" + "\n" + "/**\n" + " * Typedef representing plain JavaScript values that can go into a\n" + " * Struct.\n" + " * @typedef {null|number|string|boolean|Array|Object}\n" + " */\n" + "proto.google.protobuf.JavaScriptValue;\n" + "\n" + "\n" + "/**\n" + " * Converts this Value object to a plain JavaScript value.\n" + " * @return {?proto.google.protobuf.JavaScriptValue} a plain JavaScript\n" + " * value representing this Struct.\n" + " */\n" + "proto.google.protobuf.Value.prototype.toJavaScript = function() {\n" + " var kindCase = proto.google.protobuf.Value.KindCase;\n" + " switch (this.getKindCase()) {\n" + " case kindCase.NULL_VALUE:\n" + " return null;\n" + " case kindCase.NUMBER_VALUE:\n" + " return this.getNumberValue();\n" + " case kindCase.STRING_VALUE:\n" + " return this.getStringValue();\n" + " case kindCase.BOOL_VALUE:\n" + " return this.getBoolValue();\n" + " case kindCase.STRUCT_VALUE:\n" + " return this.getStructValue().toJavaScript();\n" + " case kindCase.LIST_VALUE:\n" + " return this.getListValue().toJavaScript();\n" + " default:\n" + " throw new Error('Unexpected struct type');\n" + " }\n" + "};\n" + "\n" + "\n" + "/**\n" + " * Converts this JavaScript value to a new Value proto.\n" + " * @param {!proto.google.protobuf.JavaScriptValue} value The value to\n" + " * convert.\n" + " * @return {!proto.google.protobuf.Value} The newly constructed value.\n" + " */\n" + "proto.google.protobuf.Value.fromJavaScript = function(value) {\n" + " var ret = new proto.google.protobuf.Value();\n" + " switch (goog.typeOf(value)) {\n" + " case 'string':\n" + " ret.setStringValue(/** @type {string} */ (value));\n" + " break;\n" + " case 'number':\n" + " ret.setNumberValue(/** @type {number} */ (value));\n" + " break;\n" + " case 'boolean':\n" + " ret.setBoolValue(/** @type {boolean} */ (value));\n" + " break;\n" + " case 'null':\n" + " ret.setNullValue(proto.google.protobuf.NullValue.NULL_VALUE);\n" + " break;\n" + " case 'array':\n" + " ret.setListValue(proto.google.protobuf.ListValue.fromJavaScript(\n" + " /** @type{!Array} */ (value)));\n" + " break;\n" + " case 'object':\n" + " ret.setStructValue(proto.google.protobuf.Struct.fromJavaScript(\n" + " /** @type{!Object} */ (value)));\n" + " break;\n" + " default:\n" + " throw new Error('Unexpected struct type.');\n" + " }\n" + "\n" + " return ret;\n" + "};\n" + "\n" + "\n" + "/**\n" + " * Converts this ListValue object to a plain JavaScript array.\n" + " * @return {!Array} a plain JavaScript array representing this List.\n" + " */\n" + "proto.google.protobuf.ListValue.prototype.toJavaScript = function() {\n" + " var ret = [];\n" + " var values = this.getValuesList();\n" + "\n" + " for (var i = 0; i < values.length; i++) {\n" + " ret[i] = values[i].toJavaScript();\n" + " }\n" + "\n" + " return ret;\n" + "};\n" + "\n" + "\n" + "/**\n" + " * Constructs a ListValue protobuf from this plain JavaScript array.\n" + " * @param {!Array} array a plain JavaScript array\n" + " * @return {proto.google.protobuf.ListValue} a new ListValue object\n" + " */\n" + "proto.google.protobuf.ListValue.fromJavaScript = function(array) {\n" + " var ret = new proto.google.protobuf.ListValue();\n" + "\n" + " for (var i = 0; i < array.length; i++) {\n" + " ret.addValues(proto.google.protobuf.Value.fromJavaScript(array[i]));\n" + " }\n" + "\n" + " return ret;\n" + "};\n" + "\n" + "\n" + "/**\n" + " * Converts this Struct object to a plain JavaScript object.\n" + " * @return {!Object<string, !proto.google.protobuf.JavaScriptValue>} a plain\n" + " * JavaScript object representing this Struct.\n" + " */\n" + "proto.google.protobuf.Struct.prototype.toJavaScript = function() {\n" + " var ret = {};\n" + "\n" + " this.getFieldsMap().forEach(function(value, key) {\n" + " ret[key] = value.toJavaScript();\n" + " });\n" + "\n" + " return ret;\n" + "};\n" + "\n" + "\n" + "/**\n" + " * Constructs a Struct protobuf from this plain JavaScript object.\n" + " * @param {!Object} obj a plain JavaScript object\n" + " * @return {proto.google.protobuf.Struct} a new Struct object\n" + " */\n" + "proto.google.protobuf.Struct.fromJavaScript = function(obj) {\n" + " var ret = new proto.google.protobuf.Struct();\n" + " var map = ret.getFieldsMap();\n" + "\n" + " for (var property in obj) {\n" + " var val = obj[property];\n" + " map.set(property, proto.google.protobuf.Value.fromJavaScript(val));\n" + " }\n" + "\n" + " return ret;\n" + "};\n" +}, +{"timestamp.js", + "// Protocol Buffers - Google's data interchange format\n" + "// Copyright 2008 Google Inc. All rights reserved.\n" + "// https://developers.google.com/protocol-buffers/\n" + "//\n" + "// Redistribution and use in source and binary forms, with or without\n" + "// modification, are permitted provided that the following conditions are\n" + "// met:\n" + "//\n" + "// * Redistributions of source code must retain the above copyright\n" + "// notice, this list of conditions and the following disclaimer.\n" + "// * Redistributions in binary form must reproduce the above\n" + "// copyright notice, this list of conditions and the following disclaimer\n" + "// in the documentation and/or other materials provided with the\n" + "// distribution.\n" + "// * Neither the name of Google Inc. nor the names of its\n" + "// contributors may be used to endorse or promote products derived from\n" + "// this software without specific prior written permission.\n" + "//\n" + "// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS\n" + "// \"AS IS\" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT\n" + "// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR\n" + "// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT\n" + "// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,\n" + "// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT\n" + "// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,\n" + "// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY\n" + "// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n" + "// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n" + "// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n" + "\n" + "/* This code will be inserted into generated code for\n" + " * google/protobuf/timestamp.proto. */\n" + "\n" + "/**\n" + " * Returns a JavaScript 'Date' object corresponding to this Timestamp.\n" + " * @return {!Date}\n" + " */\n" + "proto.google.protobuf.Timestamp.prototype.toDate = function() {\n" + " var seconds = this.getSeconds();\n" + " var nanos = this.getNanos();\n" + "\n" + " return new Date((seconds * 1000) + (nanos / 1000000));\n" + "};\n" + "\n" + "\n" + "/**\n" + " * Sets the value of this Timestamp object to be the given Date.\n" + " * @param {!Date} value The value to set.\n" + " */\n" + "proto.google.protobuf.Timestamp.prototype.fromDate = function(value) {\n" + " var millis = value.getTime();\n" + " this.setSeconds(Math.floor(value.getTime() / 1000));\n" + " this.setNanos(value.getMilliseconds() * 1000000);\n" + "};\n" +}, + {NULL, NULL} // Terminate the list. +}; diff --git a/tools/distrib/python/grpcio_tools/protoc_lib_deps.py b/tools/distrib/python/grpcio_tools/protoc_lib_deps.py index 569328e57e..c2aa6198b3 100644 --- a/tools/distrib/python/grpcio_tools/protoc_lib_deps.py +++ b/tools/distrib/python/grpcio_tools/protoc_lib_deps.py @@ -29,7 +29,7 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # AUTO-GENERATED BY make_grpcio_tools.py! -CC_FILES=['google/protobuf/compiler/zip_writer.cc', 'google/protobuf/compiler/subprocess.cc', 'google/protobuf/compiler/ruby/ruby_generator.cc', 'google/protobuf/compiler/python/python_generator.cc', 'google/protobuf/compiler/plugin.pb.cc', 'google/protobuf/compiler/plugin.cc', 'google/protobuf/compiler/php/php_generator.cc', 'google/protobuf/compiler/objectivec/objectivec_primitive_field.cc', 'google/protobuf/compiler/objectivec/objectivec_oneof.cc', 'google/protobuf/compiler/objectivec/objectivec_message_field.cc', 'google/protobuf/compiler/objectivec/objectivec_message.cc', 'google/protobuf/compiler/objectivec/objectivec_map_field.cc', 'google/protobuf/compiler/objectivec/objectivec_helpers.cc', 'google/protobuf/compiler/objectivec/objectivec_generator.cc', 'google/protobuf/compiler/objectivec/objectivec_file.cc', 'google/protobuf/compiler/objectivec/objectivec_field.cc', 'google/protobuf/compiler/objectivec/objectivec_extension.cc', 'google/protobuf/compiler/objectivec/objectivec_enum_field.cc', 'google/protobuf/compiler/objectivec/objectivec_enum.cc', 'google/protobuf/compiler/js/js_generator.cc', 'google/protobuf/compiler/javanano/javanano_primitive_field.cc', 'google/protobuf/compiler/javanano/javanano_message_field.cc', 'google/protobuf/compiler/javanano/javanano_message.cc', 'google/protobuf/compiler/javanano/javanano_map_field.cc', 'google/protobuf/compiler/javanano/javanano_helpers.cc', 'google/protobuf/compiler/javanano/javanano_generator.cc', 'google/protobuf/compiler/javanano/javanano_file.cc', 'google/protobuf/compiler/javanano/javanano_field.cc', 'google/protobuf/compiler/javanano/javanano_extension.cc', 'google/protobuf/compiler/javanano/javanano_enum_field.cc', 'google/protobuf/compiler/javanano/javanano_enum.cc', 'google/protobuf/compiler/java/java_string_field_lite.cc', 'google/protobuf/compiler/java/java_string_field.cc', 'google/protobuf/compiler/java/java_shared_code_generator.cc', 'google/protobuf/compiler/java/java_service.cc', 'google/protobuf/compiler/java/java_primitive_field_lite.cc', 'google/protobuf/compiler/java/java_primitive_field.cc', 'google/protobuf/compiler/java/java_name_resolver.cc', 'google/protobuf/compiler/java/java_message_lite.cc', 'google/protobuf/compiler/java/java_message_field_lite.cc', 'google/protobuf/compiler/java/java_message_field.cc', 'google/protobuf/compiler/java/java_message_builder_lite.cc', 'google/protobuf/compiler/java/java_message_builder.cc', 'google/protobuf/compiler/java/java_message.cc', 'google/protobuf/compiler/java/java_map_field_lite.cc', 'google/protobuf/compiler/java/java_map_field.cc', 'google/protobuf/compiler/java/java_lazy_message_field_lite.cc', 'google/protobuf/compiler/java/java_lazy_message_field.cc', 'google/protobuf/compiler/java/java_helpers.cc', 'google/protobuf/compiler/java/java_generator_factory.cc', 'google/protobuf/compiler/java/java_generator.cc', 'google/protobuf/compiler/java/java_file.cc', 'google/protobuf/compiler/java/java_field.cc', 'google/protobuf/compiler/java/java_extension_lite.cc', 'google/protobuf/compiler/java/java_extension.cc', 'google/protobuf/compiler/java/java_enum_lite.cc', 'google/protobuf/compiler/java/java_enum_field_lite.cc', 'google/protobuf/compiler/java/java_enum_field.cc', 'google/protobuf/compiler/java/java_enum.cc', 'google/protobuf/compiler/java/java_doc_comment.cc', 'google/protobuf/compiler/java/java_context.cc', 'google/protobuf/compiler/csharp/csharp_wrapper_field.cc', 'google/protobuf/compiler/csharp/csharp_source_generator_base.cc', 'google/protobuf/compiler/csharp/csharp_repeated_primitive_field.cc', 'google/protobuf/compiler/csharp/csharp_repeated_message_field.cc', 'google/protobuf/compiler/csharp/csharp_repeated_enum_field.cc', 'google/protobuf/compiler/csharp/csharp_reflection_class.cc', 'google/protobuf/compiler/csharp/csharp_primitive_field.cc', 'google/protobuf/compiler/csharp/csharp_message_field.cc', 'google/protobuf/compiler/csharp/csharp_message.cc', 'google/protobuf/compiler/csharp/csharp_map_field.cc', 'google/protobuf/compiler/csharp/csharp_helpers.cc', 'google/protobuf/compiler/csharp/csharp_generator.cc', 'google/protobuf/compiler/csharp/csharp_field_base.cc', 'google/protobuf/compiler/csharp/csharp_enum_field.cc', 'google/protobuf/compiler/csharp/csharp_enum.cc', 'google/protobuf/compiler/csharp/csharp_doc_comment.cc', 'google/protobuf/compiler/cpp/cpp_string_field.cc', 'google/protobuf/compiler/cpp/cpp_service.cc', 'google/protobuf/compiler/cpp/cpp_primitive_field.cc', 'google/protobuf/compiler/cpp/cpp_message_field.cc', 'google/protobuf/compiler/cpp/cpp_message.cc', 'google/protobuf/compiler/cpp/cpp_map_field.cc', 'google/protobuf/compiler/cpp/cpp_helpers.cc', 'google/protobuf/compiler/cpp/cpp_generator.cc', 'google/protobuf/compiler/cpp/cpp_file.cc', 'google/protobuf/compiler/cpp/cpp_field.cc', 'google/protobuf/compiler/cpp/cpp_extension.cc', 'google/protobuf/compiler/cpp/cpp_enum_field.cc', 'google/protobuf/compiler/cpp/cpp_enum.cc', 'google/protobuf/compiler/command_line_interface.cc', 'google/protobuf/compiler/code_generator.cc', 'google/protobuf/wrappers.pb.cc', 'google/protobuf/wire_format.cc', 'google/protobuf/util/type_resolver_util.cc', 'google/protobuf/util/time_util.cc', 'google/protobuf/util/message_differencer.cc', 'google/protobuf/util/json_util.cc', 'google/protobuf/util/internal/utility.cc', 'google/protobuf/util/internal/type_info_test_helper.cc', 'google/protobuf/util/internal/type_info.cc', 'google/protobuf/util/internal/protostream_objectwriter.cc', 'google/protobuf/util/internal/protostream_objectsource.cc', 'google/protobuf/util/internal/proto_writer.cc', 'google/protobuf/util/internal/object_writer.cc', 'google/protobuf/util/internal/json_stream_parser.cc', 'google/protobuf/util/internal/json_objectwriter.cc', 'google/protobuf/util/internal/json_escaping.cc', 'google/protobuf/util/internal/field_mask_utility.cc', 'google/protobuf/util/internal/error_listener.cc', 'google/protobuf/util/internal/default_value_objectwriter.cc', 'google/protobuf/util/internal/datapiece.cc', 'google/protobuf/util/field_mask_util.cc', 'google/protobuf/util/field_comparator.cc', 'google/protobuf/unknown_field_set.cc', 'google/protobuf/type.pb.cc', 'google/protobuf/timestamp.pb.cc', 'google/protobuf/text_format.cc', 'google/protobuf/stubs/substitute.cc', 'google/protobuf/stubs/mathlimits.cc', 'google/protobuf/struct.pb.cc', 'google/protobuf/source_context.pb.cc', 'google/protobuf/service.cc', 'google/protobuf/reflection_ops.cc', 'google/protobuf/message.cc', 'google/protobuf/map_field.cc', 'google/protobuf/io/zero_copy_stream_impl.cc', 'google/protobuf/io/tokenizer.cc', 'google/protobuf/io/strtod.cc', 'google/protobuf/io/printer.cc', 'google/protobuf/io/gzip_stream.cc', 'google/protobuf/generated_message_reflection.cc', 'google/protobuf/field_mask.pb.cc', 'google/protobuf/extension_set_heavy.cc', 'google/protobuf/empty.pb.cc', 'google/protobuf/dynamic_message.cc', 'google/protobuf/duration.pb.cc', 'google/protobuf/descriptor_database.cc', 'google/protobuf/descriptor.pb.cc', 'google/protobuf/descriptor.cc', 'google/protobuf/compiler/parser.cc', 'google/protobuf/compiler/importer.cc', 'google/protobuf/api.pb.cc', 'google/protobuf/any.pb.cc', 'google/protobuf/any.cc', 'google/protobuf/wire_format_lite.cc', 'google/protobuf/stubs/time.cc', 'google/protobuf/stubs/strutil.cc', 'google/protobuf/stubs/structurally_valid.cc', 'google/protobuf/stubs/stringprintf.cc', 'google/protobuf/stubs/stringpiece.cc', 'google/protobuf/stubs/statusor.cc', 'google/protobuf/stubs/status.cc', 'google/protobuf/stubs/once.cc', 'google/protobuf/stubs/int128.cc', 'google/protobuf/stubs/common.cc', 'google/protobuf/stubs/bytestream.cc', 'google/protobuf/stubs/atomicops_internals_x86_msvc.cc', 'google/protobuf/stubs/atomicops_internals_x86_gcc.cc', 'google/protobuf/repeated_field.cc', 'google/protobuf/message_lite.cc', 'google/protobuf/io/zero_copy_stream_impl_lite.cc', 'google/protobuf/io/zero_copy_stream.cc', 'google/protobuf/io/coded_stream.cc', 'google/protobuf/generated_message_util.cc', 'google/protobuf/extension_set.cc', 'google/protobuf/arenastring.cc', 'google/protobuf/arena.cc'] +CC_FILES=['google/protobuf/compiler/zip_writer.cc', 'google/protobuf/compiler/subprocess.cc', 'google/protobuf/compiler/ruby/ruby_generator.cc', 'google/protobuf/compiler/python/python_generator.cc', 'google/protobuf/compiler/plugin.pb.cc', 'google/protobuf/compiler/plugin.cc', 'google/protobuf/compiler/php/php_generator.cc', 'google/protobuf/compiler/objectivec/objectivec_primitive_field.cc', 'google/protobuf/compiler/objectivec/objectivec_oneof.cc', 'google/protobuf/compiler/objectivec/objectivec_message_field.cc', 'google/protobuf/compiler/objectivec/objectivec_message.cc', 'google/protobuf/compiler/objectivec/objectivec_map_field.cc', 'google/protobuf/compiler/objectivec/objectivec_helpers.cc', 'google/protobuf/compiler/objectivec/objectivec_generator.cc', 'google/protobuf/compiler/objectivec/objectivec_file.cc', 'google/protobuf/compiler/objectivec/objectivec_field.cc', 'google/protobuf/compiler/objectivec/objectivec_extension.cc', 'google/protobuf/compiler/objectivec/objectivec_enum_field.cc', 'google/protobuf/compiler/objectivec/objectivec_enum.cc', 'google/protobuf/compiler/js/well_known_types_embed.cc', 'google/protobuf/compiler/js/js_generator.cc', 'google/protobuf/compiler/javanano/javanano_primitive_field.cc', 'google/protobuf/compiler/javanano/javanano_message_field.cc', 'google/protobuf/compiler/javanano/javanano_message.cc', 'google/protobuf/compiler/javanano/javanano_map_field.cc', 'google/protobuf/compiler/javanano/javanano_helpers.cc', 'google/protobuf/compiler/javanano/javanano_generator.cc', 'google/protobuf/compiler/javanano/javanano_file.cc', 'google/protobuf/compiler/javanano/javanano_field.cc', 'google/protobuf/compiler/javanano/javanano_extension.cc', 'google/protobuf/compiler/javanano/javanano_enum_field.cc', 'google/protobuf/compiler/javanano/javanano_enum.cc', 'google/protobuf/compiler/java/java_string_field_lite.cc', 'google/protobuf/compiler/java/java_string_field.cc', 'google/protobuf/compiler/java/java_shared_code_generator.cc', 'google/protobuf/compiler/java/java_service.cc', 'google/protobuf/compiler/java/java_primitive_field_lite.cc', 'google/protobuf/compiler/java/java_primitive_field.cc', 'google/protobuf/compiler/java/java_name_resolver.cc', 'google/protobuf/compiler/java/java_message_lite.cc', 'google/protobuf/compiler/java/java_message_field_lite.cc', 'google/protobuf/compiler/java/java_message_field.cc', 'google/protobuf/compiler/java/java_message_builder_lite.cc', 'google/protobuf/compiler/java/java_message_builder.cc', 'google/protobuf/compiler/java/java_message.cc', 'google/protobuf/compiler/java/java_map_field_lite.cc', 'google/protobuf/compiler/java/java_map_field.cc', 'google/protobuf/compiler/java/java_lazy_message_field_lite.cc', 'google/protobuf/compiler/java/java_lazy_message_field.cc', 'google/protobuf/compiler/java/java_helpers.cc', 'google/protobuf/compiler/java/java_generator_factory.cc', 'google/protobuf/compiler/java/java_generator.cc', 'google/protobuf/compiler/java/java_file.cc', 'google/protobuf/compiler/java/java_field.cc', 'google/protobuf/compiler/java/java_extension_lite.cc', 'google/protobuf/compiler/java/java_extension.cc', 'google/protobuf/compiler/java/java_enum_lite.cc', 'google/protobuf/compiler/java/java_enum_field_lite.cc', 'google/protobuf/compiler/java/java_enum_field.cc', 'google/protobuf/compiler/java/java_enum.cc', 'google/protobuf/compiler/java/java_doc_comment.cc', 'google/protobuf/compiler/java/java_context.cc', 'google/protobuf/compiler/csharp/csharp_wrapper_field.cc', 'google/protobuf/compiler/csharp/csharp_source_generator_base.cc', 'google/protobuf/compiler/csharp/csharp_repeated_primitive_field.cc', 'google/protobuf/compiler/csharp/csharp_repeated_message_field.cc', 'google/protobuf/compiler/csharp/csharp_repeated_enum_field.cc', 'google/protobuf/compiler/csharp/csharp_reflection_class.cc', 'google/protobuf/compiler/csharp/csharp_primitive_field.cc', 'google/protobuf/compiler/csharp/csharp_message_field.cc', 'google/protobuf/compiler/csharp/csharp_message.cc', 'google/protobuf/compiler/csharp/csharp_map_field.cc', 'google/protobuf/compiler/csharp/csharp_helpers.cc', 'google/protobuf/compiler/csharp/csharp_generator.cc', 'google/protobuf/compiler/csharp/csharp_field_base.cc', 'google/protobuf/compiler/csharp/csharp_enum_field.cc', 'google/protobuf/compiler/csharp/csharp_enum.cc', 'google/protobuf/compiler/csharp/csharp_doc_comment.cc', 'google/protobuf/compiler/cpp/cpp_string_field.cc', 'google/protobuf/compiler/cpp/cpp_service.cc', 'google/protobuf/compiler/cpp/cpp_primitive_field.cc', 'google/protobuf/compiler/cpp/cpp_message_field.cc', 'google/protobuf/compiler/cpp/cpp_message.cc', 'google/protobuf/compiler/cpp/cpp_map_field.cc', 'google/protobuf/compiler/cpp/cpp_helpers.cc', 'google/protobuf/compiler/cpp/cpp_generator.cc', 'google/protobuf/compiler/cpp/cpp_file.cc', 'google/protobuf/compiler/cpp/cpp_field.cc', 'google/protobuf/compiler/cpp/cpp_extension.cc', 'google/protobuf/compiler/cpp/cpp_enum_field.cc', 'google/protobuf/compiler/cpp/cpp_enum.cc', 'google/protobuf/compiler/command_line_interface.cc', 'google/protobuf/compiler/code_generator.cc', 'google/protobuf/wrappers.pb.cc', 'google/protobuf/wire_format.cc', 'google/protobuf/util/type_resolver_util.cc', 'google/protobuf/util/time_util.cc', 'google/protobuf/util/message_differencer.cc', 'google/protobuf/util/json_util.cc', 'google/protobuf/util/internal/utility.cc', 'google/protobuf/util/internal/type_info_test_helper.cc', 'google/protobuf/util/internal/type_info.cc', 'google/protobuf/util/internal/protostream_objectwriter.cc', 'google/protobuf/util/internal/protostream_objectsource.cc', 'google/protobuf/util/internal/proto_writer.cc', 'google/protobuf/util/internal/object_writer.cc', 'google/protobuf/util/internal/json_stream_parser.cc', 'google/protobuf/util/internal/json_objectwriter.cc', 'google/protobuf/util/internal/json_escaping.cc', 'google/protobuf/util/internal/field_mask_utility.cc', 'google/protobuf/util/internal/error_listener.cc', 'google/protobuf/util/internal/default_value_objectwriter.cc', 'google/protobuf/util/internal/datapiece.cc', 'google/protobuf/util/field_mask_util.cc', 'google/protobuf/util/field_comparator.cc', 'google/protobuf/unknown_field_set.cc', 'google/protobuf/type.pb.cc', 'google/protobuf/timestamp.pb.cc', 'google/protobuf/text_format.cc', 'google/protobuf/stubs/substitute.cc', 'google/protobuf/stubs/mathlimits.cc', 'google/protobuf/struct.pb.cc', 'google/protobuf/source_context.pb.cc', 'google/protobuf/service.cc', 'google/protobuf/reflection_ops.cc', 'google/protobuf/message.cc', 'google/protobuf/map_field.cc', 'google/protobuf/io/zero_copy_stream_impl.cc', 'google/protobuf/io/tokenizer.cc', 'google/protobuf/io/strtod.cc', 'google/protobuf/io/printer.cc', 'google/protobuf/io/gzip_stream.cc', 'google/protobuf/generated_message_reflection.cc', 'google/protobuf/field_mask.pb.cc', 'google/protobuf/extension_set_heavy.cc', 'google/protobuf/empty.pb.cc', 'google/protobuf/dynamic_message.cc', 'google/protobuf/duration.pb.cc', 'google/protobuf/descriptor_database.cc', 'google/protobuf/descriptor.pb.cc', 'google/protobuf/descriptor.cc', 'google/protobuf/compiler/parser.cc', 'google/protobuf/compiler/importer.cc', 'google/protobuf/api.pb.cc', 'google/protobuf/any.pb.cc', 'google/protobuf/any.cc', 'google/protobuf/wire_format_lite.cc', 'google/protobuf/stubs/time.cc', 'google/protobuf/stubs/strutil.cc', 'google/protobuf/stubs/structurally_valid.cc', 'google/protobuf/stubs/stringprintf.cc', 'google/protobuf/stubs/stringpiece.cc', 'google/protobuf/stubs/statusor.cc', 'google/protobuf/stubs/status.cc', 'google/protobuf/stubs/once.cc', 'google/protobuf/stubs/int128.cc', 'google/protobuf/stubs/common.cc', 'google/protobuf/stubs/bytestream.cc', 'google/protobuf/stubs/atomicops_internals_x86_msvc.cc', 'google/protobuf/stubs/atomicops_internals_x86_gcc.cc', 'google/protobuf/repeated_field.cc', 'google/protobuf/message_lite.cc', 'google/protobuf/io/zero_copy_stream_impl_lite.cc', 'google/protobuf/io/zero_copy_stream.cc', 'google/protobuf/io/coded_stream.cc', 'google/protobuf/generated_message_util.cc', 'google/protobuf/extension_set.cc', 'google/protobuf/arenastring.cc', 'google/protobuf/arena.cc', 'google/protobuf/compiler/js/embed.cc'] PROTO_FILES=['google/protobuf/wrappers.proto', 'google/protobuf/type.proto', 'google/protobuf/timestamp.proto', 'google/protobuf/struct.proto', 'google/protobuf/source_context.proto', 'google/protobuf/field_mask.proto', 'google/protobuf/empty.proto', 'google/protobuf/duration.proto', 'google/protobuf/descriptor.proto', 'google/protobuf/compiler/plugin.proto', 'google/protobuf/api.proto', 'google/protobuf/any.proto'] CC_INCLUDE='third_party/protobuf/src' diff --git a/tools/distrib/python/grpcio_tools/setup.py b/tools/distrib/python/grpcio_tools/setup.py index 502d7ef27b..ed27f1f835 100644 --- a/tools/distrib/python/grpcio_tools/setup.py +++ b/tools/distrib/python/grpcio_tools/setup.py @@ -157,11 +157,28 @@ def extension_modules(): plugin_sources = [os.path.join('grpc_tools', '_protoc_compiler.pyx')] else: plugin_sources = [os.path.join('grpc_tools', '_protoc_compiler.cpp')] + plugin_sources += [ os.path.join('grpc_tools', 'main.cc'), - os.path.join('grpc_root', 'src', 'compiler', 'python_generator.cc')] + [ - os.path.join(CC_INCLUDE, cc_file) - for cc_file in CC_FILES] + os.path.join('grpc_root', 'src', 'compiler', 'python_generator.cc')] + + #HACK: Substitute the embed.cc, which is a JS to C++ + # preprocessor with the generated code. + # The generated code should not be material + # to the parts of protoc we use (it affects + # the JavaScript code generator, supposedly), + # but we need to be cautious about it. + cc_files_clone = list(CC_FILES) + embed_cc_file = os.path.normpath('google/protobuf/compiler/js/embed.cc') + well_known_types_file = os.path.normpath( + 'google/protobuf/compiler/js/well_known_types_embed.cc') + if embed_cc_file in cc_files_clone: + cc_files_clone.remove(embed_cc_file) + if well_known_types_file in cc_files_clone: + cc_files_clone.remove(well_known_types_file) + plugin_sources += [os.path.join('grpc_tools', 'protobuf_generated_well_known_types_embed.cc')] + plugin_sources += [os.path.join(CC_INCLUDE, cc_file) for cc_file in cc_files_clone] + plugin_ext = extension.Extension( name='grpc_tools._protoc_compiler', sources=plugin_sources, @@ -190,7 +207,7 @@ setuptools.setup( ext_modules=extension_modules(), packages=setuptools.find_packages('.'), install_requires=[ - 'protobuf>=3.0.0', + 'protobuf>=3.2.0', 'grpcio>={version}'.format(version=grpc_version.VERSION), ], package_data=package_data(), diff --git a/tools/dockerfile/grpc_artifact_linux_x64/Dockerfile b/tools/dockerfile/grpc_artifact_linux_x64/Dockerfile index 669e3557b8..3479944717 100644 --- a/tools/dockerfile/grpc_artifact_linux_x64/Dockerfile +++ b/tools/dockerfile/grpc_artifact_linux_x64/Dockerfile @@ -76,7 +76,7 @@ RUN apt-get update && apt-get install -y \ RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.0.0a2 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 ################## diff --git a/tools/dockerfile/grpc_artifact_linux_x86/Dockerfile b/tools/dockerfile/grpc_artifact_linux_x86/Dockerfile index 860b8f4fb9..75d156f6d8 100644 --- a/tools/dockerfile/grpc_artifact_linux_x86/Dockerfile +++ b/tools/dockerfile/grpc_artifact_linux_x86/Dockerfile @@ -76,7 +76,7 @@ RUN apt-get update && apt-get install -y \ RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.0.0a2 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 ################## diff --git a/tools/dockerfile/interoptest/grpc_interop_csharp/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_csharp/Dockerfile index 087cc4e2bb..14a2468abc 100644 --- a/tools/dockerfile/interoptest/grpc_interop_csharp/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_csharp/Dockerfile @@ -76,7 +76,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.0.0a2 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 #================ # C# dependencies diff --git a/tools/dockerfile/interoptest/grpc_interop_csharpcoreclr/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_csharpcoreclr/Dockerfile index efe6e39118..91639829dc 100644 --- a/tools/dockerfile/interoptest/grpc_interop_csharpcoreclr/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_csharpcoreclr/Dockerfile @@ -76,7 +76,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.0.0a2 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 #================ # C# dependencies diff --git a/tools/dockerfile/interoptest/grpc_interop_cxx/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_cxx/Dockerfile index aa77d5f127..2d10e3fdfe 100644 --- a/tools/dockerfile/interoptest/grpc_interop_cxx/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_cxx/Dockerfile @@ -76,7 +76,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.0.0a2 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 #================= # C++ dependencies diff --git a/tools/dockerfile/interoptest/grpc_interop_cxx/build_interop.sh b/tools/dockerfile/interoptest/grpc_interop_cxx/build_interop.sh index 7a7ca0d3d1..0a61334103 100755 --- a/tools/dockerfile/interoptest/grpc_interop_cxx/build_interop.sh +++ b/tools/dockerfile/interoptest/grpc_interop_cxx/build_interop.sh @@ -47,3 +47,6 @@ make install-certs # build C++ interop client & server make interop_client interop_server + +# build C++ http2 client +make http2_client diff --git a/tools/dockerfile/interoptest/grpc_interop_go/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_go/Dockerfile index 05e963d1e6..d3bf071c72 100644 --- a/tools/dockerfile/interoptest/grpc_interop_go/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_go/Dockerfile @@ -45,7 +45,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.0.0a2 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 # Define the default command. CMD ["bash"] diff --git a/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile index 3a5e15d21b..66d9b4f640 100644 --- a/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile @@ -45,7 +45,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.0.0a2 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 RUN pip install twisted h2 diff --git a/tools/dockerfile/interoptest/grpc_interop_java/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_java/Dockerfile index b5fe54f991..2023467d59 100644 --- a/tools/dockerfile/interoptest/grpc_interop_java/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_java/Dockerfile @@ -60,7 +60,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.0.0a2 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 # Trigger download of as many Gradle artifacts as possible. diff --git a/tools/dockerfile/interoptest/grpc_interop_node/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_node/Dockerfile index d9a7501829..9945260ea4 100644 --- a/tools/dockerfile/interoptest/grpc_interop_node/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_node/Dockerfile @@ -76,7 +76,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.0.0a2 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 #================== # Node dependencies diff --git a/tools/dockerfile/interoptest/grpc_interop_python/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_python/Dockerfile index 10a88916ad..94c17078d3 100644 --- a/tools/dockerfile/interoptest/grpc_interop_python/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_python/Dockerfile @@ -76,7 +76,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.0.0a2 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 # Prepare ccache RUN ln -s /usr/bin/ccache /usr/local/bin/gcc diff --git a/tools/dockerfile/interoptest/grpc_interop_ruby/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_ruby/Dockerfile index dae64e5c8c..679c8ff47a 100644 --- a/tools/dockerfile/interoptest/grpc_interop_ruby/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_ruby/Dockerfile @@ -76,7 +76,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.0.0a2 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 #================== # Ruby dependencies diff --git a/tools/dockerfile/push_testing_images.sh b/tools/dockerfile/push_testing_images.sh index 9dceb29a87..973e045ffe 100755 --- a/tools/dockerfile/push_testing_images.sh +++ b/tools/dockerfile/push_testing_images.sh @@ -44,7 +44,7 @@ cd - DOCKERHUB_ORGANIZATION=grpctesting -for DOCKERFILE_DIR in tools/dockerfile/test/fuzzer tools/dockerfile/test/sanity +for DOCKERFILE_DIR in tools/dockerfile/test/* do # Generate image name based on Dockerfile checksum. That works well as long # as can count on dockerfiles being written in a way that changing the logical diff --git a/tools/dockerfile/stress_test/grpc_interop_stress_csharp/Dockerfile b/tools/dockerfile/stress_test/grpc_interop_stress_csharp/Dockerfile index 328825392b..cd1e934341 100644 --- a/tools/dockerfile/stress_test/grpc_interop_stress_csharp/Dockerfile +++ b/tools/dockerfile/stress_test/grpc_interop_stress_csharp/Dockerfile @@ -76,7 +76,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.0.0a2 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 # Prepare ccache RUN ln -s /usr/bin/ccache /usr/local/bin/gcc diff --git a/tools/dockerfile/stress_test/grpc_interop_stress_cxx/Dockerfile b/tools/dockerfile/stress_test/grpc_interop_stress_cxx/Dockerfile index e082da648b..d0f66d9955 100644 --- a/tools/dockerfile/stress_test/grpc_interop_stress_cxx/Dockerfile +++ b/tools/dockerfile/stress_test/grpc_interop_stress_cxx/Dockerfile @@ -76,7 +76,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.0.0a2 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 # Prepare ccache RUN ln -s /usr/bin/ccache /usr/local/bin/gcc diff --git a/tools/dockerfile/stress_test/grpc_interop_stress_go/Dockerfile b/tools/dockerfile/stress_test/grpc_interop_stress_go/Dockerfile index 1e2b7d8c67..bbf7de7f91 100644 --- a/tools/dockerfile/stress_test/grpc_interop_stress_go/Dockerfile +++ b/tools/dockerfile/stress_test/grpc_interop_stress_go/Dockerfile @@ -47,7 +47,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.0.0a2 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 # Using login shell removes Go from path, so we add it. RUN ln -s /usr/local/go/bin/go /usr/local/bin diff --git a/tools/dockerfile/stress_test/grpc_interop_stress_java/Dockerfile b/tools/dockerfile/stress_test/grpc_interop_stress_java/Dockerfile index 0c17ff595e..229ea469c4 100644 --- a/tools/dockerfile/stress_test/grpc_interop_stress_java/Dockerfile +++ b/tools/dockerfile/stress_test/grpc_interop_stress_java/Dockerfile @@ -76,7 +76,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.0.0a2 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 # Prepare ccache RUN ln -s /usr/bin/ccache /usr/local/bin/gcc diff --git a/tools/dockerfile/stress_test/grpc_interop_stress_node/Dockerfile b/tools/dockerfile/stress_test/grpc_interop_stress_node/Dockerfile index 0594f69a5b..5fd0bc0eb2 100644 --- a/tools/dockerfile/stress_test/grpc_interop_stress_node/Dockerfile +++ b/tools/dockerfile/stress_test/grpc_interop_stress_node/Dockerfile @@ -76,7 +76,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.0.0a2 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 #================== # Node dependencies diff --git a/tools/dockerfile/stress_test/grpc_interop_stress_php/Dockerfile b/tools/dockerfile/stress_test/grpc_interop_stress_php/Dockerfile index 0fe9c151e5..b5198b4652 100644 --- a/tools/dockerfile/stress_test/grpc_interop_stress_php/Dockerfile +++ b/tools/dockerfile/stress_test/grpc_interop_stress_php/Dockerfile @@ -76,7 +76,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.0.0a2 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 #================== # Ruby dependencies diff --git a/tools/dockerfile/stress_test/grpc_interop_stress_python/Dockerfile b/tools/dockerfile/stress_test/grpc_interop_stress_python/Dockerfile index 20d2d3f57b..8e1de51f33 100644 --- a/tools/dockerfile/stress_test/grpc_interop_stress_python/Dockerfile +++ b/tools/dockerfile/stress_test/grpc_interop_stress_python/Dockerfile @@ -93,7 +93,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.0.0a2 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 RUN pip install coverage diff --git a/tools/dockerfile/stress_test/grpc_interop_stress_ruby/Dockerfile b/tools/dockerfile/stress_test/grpc_interop_stress_ruby/Dockerfile index f459153fe5..9d291aac58 100644 --- a/tools/dockerfile/stress_test/grpc_interop_stress_ruby/Dockerfile +++ b/tools/dockerfile/stress_test/grpc_interop_stress_ruby/Dockerfile @@ -76,7 +76,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.0.0a2 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 # Prepare ccache RUN ln -s /usr/bin/ccache /usr/local/bin/gcc diff --git a/tools/dockerfile/test/csharp_coreclr_x64/Dockerfile b/tools/dockerfile/test/csharp_coreclr_x64/Dockerfile index efe6e39118..91639829dc 100644 --- a/tools/dockerfile/test/csharp_coreclr_x64/Dockerfile +++ b/tools/dockerfile/test/csharp_coreclr_x64/Dockerfile @@ -76,7 +76,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.0.0a2 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 #================ # C# dependencies diff --git a/tools/dockerfile/test/csharp_jessie_x64/Dockerfile b/tools/dockerfile/test/csharp_jessie_x64/Dockerfile index 087cc4e2bb..14a2468abc 100644 --- a/tools/dockerfile/test/csharp_jessie_x64/Dockerfile +++ b/tools/dockerfile/test/csharp_jessie_x64/Dockerfile @@ -76,7 +76,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.0.0a2 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 #================ # C# dependencies diff --git a/tools/dockerfile/test/cxx_jessie_x64/Dockerfile b/tools/dockerfile/test/cxx_jessie_x64/Dockerfile index e968a0f589..4bb97c7aa9 100644 --- a/tools/dockerfile/test/cxx_jessie_x64/Dockerfile +++ b/tools/dockerfile/test/cxx_jessie_x64/Dockerfile @@ -76,7 +76,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.0.0a2 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 #================= # C++ dependencies diff --git a/tools/dockerfile/test/cxx_jessie_x86/Dockerfile b/tools/dockerfile/test/cxx_jessie_x86/Dockerfile index f985480254..c4b710b5df 100644 --- a/tools/dockerfile/test/cxx_jessie_x86/Dockerfile +++ b/tools/dockerfile/test/cxx_jessie_x86/Dockerfile @@ -76,7 +76,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.0.0a2 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 #================= # C++ dependencies diff --git a/tools/dockerfile/test/cxx_ubuntu1404_x64/Dockerfile b/tools/dockerfile/test/cxx_ubuntu1404_x64/Dockerfile index 2b3f4af3e6..bd742dff34 100644 --- a/tools/dockerfile/test/cxx_ubuntu1404_x64/Dockerfile +++ b/tools/dockerfile/test/cxx_ubuntu1404_x64/Dockerfile @@ -76,7 +76,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.0.0a2 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 #================= # C++ dependencies diff --git a/tools/dockerfile/test/cxx_ubuntu1604_x64/Dockerfile b/tools/dockerfile/test/cxx_ubuntu1604_x64/Dockerfile index 2d282276d3..bc46b3055a 100644 --- a/tools/dockerfile/test/cxx_ubuntu1604_x64/Dockerfile +++ b/tools/dockerfile/test/cxx_ubuntu1604_x64/Dockerfile @@ -76,7 +76,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.0.0a2 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 #================= # C++ dependencies diff --git a/tools/dockerfile/test/cxx_wheezy_x64/Dockerfile b/tools/dockerfile/test/cxx_wheezy_x64/Dockerfile index f22fcacc13..f7d7f542c1 100644 --- a/tools/dockerfile/test/cxx_wheezy_x64/Dockerfile +++ b/tools/dockerfile/test/cxx_wheezy_x64/Dockerfile @@ -76,7 +76,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.0.0a2 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 #================= # C++ dependencies diff --git a/tools/dockerfile/test/fuzzer/Dockerfile b/tools/dockerfile/test/fuzzer/Dockerfile index bd04f07cea..b398b70b64 100644 --- a/tools/dockerfile/test/fuzzer/Dockerfile +++ b/tools/dockerfile/test/fuzzer/Dockerfile @@ -76,7 +76,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.0.0a2 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 #================= # C++ dependencies diff --git a/tools/dockerfile/test/multilang_jessie_x64/Dockerfile b/tools/dockerfile/test/multilang_jessie_x64/Dockerfile index 2540b52ec8..5d0c1686f1 100644 --- a/tools/dockerfile/test/multilang_jessie_x64/Dockerfile +++ b/tools/dockerfile/test/multilang_jessie_x64/Dockerfile @@ -133,7 +133,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.0.0a2 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 # Prepare ccache RUN ln -s /usr/bin/ccache /usr/local/bin/gcc diff --git a/tools/dockerfile/test/node_jessie_x64/Dockerfile b/tools/dockerfile/test/node_jessie_x64/Dockerfile index 7f93933eca..4595aa6bea 100644 --- a/tools/dockerfile/test/node_jessie_x64/Dockerfile +++ b/tools/dockerfile/test/node_jessie_x64/Dockerfile @@ -87,7 +87,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.0.0a2 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 #================== # Node dependencies diff --git a/tools/dockerfile/test/php7_jessie_x64/Dockerfile b/tools/dockerfile/test/php7_jessie_x64/Dockerfile index 221338956e..0e2c103afd 100644 --- a/tools/dockerfile/test/php7_jessie_x64/Dockerfile +++ b/tools/dockerfile/test/php7_jessie_x64/Dockerfile @@ -88,7 +88,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.0.0a2 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 # Prepare ccache RUN ln -s /usr/bin/ccache /usr/local/bin/gcc diff --git a/tools/dockerfile/test/php_jessie_x64/Dockerfile b/tools/dockerfile/test/php_jessie_x64/Dockerfile index 17ea36b76c..c6f3dde39a 100644 --- a/tools/dockerfile/test/php_jessie_x64/Dockerfile +++ b/tools/dockerfile/test/php_jessie_x64/Dockerfile @@ -76,7 +76,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.0.0a2 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 #================= # PHP dependencies diff --git a/tools/dockerfile/test/python_jessie_x64/Dockerfile b/tools/dockerfile/test/python_jessie_x64/Dockerfile index 10a88916ad..94c17078d3 100644 --- a/tools/dockerfile/test/python_jessie_x64/Dockerfile +++ b/tools/dockerfile/test/python_jessie_x64/Dockerfile @@ -76,7 +76,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.0.0a2 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 # Prepare ccache RUN ln -s /usr/bin/ccache /usr/local/bin/gcc diff --git a/tools/dockerfile/test/python_pyenv_x64/Dockerfile b/tools/dockerfile/test/python_pyenv_x64/Dockerfile index ecd785a86d..435a9fdc97 100644 --- a/tools/dockerfile/test/python_pyenv_x64/Dockerfile +++ b/tools/dockerfile/test/python_pyenv_x64/Dockerfile @@ -76,7 +76,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.0.0a2 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 # Install dependencies for pyenv RUN apt-get update && apt-get install -y \ @@ -92,6 +92,9 @@ RUN apt-get update && apt-get install -y \ # Install Pyenv and dev Python versions 3.5 and 3.6 RUN curl -L https://raw.githubusercontent.com/yyuu/pyenv-installer/master/bin/pyenv-installer | bash +ENV PATH /root/.pyenv/bin:$PATH +RUN eval "$(pyenv init -)" +RUN eval "$(pyenv virtualenv-init -)" RUN pyenv update RUN pyenv install 3.5-dev RUN pyenv install 3.6-dev diff --git a/tools/dockerfile/test/ruby_jessie_x64/Dockerfile b/tools/dockerfile/test/ruby_jessie_x64/Dockerfile index dae64e5c8c..679c8ff47a 100644 --- a/tools/dockerfile/test/ruby_jessie_x64/Dockerfile +++ b/tools/dockerfile/test/ruby_jessie_x64/Dockerfile @@ -76,7 +76,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.0.0a2 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 #================== # Ruby dependencies diff --git a/tools/dockerfile/test/sanity/Dockerfile b/tools/dockerfile/test/sanity/Dockerfile index 811384fda1..0da2a1914a 100644 --- a/tools/dockerfile/test/sanity/Dockerfile +++ b/tools/dockerfile/test/sanity/Dockerfile @@ -76,7 +76,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.0.0a2 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 #======================== # Sanity test dependencies diff --git a/tools/doxygen/Doxyfile.c++ b/tools/doxygen/Doxyfile.c++ index 551ba46b3b..965259746c 100644 --- a/tools/doxygen/Doxyfile.c++ +++ b/tools/doxygen/Doxyfile.c++ @@ -779,12 +779,14 @@ doc/fail_fast.md \ doc/g_stands_for.md \ doc/health-checking.md \ doc/http-grpc-status-mapping.md \ +doc/internationalization.md \ doc/interop-test-descriptions.md \ doc/load-balancing.md \ doc/naming.md \ doc/negative-http2-interop-test-descriptions.md \ doc/server-reflection.md \ doc/server_reflection_tutorial.md \ +doc/server_side_auth.md \ doc/service_config.md \ doc/statuscodes.md \ doc/stress_test_framework.md \ diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal index 0d7ddfc6d6..d98475fb00 100644 --- a/tools/doxygen/Doxyfile.c++.internal +++ b/tools/doxygen/Doxyfile.c++.internal @@ -779,12 +779,14 @@ doc/fail_fast.md \ doc/g_stands_for.md \ doc/health-checking.md \ doc/http-grpc-status-mapping.md \ +doc/internationalization.md \ doc/interop-test-descriptions.md \ doc/load-balancing.md \ doc/naming.md \ doc/negative-http2-interop-test-descriptions.md \ doc/server-reflection.md \ doc/server_reflection_tutorial.md \ +doc/server_side_auth.md \ doc/service_config.md \ doc/statuscodes.md \ doc/stress_test_framework.md \ diff --git a/tools/doxygen/Doxyfile.core b/tools/doxygen/Doxyfile.core index 896ee0213b..478bdb2d04 100644 --- a/tools/doxygen/Doxyfile.core +++ b/tools/doxygen/Doxyfile.core @@ -778,12 +778,14 @@ doc/fail_fast.md \ doc/g_stands_for.md \ doc/health-checking.md \ doc/http-grpc-status-mapping.md \ +doc/internationalization.md \ doc/interop-test-descriptions.md \ doc/load-balancing.md \ doc/naming.md \ doc/negative-http2-interop-test-descriptions.md \ doc/server-reflection.md \ doc/server_reflection_tutorial.md \ +doc/server_side_auth.md \ doc/service_config.md \ doc/statuscodes.md \ doc/stress_test_framework.md \ diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index 10801254ef..a7d6aaeb9e 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -778,12 +778,14 @@ doc/fail_fast.md \ doc/g_stands_for.md \ doc/health-checking.md \ doc/http-grpc-status-mapping.md \ +doc/internationalization.md \ doc/interop-test-descriptions.md \ doc/load-balancing.md \ doc/naming.md \ doc/negative-http2-interop-test-descriptions.md \ doc/server-reflection.md \ doc/server_reflection_tutorial.md \ +doc/server_side_auth.md \ doc/service_config.md \ doc/statuscodes.md \ doc/stress_test_framework.md \ @@ -1249,6 +1251,7 @@ src/core/lib/support/mpscq.c \ src/core/lib/support/mpscq.h \ src/core/lib/support/murmur_hash.c \ src/core/lib/support/murmur_hash.h \ +src/core/lib/support/spinlock.h \ src/core/lib/support/stack_lockfree.c \ src/core/lib/support/stack_lockfree.h \ src/core/lib/support/string.c \ diff --git a/tools/gce/linux_performance_worker_init.sh b/tools/gce/linux_performance_worker_init.sh index ab29e015e0..3380f3de3e 100755 --- a/tools/gce/linux_performance_worker_init.sh +++ b/tools/gce/linux_performance_worker_init.sh @@ -142,7 +142,7 @@ gem install bundler # Significant performance improvements with grpc-go have been observed after # upgrading from go 1.5 to a later version, so a later go version is preferred. # Following go install instructions from https://golang.org/doc/install -GO_VERSION=1.7.1 +GO_VERSION=1.8 OS=linux ARCH=amd64 curl -O https://storage.googleapis.com/golang/go${GO_VERSION}.${OS}-${ARCH}.tar.gz diff --git a/tools/internal_ci/windows/grpc_master.bat b/tools/internal_ci/windows/grpc_master.bat index 4041c50313..8943390a8d 100644 --- a/tools/internal_ci/windows/grpc_master.bat +++ b/tools/internal_ci/windows/grpc_master.bat @@ -27,18 +27,17 @@ @rem (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE @rem OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -setlocal +@rem make sure msys binaries are preferred over cygwin binaries +@rem set path to python 2.7 +set PATH=C:\tools\msys64\usr\bin;C:\Python27;%PATH% @rem enter repo root cd /d %~dp0\..\..\.. git submodule update --init -sh tools\run_tests\helper_scripts\run_tests_in_workspace.sh -t -j 4 -x c_windows_dbg_sponge_log.xml --report_suite_name c_windows_dbg -l c -c dbg -sh tools\run_tests\helper_scripts\run_tests_in_workspace.sh -t -j 4 -x c_windows_opt_sponge_log.xml --report_suite_name c_windows_opt -l c -c opt -sh tools\run_tests\helper_scripts\run_tests_in_workspace.sh -t -j 4 -x csharp_windows_dbg_sponge_log.xml --report_suite_name csharp_windows_dbg -l csharp -c dbg -sh tools\run_tests\helper_scripts\run_tests_in_workspace.sh -t -j 4 -x csharp_windows_opt_sponge_log.xml --report_suite_name csharp_windows_opt -l csharp -c opt -sh tools\run_tests\helper_scripts\run_tests_in_workspace.sh -t -j 4 -x node_windows_dbg_sponge_log.xml --report_suite_name node_windows_dbg -l node -c dbg -sh tools\run_tests\helper_scripts\run_tests_in_workspace.sh -t -j 4 -x node_windows_opt_sponge_log.xml --report_suite_name node_windows_opt -l node -c opt -sh tools\run_tests\helper_scripts\run_tests_in_workspace.sh -t -j 4 -x python_windows_dbg_sponge_log.xml --report_suite_name python_windows_dbg -l python -c dbg -sh tools\run_tests\helper_scripts\run_tests_in_workspace.sh -t -j 4 -x python_windows_opt_sponge_log.xml --report_suite_name python_windows_opt -l python -c opt +python tools/run_tests/run_tests_matrix.py -f basictests windows -j 1 --inner_jobs 8 || goto :error +goto :EOF + +:error +exit /b %errorlevel% diff --git a/tools/jenkins/run_full_performance_released.sh b/tools/jenkins/run_full_performance_released.sh new file mode 100755 index 0000000000..ae2289e32c --- /dev/null +++ b/tools/jenkins/run_full_performance_released.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +# Copyright 2017, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# A frozen version of run_full_performance.sh that runs full performance test +# suite for the latest released stable version of gRPC. +set -ex + +# Enter the gRPC repo root +cd $(dirname $0)/../.. + +# run 8core client vs 8core server +tools/run_tests/run_performance_tests.py \ + -l c++ csharp node ruby java python go node_express \ + --netperf \ + --category scalable \ + --bq_result_table performance_released.performance_experiment \ + --remote_worker_host grpc-performance-server-8core grpc-performance-client-8core grpc-performance-client2-8core \ + --xml_report report_8core.xml \ + || EXIT_CODE=1 + +# prevent pushing leftover build files to remote hosts in the next step. +git clean -fdxq --exclude='report*.xml' + +# scalability with 32cores (and upload to a different BQ table) +tools/run_tests/run_performance_tests.py \ + -l c++ java csharp go \ + --netperf \ + --category scalable \ + --bq_result_table performance_released.performance_experiment_32core \ + --remote_worker_host grpc-performance-server-32core grpc-performance-client-32core grpc-performance-client2-32core \ + --xml_report report_32core.xml \ + || EXIT_CODE=1 + +# prevent pushing leftover build files to remote hosts in the next step. +git clean -fdxq --exclude='report*.xml' + +# selected scenarios on Windows +tools/run_tests/run_performance_tests.py \ + -l csharp \ + --category scalable \ + --bq_result_table performance_released.performance_experiment_windows \ + --remote_worker_host grpc-performance-windows1 grpc-performance-windows2 \ + --xml_report report_windows.xml \ + || EXIT_CODE=1 + +exit $EXIT_CODE diff --git a/tools/profiling/microbenchmarks/bm2bq.py b/tools/profiling/microbenchmarks/bm2bq.py index 280f217e69..a83b3be89c 100755 --- a/tools/profiling/microbenchmarks/bm2bq.py +++ b/tools/profiling/microbenchmarks/bm2bq.py @@ -80,6 +80,12 @@ if sys.argv[1] == '--schema': with open(sys.argv[1]) as f: js = json.loads(f.read()) +if len(sys.argv) > 2: + with open(sys.argv[2]) as f: + js2 = json.loads(f.read()) +else: + js2 = None + writer = csv.DictWriter(sys.stdout, [c for c,t in columns]) bm_specs = { @@ -143,6 +149,10 @@ bm_specs = { 'tpl': ['fixture'], 'dyn': [], }, + 'BM_CallCreateDestroy' : { + 'tpl': ['fixture'], + 'dyn': [], + }, } def numericalize(s): @@ -215,4 +225,10 @@ for bm in js['benchmarks']: row.update(labels) if 'label' in row: del row['label'] + if js2: + for bm2 in js2['benchmarks']: + if bm['name'] == bm2['name']: + row['cpu_time'] = bm2['cpu_time'] + row['real_time'] = bm2['real_time'] + row['iterations'] = bm2['iterations'] writer.writerow(row) diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json index 1172eaf353..22cf618c22 100644 --- a/tools/run_tests/generated/sources_and_headers.json +++ b/tools/run_tests/generated/sources_and_headers.json @@ -739,6 +739,21 @@ "headers": [], "is_filegroup": false, "language": "c", + "name": "gpr_spinlock_test", + "src": [ + "test/core/support/spinlock_test.c" + ], + "third_party": false, + "type": "target" + }, + { + "deps": [ + "gpr", + "gpr_test_util" + ], + "headers": [], + "is_filegroup": false, + "language": "c", "name": "gpr_stack_lockfree_test", "src": [ "test/core/support/stack_lockfree_test.c" @@ -1652,6 +1667,23 @@ "headers": [], "is_filegroup": false, "language": "c", + "name": "parse_address_test", + "src": [ + "test/core/client_channel/parse_address_test.c" + ], + "third_party": false, + "type": "target" + }, + { + "deps": [ + "gpr", + "gpr_test_util", + "grpc", + "grpc_test_util" + ], + "headers": [], + "is_filegroup": false, + "language": "c", "name": "percent_decode_fuzzer", "src": [ "test/core/slice/percent_decode_fuzzer.c" @@ -2027,6 +2059,23 @@ "headers": [], "is_filegroup": false, "language": "c", + "name": "stream_owned_slice_test", + "src": [ + "test/core/transport/stream_owned_slice_test.c" + ], + "third_party": false, + "type": "target" + }, + { + "deps": [ + "gpr", + "gpr_test_util", + "grpc", + "grpc_test_util" + ], + "headers": [], + "is_filegroup": false, + "language": "c", "name": "tcp_client_posix_test", "src": [ "test/core/iomgr/tcp_client_posix_test.c" @@ -2371,6 +2420,7 @@ "grpc", "grpc++", "grpc++_test_util", + "grpc_benchmark", "grpc_test_util" ], "headers": [], @@ -2391,6 +2441,7 @@ "grpc", "grpc++", "grpc++_test_util", + "grpc_benchmark", "grpc_test_util" ], "headers": [], @@ -2411,6 +2462,7 @@ "grpc", "grpc++", "grpc++_test_util", + "grpc_benchmark", "grpc_test_util" ], "headers": [], @@ -2431,6 +2483,7 @@ "grpc", "grpc++", "grpc++_test_util", + "grpc_benchmark", "grpc_test_util" ], "headers": [], @@ -2451,6 +2504,7 @@ "grpc", "grpc++", "grpc++_test_util", + "grpc_benchmark", "grpc_test_util" ], "headers": [], @@ -2471,14 +2525,15 @@ "grpc", "grpc++", "grpc++_test_util", + "grpc_benchmark", "grpc_test_util" ], "headers": [], "is_filegroup": false, "language": "c++", - "name": "bm_fullstack", + "name": "bm_fullstack_streaming_ping_pong", "src": [ - "test/cpp/microbenchmarks/bm_fullstack.cc" + "test/cpp/microbenchmarks/bm_fullstack_streaming_ping_pong.cc" ], "third_party": false, "type": "target" @@ -2489,6 +2544,72 @@ "gpr", "gpr_test_util", "grpc", + "grpc++", + "grpc++_test_util", + "grpc_benchmark", + "grpc_test_util" + ], + "headers": [], + "is_filegroup": false, + "language": "c++", + "name": "bm_fullstack_streaming_pump", + "src": [ + "test/cpp/microbenchmarks/bm_fullstack_streaming_pump.cc" + ], + "third_party": false, + "type": "target" + }, + { + "deps": [ + "benchmark", + "gpr", + "gpr_test_util", + "grpc", + "grpc++", + "grpc++_test_util", + "grpc_benchmark", + "grpc_test_util" + ], + "headers": [], + "is_filegroup": false, + "language": "c++", + "name": "bm_fullstack_trickle", + "src": [ + "test/cpp/microbenchmarks/bm_fullstack_trickle.cc" + ], + "third_party": false, + "type": "target" + }, + { + "deps": [ + "benchmark", + "gpr", + "gpr_test_util", + "grpc", + "grpc++", + "grpc++_test_util", + "grpc_benchmark", + "grpc_test_util" + ], + "headers": [], + "is_filegroup": false, + "language": "c++", + "name": "bm_fullstack_unary_ping_pong", + "src": [ + "test/cpp/microbenchmarks/bm_fullstack_unary_ping_pong.cc" + ], + "third_party": false, + "type": "target" + }, + { + "deps": [ + "benchmark", + "gpr", + "gpr_test_util", + "grpc", + "grpc++", + "grpc++_test_util", + "grpc_benchmark", "grpc_test_util" ], "headers": [], @@ -5694,6 +5815,30 @@ }, { "deps": [ + "benchmark", + "grpc", + "grpc++", + "grpc_test_util" + ], + "headers": [ + "test/cpp/microbenchmarks/fullstack_context_mutators.h", + "test/cpp/microbenchmarks/fullstack_fixtures.h", + "test/cpp/microbenchmarks/helpers.h" + ], + "is_filegroup": false, + "language": "c++", + "name": "grpc_benchmark", + "src": [ + "test/cpp/microbenchmarks/fullstack_context_mutators.h", + "test/cpp/microbenchmarks/fullstack_fixtures.h", + "test/cpp/microbenchmarks/helpers.cc", + "test/cpp/microbenchmarks/helpers.h" + ], + "third_party": false, + "type": "lib" + }, + { + "deps": [ "grpc++", "grpc++_config_proto", "grpc++_proto_reflection_desc_db", @@ -7062,6 +7207,7 @@ "src/core/lib/support/env.h", "src/core/lib/support/mpscq.h", "src/core/lib/support/murmur_hash.h", + "src/core/lib/support/spinlock.h", "src/core/lib/support/stack_lockfree.h", "src/core/lib/support/string.h", "src/core/lib/support/string_windows.h", @@ -7127,6 +7273,7 @@ "src/core/lib/support/mpscq.h", "src/core/lib/support/murmur_hash.c", "src/core/lib/support/murmur_hash.h", + "src/core/lib/support/spinlock.h", "src/core/lib/support/stack_lockfree.c", "src/core/lib/support/stack_lockfree.h", "src/core/lib/support/string.c", @@ -7895,7 +8042,7 @@ "headers": [ "test/core/end2end/cq_verifier.h", "test/core/end2end/fake_resolver.h", - "test/core/end2end/fixtures/http_proxy.h", + "test/core/end2end/fixtures/http_proxy_fixture.h", "test/core/end2end/fixtures/proxy.h", "test/core/iomgr/endpoint_tests.h", "test/core/util/debugger_macros.h", @@ -7917,8 +8064,8 @@ "test/core/end2end/cq_verifier.h", "test/core/end2end/fake_resolver.c", "test/core/end2end/fake_resolver.h", - "test/core/end2end/fixtures/http_proxy.c", - "test/core/end2end/fixtures/http_proxy.h", + "test/core/end2end/fixtures/http_proxy_fixture.c", + "test/core/end2end/fixtures/http_proxy_fixture.h", "test/core/end2end/fixtures/proxy.c", "test/core/end2end/fixtures/proxy.h", "test/core/iomgr/endpoint_tests.c", @@ -7935,12 +8082,10 @@ "test/core/util/parse_hexstring.h", "test/core/util/passthru_endpoint.c", "test/core/util/passthru_endpoint.h", + "test/core/util/port.c", "test/core/util/port.h", - "test/core/util/port_posix.c", "test/core/util/port_server_client.c", "test/core/util/port_server_client.h", - "test/core/util/port_uv.c", - "test/core/util/port_windows.c", "test/core/util/slice_splitter.c", "test/core/util/slice_splitter.h", "test/core/util/trickle_endpoint.c", diff --git a/tools/run_tests/generated/tests.json b/tools/run_tests/generated/tests.json index 7a644f4a98..402dabc554 100644 --- a/tools/run_tests/generated/tests.json +++ b/tools/run_tests/generated/tests.json @@ -845,6 +845,28 @@ "posix", "windows" ], + "cpu_cost": 10, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "gtest": false, + "language": "c", + "name": "gpr_spinlock_test", + "platforms": [ + "linux", + "mac", + "posix", + "windows" + ] + }, + { + "args": [], + "ci_platforms": [ + "linux", + "mac", + "posix", + "windows" + ], "cpu_cost": 7, "exclude_configs": [], "exclude_iomgrs": [], @@ -1711,6 +1733,28 @@ "flaky": false, "gtest": false, "language": "c", + "name": "parse_address_test", + "platforms": [ + "linux", + "mac", + "posix", + "windows" + ] + }, + { + "args": [], + "ci_platforms": [ + "linux", + "mac", + "posix", + "windows" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "gtest": false, + "language": "c", "name": "percent_encoding_test", "platforms": [ "linux", @@ -2098,6 +2142,28 @@ "ci_platforms": [ "linux", "mac", + "posix", + "windows" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "gtest": false, + "language": "c", + "name": "stream_owned_slice_test", + "platforms": [ + "linux", + "mac", + "posix", + "windows" + ] + }, + { + "args": [], + "ci_platforms": [ + "linux", + "mac", "posix" ], "cpu_cost": 0.5, @@ -2646,7 +2712,88 @@ "flaky": false, "gtest": false, "language": "c++", - "name": "bm_fullstack", + "name": "bm_fullstack_streaming_ping_pong", + "platforms": [ + "linux", + "mac", + "posix" + ], + "timeout_seconds": 1200 + }, + { + "args": [ + "--benchmark_min_time=0" + ], + "ci_platforms": [ + "linux", + "mac", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [], + "excluded_poll_engines": [ + "poll", + "poll-cv" + ], + "flaky": false, + "gtest": false, + "language": "c++", + "name": "bm_fullstack_streaming_pump", + "platforms": [ + "linux", + "mac", + "posix" + ], + "timeout_seconds": 1200 + }, + { + "args": [ + "--benchmark_min_time=0" + ], + "ci_platforms": [ + "linux", + "mac", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [], + "excluded_poll_engines": [ + "poll", + "poll-cv" + ], + "flaky": false, + "gtest": false, + "language": "c++", + "name": "bm_fullstack_trickle", + "platforms": [ + "linux", + "mac", + "posix" + ], + "timeout_seconds": 1200 + }, + { + "args": [ + "--benchmark_min_time=0" + ], + "ci_platforms": [ + "linux", + "mac", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [], + "excluded_poll_engines": [ + "poll", + "poll-cv" + ], + "flaky": false, + "gtest": false, + "language": "c++", + "name": "bm_fullstack_unary_ping_pong", "platforms": [ "linux", "mac", @@ -2983,7 +3130,9 @@ ] }, { - "args": [], + "args": [ + "--generated_file_path=gens/src/proto/grpc/testing/compiler_test.grpc.pb.h" + ], "ci_platforms": [ "linux", "mac", @@ -80516,6 +80665,50 @@ }, { "args": [ + "test/core/end2end/fuzzers/api_fuzzer_corpus/clusterfuzz-testcase-5834320218423296" + ], + "ci_platforms": [ + "linux" + ], + "cpu_cost": 0.1, + "exclude_configs": [ + "tsan" + ], + "exclude_iomgrs": [ + "uv" + ], + "flaky": false, + "language": "c", + "name": "api_fuzzer_one_entry", + "platforms": [ + "linux" + ], + "uses_polling": false + }, + { + "args": [ + "test/core/end2end/fuzzers/api_fuzzer_corpus/clusterfuzz-testcase-6520142139752448" + ], + "ci_platforms": [ + "linux" + ], + "cpu_cost": 0.1, + "exclude_configs": [ + "tsan" + ], + "exclude_iomgrs": [ + "uv" + ], + "flaky": false, + "language": "c", + "name": "api_fuzzer_one_entry", + "platforms": [ + "linux" + ], + "uses_polling": false + }, + { + "args": [ "test/core/end2end/fuzzers/api_fuzzer_corpus/crash-0597bbdd657fa4ed14443994c9147a1a7bbc205f" ], "ci_platforms": [ diff --git a/tools/run_tests/helper_scripts/pre_build_csharp.bat b/tools/run_tests/helper_scripts/pre_build_csharp.bat index f37f63b584..99df1c6626 100644 --- a/tools/run_tests/helper_scripts/pre_build_csharp.bat +++ b/tools/run_tests/helper_scripts/pre_build_csharp.bat @@ -89,18 +89,6 @@ if exist %NUGET% ( %NUGET% restore -PackagesDirectory ../packages || goto :error cd .. - cd Grpc.IntegrationTesting.Client || goto :error - %NUGET% restore -PackagesDirectory ../packages || goto :error - cd .. - - cd Grpc.IntegrationTesting.QpsWorker || goto :error - %NUGET% restore -PackagesDirectory ../packages || goto :error - cd .. - - cd Grpc.IntegrationTesting.StressClient || goto :error - %NUGET% restore -PackagesDirectory ../packages || goto :error - cd .. - cd Grpc.IntegrationTesting || goto :error %NUGET% restore -PackagesDirectory ../packages || goto :error diff --git a/tools/run_tests/helper_scripts/pre_build_csharp.sh b/tools/run_tests/helper_scripts/pre_build_csharp.sh index 1f808556f4..d7665e15af 100755 --- a/tools/run_tests/helper_scripts/pre_build_csharp.sh +++ b/tools/run_tests/helper_scripts/pre_build_csharp.sh @@ -73,18 +73,6 @@ then nuget restore -PackagesDirectory ../packages cd .. - cd Grpc.IntegrationTesting.Client - nuget restore -PackagesDirectory ../packages - cd .. - - cd Grpc.IntegrationTesting.QpsWorker - nuget restore -PackagesDirectory ../packages - cd .. - - cd Grpc.IntegrationTesting.StressClient - nuget restore -PackagesDirectory ../packages - cd .. - cd Grpc.IntegrationTesting nuget restore -PackagesDirectory ../packages cd .. diff --git a/tools/run_tests/performance/scenario_config.py b/tools/run_tests/performance/scenario_config.py index 865125fcd7..830b8c0849 100644 --- a/tools/run_tests/performance/scenario_config.py +++ b/tools/run_tests/performance/scenario_config.py @@ -413,11 +413,12 @@ class NodeLanguage: client_type='ASYNC_CLIENT', server_type='async_server', client_language='c++') - yield _ping_pong_scenario( - 'node_protobuf_async_unary_qps_unconstrained', rpc_type='UNARY', - client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', - unconstrained_client='async', - categories=[SCALABLE, SMOKETEST]) + # TODO(murgatroid99): fix bugs with this scenario and re-enable it + # yield _ping_pong_scenario( + # 'node_protobuf_async_unary_qps_unconstrained', rpc_type='UNARY', + # client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', + # unconstrained_client='async', + # categories=[SCALABLE, SMOKETEST]) # TODO(jtattermusch): make this scenario work #yield _ping_pong_scenario( diff --git a/tools/run_tests/python_utils/jobset.py b/tools/run_tests/python_utils/jobset.py index 7b2c62d1a2..f3047431e2 100755 --- a/tools/run_tests/python_utils/jobset.py +++ b/tools/run_tests/python_utils/jobset.py @@ -31,6 +31,7 @@ from __future__ import print_function +import logging import multiprocessing import os import platform @@ -128,6 +129,8 @@ _TAG_COLOR = { 'SKIPPED': 'cyan' } +_FORMAT = '%(asctime)-15s %(message)s' +logging.basicConfig(level=logging.INFO, format=_FORMAT) def message(tag, msg, explanatory_text=None, do_newline=False): if message.old_tag == tag and message.old_msg == msg and not explanatory_text: @@ -137,8 +140,8 @@ def message(tag, msg, explanatory_text=None, do_newline=False): try: if platform_string() == 'windows' or not sys.stdout.isatty(): if explanatory_text: - print(explanatory_text) - print('%s: %s' % (tag, msg)) + logging.info(explanatory_text) + logging.info('%s: %s', tag, msg) else: sys.stdout.write('%s%s%s\x1b[%d;%dm%s\x1b[0m: %s%s' % ( _BEGINNING_OF_LINE, diff --git a/tools/run_tests/python_utils/start_port_server.py b/tools/run_tests/python_utils/start_port_server.py index 4c9f6aac63..deb7354438 100644 --- a/tools/run_tests/python_utils/start_port_server.py +++ b/tools/run_tests/python_utils/start_port_server.py @@ -40,7 +40,10 @@ import tempfile import time -def start_port_server(port_server_port): +# must be synchronized with test/core/utils/port_server_client.h +_PORT_SERVER_PORT = 32766 + +def start_port_server(): # check if a compatible port server is running # if incompatible (version mismatch) ==> start a new one # if not running ==> start a new one @@ -48,7 +51,7 @@ def start_port_server(port_server_port): try: version = int( urllib.request.urlopen( - 'http://localhost:%d/version_number' % port_server_port, + 'http://localhost:%d/version_number' % _PORT_SERVER_PORT, timeout=10).read()) logging.info('detected port server running version %d', version) running = True @@ -67,7 +70,7 @@ def start_port_server(port_server_port): if not running: logging.info('port_server version mismatch: killing the old one') urllib.request.urlopen('http://localhost:%d/quitquitquit' % - port_server_port).read() + _PORT_SERVER_PORT).read() time.sleep(1) if not running: fd, logfile = tempfile.mkstemp() @@ -76,7 +79,7 @@ def start_port_server(port_server_port): args = [ sys.executable, os.path.abspath('tools/run_tests/python_utils/port_server.py'), - '-p', '%d' % port_server_port, '-l', logfile + '-p', '%d' % _PORT_SERVER_PORT, '-l', logfile ] env = dict(os.environ) env['BUILD_ID'] = 'pleaseDontKillMeJenkins' @@ -107,7 +110,7 @@ def start_port_server(port_server_port): time.sleep(1) try: urllib.request.urlopen( - 'http://localhost:%d/get' % port_server_port, + 'http://localhost:%d/get' % _PORT_SERVER_PORT, timeout=1).read() logging.info( 'last ditch attempt to contact port server succeeded') @@ -119,7 +122,7 @@ def start_port_server(port_server_port): print(port_log) sys.exit(1) try: - port_server_url = 'http://localhost:%d/get' % port_server_port + port_server_url = 'http://localhost:%d/get' % _PORT_SERVER_PORT urllib.request.urlopen(port_server_url, timeout=1).read() logging.info('port server is up and ready') break diff --git a/tools/run_tests/run_interop_tests.py b/tools/run_tests/run_interop_tests.py index b47dc1e8f2..0d5bec1d67 100755 --- a/tools/run_tests/run_interop_tests.py +++ b/tools/run_tests/run_interop_tests.py @@ -77,11 +77,15 @@ class CXXLanguage: def __init__(self): self.client_cwd = None self.server_cwd = None + self.http2_cwd = None self.safename = 'cxx' def client_cmd(self, args): return ['bins/opt/interop_client'] + args + def client_cmd_http2interop(self, args): + return ['bins/opt/http2_client'] + args + def cloud_to_prod_env(self): return {} @@ -164,6 +168,7 @@ class JavaLanguage: def __init__(self): self.client_cwd = '../grpc-java' self.server_cwd = '../grpc-java' + self.http2_cwd = '../grpc-java' self.safename = str(self) def client_cmd(self, args): @@ -197,11 +202,15 @@ class GoLanguage: # TODO: this relies on running inside docker self.client_cwd = '/go/src/google.golang.org/grpc/interop/client' self.server_cwd = '/go/src/google.golang.org/grpc/interop/server' + self.http2_cwd = '/go/src/google.golang.org/grpc/interop/http2' self.safename = str(self) def client_cmd(self, args): return ['go', 'run', 'client.go'] + args + def client_cmd_http2interop(self, args): + return ['go', 'run', 'negative_http2_client.go'] + args + def cloud_to_prod_env(self): return {} @@ -393,6 +402,7 @@ class PythonLanguage: def __init__(self): self.client_cwd = None self.server_cwd = None + self.http2_cwd = None self.safename = str(self) def client_cmd(self, args): @@ -468,8 +478,7 @@ _HTTP2_TEST_CASES = ['tls', 'framing'] _HTTP2_BADSERVER_TEST_CASES = ['rst_after_header', 'rst_after_data', 'rst_during_data', 'goaway', 'ping', 'max_streams'] -# TODO: Add python once the tests are fixed. -_LANGUAGES_FOR_HTTP2_BADSERVER_TESTS = ['java'] +_LANGUAGES_FOR_HTTP2_BADSERVER_TESTS = ['java', 'go', 'python', 'c++'] DOCKER_WORKDIR_ROOT = '/var/local/git/grpc' @@ -605,11 +614,12 @@ def cloud_to_cloud_jobspec(language, test_case, server_name, server_host, client_options = common_options + ['--server_port=%s' % (int(server_port)+offset)] cmdline = bash_cmdline(language.client_cmd_http2interop(client_options)) + cwd = language.http2_cwd else: client_options = interop_only_options + common_options + ['--server_port=%s' % server_port] cmdline = bash_cmdline(language.client_cmd(client_options)) + cwd = language.client_cwd - cwd = language.client_cwd environ = language.global_env() if docker_image: container_name = dockerjob.random_name('interop_client_%s' % language.safename) diff --git a/tools/run_tests/run_microbenchmark.py b/tools/run_tests/run_microbenchmark.py index 4307906a7e..4fb1d5fc7c 100755 --- a/tools/run_tests/run_microbenchmark.py +++ b/tools/run_tests/run_microbenchmark.py @@ -44,8 +44,7 @@ os.chdir(os.path.join(os.path.dirname(sys.argv[0]), '../..')) if not os.path.exists('reports'): os.makedirs('reports') -port_server_port = 32766 -start_port_server.start_port_server(port_server_port) +start_port_server.start_port_server() def fnize(s): out = '' @@ -110,8 +109,7 @@ def collect_latency(bm_name, args): if len(benchmarks) >= min(16, multiprocessing.cpu_count()): # run up to half the cpu count: each benchmark can use up to two cores # (one for the microbenchmark, one for the data flush) - jobset.run(benchmarks, maxjobs=max(1, multiprocessing.cpu_count()/2), - add_env={'GRPC_TEST_PORT_SERVER': 'localhost:%d' % port_server_port}) + jobset.run(benchmarks, maxjobs=max(1, multiprocessing.cpu_count()/2)) jobset.run(profile_analysis, maxjobs=multiprocessing.cpu_count()) jobset.run(cleanup, maxjobs=multiprocessing.cpu_count()) benchmarks = [] @@ -119,8 +117,7 @@ def collect_latency(bm_name, args): cleanup = [] # run the remaining benchmarks that weren't flushed if len(benchmarks): - jobset.run(benchmarks, maxjobs=max(1, multiprocessing.cpu_count()/2), - add_env={'GRPC_TEST_PORT_SERVER': 'localhost:%d' % port_server_port}) + jobset.run(benchmarks, maxjobs=max(1, multiprocessing.cpu_count()/2)) jobset.run(profile_analysis, maxjobs=multiprocessing.cpu_count()) jobset.run(cleanup, maxjobs=multiprocessing.cpu_count()) @@ -156,8 +153,7 @@ def collect_perf(bm_name, args): if len(benchmarks) >= 20: # run up to half the cpu count: each benchmark can use up to two cores # (one for the microbenchmark, one for the data flush) - jobset.run(benchmarks, maxjobs=1, - add_env={'GRPC_TEST_PORT_SERVER': 'localhost:%d' % port_server_port}) + jobset.run(benchmarks, maxjobs=1) jobset.run(profile_analysis, maxjobs=multiprocessing.cpu_count()) jobset.run(cleanup, maxjobs=multiprocessing.cpu_count()) benchmarks = [] @@ -165,25 +161,29 @@ def collect_perf(bm_name, args): cleanup = [] # run the remaining benchmarks that weren't flushed if len(benchmarks): - jobset.run(benchmarks, maxjobs=1, - add_env={'GRPC_TEST_PORT_SERVER': 'localhost:%d' % port_server_port}) + jobset.run(benchmarks, maxjobs=1) jobset.run(profile_analysis, maxjobs=multiprocessing.cpu_count()) jobset.run(cleanup, maxjobs=multiprocessing.cpu_count()) -def collect_summary(bm_name, args): - heading('Summary: %s' % bm_name) +def run_summary(bm_name, cfg, base_json_name): subprocess.check_call( ['make', bm_name, - 'CONFIG=counters', '-j', '%d' % multiprocessing.cpu_count()]) - cmd = ['bins/counters/%s' % bm_name, - '--benchmark_out=out.json', + 'CONFIG=%s' % cfg, '-j', '%d' % multiprocessing.cpu_count()]) + cmd = ['bins/%s/%s' % (cfg, bm_name), + '--benchmark_out=%s.%s.json' % (base_json_name, cfg), '--benchmark_out_format=json'] if args.summary_time is not None: cmd += ['--benchmark_min_time=%d' % args.summary_time] - text(subprocess.check_output(cmd)) + return subprocess.check_output(cmd) + +def collect_summary(bm_name, args): + heading('Summary: %s [no counters]' % bm_name) + text(run_summary(bm_name, 'opt', 'out')) + heading('Summary: %s [with counters]' % bm_name) + text(run_summary(bm_name, 'counters', 'out')) if args.bigquery_upload: with open('out.csv', 'w') as f: - f.write(subprocess.check_output(['tools/profiling/microbenchmarks/bm2bq.py', 'out.json'])) + f.write(subprocess.check_output(['tools/profiling/microbenchmarks/bm2bq.py', 'out.counters.json', 'out.opt.json'])) subprocess.check_call(['bq', 'load', 'microbenchmarks.microbenchmarks', 'out.csv']) collectors = { @@ -195,20 +195,28 @@ collectors = { argp = argparse.ArgumentParser(description='Collect data from microbenchmarks') argp.add_argument('-c', '--collect', choices=sorted(collectors.keys()), - nargs='+', + nargs='*', default=sorted(collectors.keys()), help='Which collectors should be run against each benchmark') argp.add_argument('-b', '--benchmarks', - default=['bm_fullstack', + default=['bm_fullstack_unary_ping_pong', + 'bm_fullstack_streaming_ping_pong', + 'bm_fullstack_streaming_pump', 'bm_closure', 'bm_cq', 'bm_call_create', 'bm_error', 'bm_chttp2_hpack', - 'bm_metadata'], + 'bm_metadata', + 'bm_fullstack_trickle', + ], nargs='+', type=str, help='Which microbenchmarks should be run') +argp.add_argument('--diff_perf', + default=None, + type=str, + help='Diff microbenchmarks against this git revision') argp.add_argument('--bigquery_upload', default=False, action='store_const', @@ -223,6 +231,26 @@ args = argp.parse_args() for bm_name in args.benchmarks: for collect in args.collect: collectors[collect](bm_name, args) +if args.diff_perf: + for bm_name in args.benchmarks: + run_summary(bm_name, 'opt', '%s.new' % bm_name) + where_am_i = subprocess.check_output(['git', 'rev-parse', '--abbrev-ref', 'HEAD']).strip() + subprocess.check_call(['git', 'checkout', args.diff_perf]) + comparables = [] + subprocess.check_call(['make', 'clean']) + try: + for bm_name in args.benchmarks: + try: + run_summary(bm_name, 'opt', '%s.old' % bm_name) + comparables.append(bm_name) + except subprocess.CalledProcessError, e: + pass + finally: + subprocess.check_call(['git', 'checkout', where_am_i]) + for bm_name in comparables: + subprocess.check_call(['third_party/benchmark/tools/compare_bench.py', + '%s.new.opt.json' % bm_name, + '%s.old.opt.json' % bm_name]) index_html += "</body>\n</html>\n" with open('reports/index.html', 'w') as f: diff --git a/tools/run_tests/run_performance_tests.py b/tools/run_tests/run_performance_tests.py index 7c04d228ba..ee4102c591 100755 --- a/tools/run_tests/run_performance_tests.py +++ b/tools/run_tests/run_performance_tests.py @@ -101,7 +101,7 @@ def create_qpsworker_job(language, shortname=None, port=10000, remote_host=None, user_at_host = '%s@%s' % (_REMOTE_HOST_USERNAME, remote_host) ssh_cmd = ['ssh'] cmdline = ['timeout', '%s' % (worker_timeout + 30)] + cmdline - ssh_cmd.extend([str(user_at_host), 'cd ~/performance_workspace/grpc/ && %s' % ' '.join(cmdline)]) + ssh_cmd.extend([str(user_at_host), 'cd ~/performance_workspace/grpc/ && tools/run_tests/start_port_server.py && %s' % ' '.join(cmdline)]) cmdline = ssh_cmd jobspec = jobset.JobSpec( diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index 9e9af59c25..43b8f8184e 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -307,9 +307,9 @@ class CLanguage(object): assert base is not None assert line[1] == ' ' test = base + line.strip() - cmdline = [binary] + ['--gtest_filter=%s' % test] + cmdline = [binary, '--gtest_filter=%s' % test] + target['args'] out.append(self.config.job_spec(cmdline, - shortname='%s --gtest_filter=%s %s' % (binary, test, shortname_ext), + shortname='%s %s' % (' '.join(cmdline), shortname_ext), cpu_cost=cpu_cost, timeout_seconds=_DEFAULT_TIMEOUT_SECONDS * timeout_scaling, environ=env)) @@ -692,18 +692,11 @@ class RubyLanguage(object): _check_compiler(self.args.compiler, ['default']) def test_specs(self): - #TODO(apolcyn) turn mac ruby tests back on once ruby 2.4 issues done - if platform_string() == 'mac': - print('skipping ruby test_specs on mac until running on 2.4') - return [] return [self.config.job_spec(['tools/run_tests/helper_scripts/run_ruby.sh'], timeout_seconds=10*60, environ=_FORCE_ENVIRON_FOR_WRAPPERS)] def pre_build_steps(self): - if platform_string() == 'mac': - print('skipping ruby pre_build_steps on mac until running on 2.4') - return [] return [['tools/run_tests/helper_scripts/pre_build_ruby.sh']] def make_targets(self): @@ -713,15 +706,9 @@ class RubyLanguage(object): return [] def build_steps(self): - if platform_string() == 'mac': - print('skipping ruby build_steps on mac until running on 2.4') - return [] return [['tools/run_tests/helper_scripts/build_ruby.sh']] def post_tests_steps(self): - if platform_string() == 'mac': - print('skipping ruby post_test_steps on mac until running on 2.4') - return [] return [['tools/run_tests/helper_scripts/post_tests_ruby.sh']] def makefile_name(self): @@ -1311,7 +1298,9 @@ if args.use_docker: if not args.travis: env['TTY_FLAG'] = '-t' # enables Ctrl-C when not on Jenkins. - run_shell_command('tools/run_tests/dockerize/build_docker_and_run_tests.sh', env=env) + subprocess.check_call('tools/run_tests/dockerize/build_docker_and_run_tests.sh', + shell=True, + env=env) sys.exit(0) _check_arch_option(args.arch) @@ -1451,8 +1440,7 @@ def _build_and_run( # start antagonists antagonists = [subprocess.Popen(['tools/run_tests/python_utils/antagonist.py']) for _ in range(0, args.antagonists)] - port_server_port = 32766 - start_port_server.start_port_server(port_server_port) + start_port_server.start_port_server() resultset = None num_test_failures = 0 try: @@ -1479,10 +1467,9 @@ def _build_and_run( sample_size = int(num_jobs * args.sample_percent/100.0) massaged_one_run = random.sample(massaged_one_run, sample_size) if not isclose(args.sample_percent, 100.0): + assert args.runs_per_test == 1, "Can't do sampling (-p) over multiple runs (-n)." print("Running %d tests out of %d (~%d%%)" % (sample_size, num_jobs, args.sample_percent)) - else: - assert args.runs_per_test == 1, "Can't do sampling (-p) over multiple runs (-n)." if infinite_runs: assert len(massaged_one_run) > 0, 'Must have at least one test for a -n inf run' runs_sequence = (itertools.repeat(massaged_one_run) if infinite_runs @@ -1495,7 +1482,6 @@ def _build_and_run( all_runs, check_cancelled, newline_on_success=newline_on_success, travis=args.travis, maxjobs=args.jobs, stop_on_failure=args.stop_on_failure, - add_env={'GRPC_TEST_PORT_SERVER': 'localhost:%d' % port_server_port}, quiet_success=args.quiet_success) if resultset: for k, v in sorted(resultset.items()): diff --git a/tools/run_tests/run_tests_matrix.py b/tools/run_tests/run_tests_matrix.py index bc4fdaba71..47422451f8 100755 --- a/tools/run_tests/run_tests_matrix.py +++ b/tools/run_tests/run_tests_matrix.py @@ -49,6 +49,9 @@ _RUNTESTS_TIMEOUT = 4*60*60 # Number of jobs assigned to each run_tests.py instance _DEFAULT_INNER_JOBS = 2 +# report suffix is important for reports to get picked up by internal CI +_REPORT_SUFFIX = 'sponge_log.xml' + def _docker_jobspec(name, runtests_args=[], inner_jobs=_DEFAULT_INNER_JOBS): """Run a single instance of run_tests.py in a docker container""" @@ -57,7 +60,7 @@ def _docker_jobspec(name, runtests_args=[], inner_jobs=_DEFAULT_INNER_JOBS): '--use_docker', '-t', '-j', str(inner_jobs), - '-x', 'report_%s.xml' % name, + '-x', 'report_%s_%s' % (name, _REPORT_SUFFIX), '--report_suite_name', '%s' % name] + runtests_args, shortname='run_tests_%s' % name, timeout_seconds=_RUNTESTS_TIMEOUT) @@ -70,10 +73,11 @@ def _workspace_jobspec(name, runtests_args=[], workspace_name=None, inner_jobs=_ workspace_name = 'workspace_%s' % name env = {'WORKSPACE_NAME': workspace_name} test_job = jobset.JobSpec( - cmdline=['tools/run_tests/helper_scripts/run_tests_in_workspace.sh', + cmdline=['bash', + 'tools/run_tests/helper_scripts/run_tests_in_workspace.sh', '-t', '-j', str(inner_jobs), - '-x', '../report_%s.xml' % name, + '-x', '../report_%s_%s' % (name, _REPORT_SUFFIX), '--report_suite_name', '%s' % name] + runtests_args, environ=env, shortname='run_tests_%s' % name, @@ -414,7 +418,7 @@ if __name__ == "__main__": skipped_results = jobset.run(skipped_jobs, skip_jobs=True) resultset.update(skipped_results) - report_utils.render_junit_xml_report(resultset, 'report.xml', + report_utils.render_junit_xml_report(resultset, 'report_%s' % _REPORT_SUFFIX, suite_name='aggregate_tests') if num_failures == 0: diff --git a/tools/run_tests/sanity/check_submodules.sh b/tools/run_tests/sanity/check_submodules.sh index cfe4e2731c..3c5ba16b93 100755 --- a/tools/run_tests/sanity/check_submodules.sh +++ b/tools/run_tests/sanity/check_submodules.sh @@ -46,7 +46,7 @@ cat << EOF | awk '{ print $1 }' | sort > $want_submodules 886e7d75368e3f4fab3f4d0d3584e4abfc557755 third_party/boringssl-with-bazel (version_for_cocoapods_7.0-857-g886e7d7) 30dbc81fb5ffdc98ea9b14b1918bfe4e8779b26e third_party/gflags (v2.2.0) c99458533a9b4c743ed51537e25989ea55944908 third_party/googletest (release-1.7.0) - a428e42072765993ff674fda72863c9f1aa2d268 third_party/protobuf (v3.1.0-alpha-1) + 593e917c176b5bc5aafa57bf9f6030d749d91cd5 third_party/protobuf (v3.1.0-alpha-1-326-g593e917) bcad91771b7f0bff28a1cac1981d7ef2b9bcef3c third_party/thrift (bcad917) 50893291621658f355bc5b4d450a8d06a563053d third_party/zlib (v1.2.8) EOF diff --git a/tools/dockerfile/bazel/Dockerfile b/tools/run_tests/start_port_server.py index 2a80a4d4d5..e33ac12bd3 100644..100755 --- a/tools/dockerfile/bazel/Dockerfile +++ b/tools/run_tests/start_port_server.py @@ -1,4 +1,6 @@ -# Copyright 2015, Google Inc. +#!/usr/bin/env python + +# Copyright 2017, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -27,26 +29,18 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -FROM ubuntu:wily -RUN apt-get update -RUN apt-get -y install software-properties-common python-software-properties -RUN add-apt-repository ppa:webupd8team/java -RUN apt-get update -RUN apt-get -y install \ - vim \ - wget \ - openjdk-8-jdk \ - pkg-config \ - zip \ - g++ \ - zlib1g-dev \ - unzip \ - git +""" +Wrapper around port server starting code. + +Used by developers who wish to run individual C/C++ tests outside of the +run_tests.py infrastructure. + +The path to this file is called out in test/core/util/port.c, and printed as +an error message to users. +""" -RUN git clone https://github.com/bazelbuild/bazel.git /bazel -RUN cd /bazel && ./compile.sh +import python_utils.start_port_server as start_port_server -RUN ln -s /bazel/output/bazel /bin/ +start_port_server.start_port_server() -# ensure the installation has been extracted -RUN bazel +print "Port server started successfully" |