diff options
author | Yash Tibrewal <yashkt@google.com> | 2018-08-13 10:53:06 -0700 |
---|---|---|
committer | Yash Tibrewal <yashkt@google.com> | 2018-08-13 10:53:06 -0700 |
commit | 556775d7c78125e051c6571d4c528de214024c23 (patch) | |
tree | 77d3e0effdb58b01e9e257a8dcd03a024acedf31 /tools | |
parent | 0ec6973b743e8e1f463bdcc8691e1869097c92f0 (diff) | |
parent | ba7ca9742eb7484009b75c268dba7e56b8feaebb (diff) |
Merge master
Diffstat (limited to 'tools')
57 files changed, 1211 insertions, 169 deletions
diff --git a/tools/codegen/core/gen_nano_proto.sh b/tools/codegen/core/gen_nano_proto.sh index 5c781761ba..1549591746 100755 --- a/tools/codegen/core/gen_nano_proto.sh +++ b/tools/codegen/core/gen_nano_proto.sh @@ -68,7 +68,7 @@ popd # this should be the same version as the submodule we compile against # ideally we'd update this as a template to ensure that -pip install protobuf==3.5.2 +pip install protobuf==3.6.0 pushd "$(dirname $INPUT_PROTO)" > /dev/null diff --git a/tools/distrib/check_copyright.py b/tools/distrib/check_copyright.py index e6388251cf..77f602ad1f 100755 --- a/tools/distrib/check_copyright.py +++ b/tools/distrib/check_copyright.py @@ -100,6 +100,10 @@ _EXEMPT = frozenset(( # Gradle wrappers used to build for Android 'examples/android/helloworld/gradlew.bat', 'src/android/test/interop/gradlew.bat', + + # Designer-generated source + 'examples/csharp/HelloworldXamarin/Droid/Resources/Resource.designer.cs', + 'examples/csharp/HelloworldXamarin/iOS/ViewController.designer.cs', )) RE_YEAR = r'Copyright (?P<first_year>[0-9]+\-)?(?P<last_year>[0-9]+) ([Tt]he )?gRPC [Aa]uthors(\.|)' diff --git a/tools/distrib/python/grpcio_tools/grpc_version.py b/tools/distrib/python/grpcio_tools/grpc_version.py index ae249e7386..ccb69a8ebc 100644 --- a/tools/distrib/python/grpcio_tools/grpc_version.py +++ b/tools/distrib/python/grpcio_tools/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/tools/distrib/python/grpcio_tools/grpc_version.py.template`!!! -VERSION = '1.14.0.dev0' +VERSION = '1.15.0.dev0' diff --git a/tools/distrib/python/grpcio_tools/protoc_lib_deps.py b/tools/distrib/python/grpcio_tools/protoc_lib_deps.py index a0e1419ac1..7d10db0329 100644 --- a/tools/distrib/python/grpcio_tools/protoc_lib_deps.py +++ b/tools/distrib/python/grpcio_tools/protoc_lib_deps.py @@ -14,10 +14,10 @@ # limitations under the License. # AUTO-GENERATED BY make_grpcio_tools.py! -CC_FILES=['google/protobuf/compiler/zip_writer.cc', 'google/protobuf/compiler/subprocess.cc', 'google/protobuf/compiler/ruby/ruby_generator.cc', 'google/protobuf/compiler/python/python_generator.cc', 'google/protobuf/compiler/plugin.pb.cc', 'google/protobuf/compiler/plugin.cc', 'google/protobuf/compiler/php/php_generator.cc', 'google/protobuf/compiler/objectivec/objectivec_primitive_field.cc', 'google/protobuf/compiler/objectivec/objectivec_oneof.cc', 'google/protobuf/compiler/objectivec/objectivec_message_field.cc', 'google/protobuf/compiler/objectivec/objectivec_message.cc', 'google/protobuf/compiler/objectivec/objectivec_map_field.cc', 'google/protobuf/compiler/objectivec/objectivec_helpers.cc', 'google/protobuf/compiler/objectivec/objectivec_generator.cc', 'google/protobuf/compiler/objectivec/objectivec_file.cc', 'google/protobuf/compiler/objectivec/objectivec_field.cc', 'google/protobuf/compiler/objectivec/objectivec_extension.cc', 'google/protobuf/compiler/objectivec/objectivec_enum_field.cc', 'google/protobuf/compiler/objectivec/objectivec_enum.cc', 'google/protobuf/compiler/js/well_known_types_embed.cc', 'google/protobuf/compiler/js/js_generator.cc', 'google/protobuf/compiler/javanano/javanano_primitive_field.cc', 'google/protobuf/compiler/javanano/javanano_message_field.cc', 'google/protobuf/compiler/javanano/javanano_message.cc', 'google/protobuf/compiler/javanano/javanano_map_field.cc', 'google/protobuf/compiler/javanano/javanano_helpers.cc', 'google/protobuf/compiler/javanano/javanano_generator.cc', 'google/protobuf/compiler/javanano/javanano_file.cc', 'google/protobuf/compiler/javanano/javanano_field.cc', 'google/protobuf/compiler/javanano/javanano_extension.cc', 'google/protobuf/compiler/javanano/javanano_enum_field.cc', 'google/protobuf/compiler/javanano/javanano_enum.cc', 'google/protobuf/compiler/java/java_string_field_lite.cc', 'google/protobuf/compiler/java/java_string_field.cc', 'google/protobuf/compiler/java/java_shared_code_generator.cc', 'google/protobuf/compiler/java/java_service.cc', 'google/protobuf/compiler/java/java_primitive_field_lite.cc', 'google/protobuf/compiler/java/java_primitive_field.cc', 'google/protobuf/compiler/java/java_name_resolver.cc', 'google/protobuf/compiler/java/java_message_lite.cc', 'google/protobuf/compiler/java/java_message_field_lite.cc', 'google/protobuf/compiler/java/java_message_field.cc', 'google/protobuf/compiler/java/java_message_builder_lite.cc', 'google/protobuf/compiler/java/java_message_builder.cc', 'google/protobuf/compiler/java/java_message.cc', 'google/protobuf/compiler/java/java_map_field_lite.cc', 'google/protobuf/compiler/java/java_map_field.cc', 'google/protobuf/compiler/java/java_lazy_message_field_lite.cc', 'google/protobuf/compiler/java/java_lazy_message_field.cc', 'google/protobuf/compiler/java/java_helpers.cc', 'google/protobuf/compiler/java/java_generator_factory.cc', 'google/protobuf/compiler/java/java_generator.cc', 'google/protobuf/compiler/java/java_file.cc', 'google/protobuf/compiler/java/java_field.cc', 'google/protobuf/compiler/java/java_extension_lite.cc', 'google/protobuf/compiler/java/java_extension.cc', 'google/protobuf/compiler/java/java_enum_lite.cc', 'google/protobuf/compiler/java/java_enum_field_lite.cc', 'google/protobuf/compiler/java/java_enum_field.cc', 'google/protobuf/compiler/java/java_enum.cc', 'google/protobuf/compiler/java/java_doc_comment.cc', 'google/protobuf/compiler/java/java_context.cc', 'google/protobuf/compiler/csharp/csharp_wrapper_field.cc', 'google/protobuf/compiler/csharp/csharp_source_generator_base.cc', 'google/protobuf/compiler/csharp/csharp_repeated_primitive_field.cc', 'google/protobuf/compiler/csharp/csharp_repeated_message_field.cc', 'google/protobuf/compiler/csharp/csharp_repeated_enum_field.cc', 'google/protobuf/compiler/csharp/csharp_reflection_class.cc', 'google/protobuf/compiler/csharp/csharp_primitive_field.cc', 'google/protobuf/compiler/csharp/csharp_message_field.cc', 'google/protobuf/compiler/csharp/csharp_message.cc', 'google/protobuf/compiler/csharp/csharp_map_field.cc', 'google/protobuf/compiler/csharp/csharp_helpers.cc', 'google/protobuf/compiler/csharp/csharp_generator.cc', 'google/protobuf/compiler/csharp/csharp_field_base.cc', 'google/protobuf/compiler/csharp/csharp_enum_field.cc', 'google/protobuf/compiler/csharp/csharp_enum.cc', 'google/protobuf/compiler/csharp/csharp_doc_comment.cc', 'google/protobuf/compiler/cpp/cpp_string_field.cc', 'google/protobuf/compiler/cpp/cpp_service.cc', 'google/protobuf/compiler/cpp/cpp_primitive_field.cc', 'google/protobuf/compiler/cpp/cpp_padding_optimizer.cc', 'google/protobuf/compiler/cpp/cpp_message_field.cc', 'google/protobuf/compiler/cpp/cpp_message.cc', 'google/protobuf/compiler/cpp/cpp_map_field.cc', 'google/protobuf/compiler/cpp/cpp_helpers.cc', 'google/protobuf/compiler/cpp/cpp_generator.cc', 'google/protobuf/compiler/cpp/cpp_file.cc', 'google/protobuf/compiler/cpp/cpp_field.cc', 'google/protobuf/compiler/cpp/cpp_extension.cc', 'google/protobuf/compiler/cpp/cpp_enum_field.cc', 'google/protobuf/compiler/cpp/cpp_enum.cc', 'google/protobuf/compiler/command_line_interface.cc', 'google/protobuf/compiler/code_generator.cc', 'google/protobuf/wrappers.pb.cc', 'google/protobuf/wire_format.cc', 'google/protobuf/util/type_resolver_util.cc', 'google/protobuf/util/time_util.cc', 'google/protobuf/util/message_differencer.cc', 'google/protobuf/util/json_util.cc', 'google/protobuf/util/internal/utility.cc', 'google/protobuf/util/internal/type_info_test_helper.cc', 'google/protobuf/util/internal/type_info.cc', 'google/protobuf/util/internal/protostream_objectwriter.cc', 'google/protobuf/util/internal/protostream_objectsource.cc', 'google/protobuf/util/internal/proto_writer.cc', 'google/protobuf/util/internal/object_writer.cc', 'google/protobuf/util/internal/json_stream_parser.cc', 'google/protobuf/util/internal/json_objectwriter.cc', 'google/protobuf/util/internal/json_escaping.cc', 'google/protobuf/util/internal/field_mask_utility.cc', 'google/protobuf/util/internal/error_listener.cc', 'google/protobuf/util/internal/default_value_objectwriter.cc', 'google/protobuf/util/internal/datapiece.cc', 'google/protobuf/util/field_mask_util.cc', 'google/protobuf/util/field_comparator.cc', 'google/protobuf/util/delimited_message_util.cc', 'google/protobuf/unknown_field_set.cc', 'google/protobuf/type.pb.cc', 'google/protobuf/timestamp.pb.cc', 'google/protobuf/text_format.cc', 'google/protobuf/stubs/substitute.cc', 'google/protobuf/stubs/mathlimits.cc', 'google/protobuf/struct.pb.cc', 'google/protobuf/source_context.pb.cc', 'google/protobuf/service.cc', 'google/protobuf/reflection_ops.cc', 'google/protobuf/message.cc', 'google/protobuf/map_field.cc', 'google/protobuf/io/zero_copy_stream_impl.cc', 'google/protobuf/io/tokenizer.cc', 'google/protobuf/io/strtod.cc', 'google/protobuf/io/printer.cc', 'google/protobuf/io/gzip_stream.cc', 'google/protobuf/generated_message_table_driven.cc', 'google/protobuf/generated_message_reflection.cc', 'google/protobuf/field_mask.pb.cc', 'google/protobuf/extension_set_heavy.cc', 'google/protobuf/empty.pb.cc', 'google/protobuf/dynamic_message.cc', 'google/protobuf/duration.pb.cc', 'google/protobuf/descriptor_database.cc', 'google/protobuf/descriptor.pb.cc', 'google/protobuf/descriptor.cc', 'google/protobuf/compiler/parser.cc', 'google/protobuf/compiler/importer.cc', 'google/protobuf/api.pb.cc', 'google/protobuf/any.pb.cc', 'google/protobuf/any.cc', 'google/protobuf/wire_format_lite.cc', 'google/protobuf/stubs/time.cc', 'google/protobuf/stubs/strutil.cc', 'google/protobuf/stubs/structurally_valid.cc', 'google/protobuf/stubs/stringprintf.cc', 'google/protobuf/stubs/stringpiece.cc', 'google/protobuf/stubs/statusor.cc', 'google/protobuf/stubs/status.cc', 'google/protobuf/stubs/once.cc', 'google/protobuf/stubs/io_win32.cc', 'google/protobuf/stubs/int128.cc', 'google/protobuf/stubs/common.cc', 'google/protobuf/stubs/bytestream.cc', 'google/protobuf/stubs/atomicops_internals_x86_msvc.cc', 'google/protobuf/stubs/atomicops_internals_x86_gcc.cc', 'google/protobuf/repeated_field.cc', 'google/protobuf/message_lite.cc', 'google/protobuf/io/zero_copy_stream_impl_lite.cc', 'google/protobuf/io/zero_copy_stream.cc', 'google/protobuf/io/coded_stream.cc', 'google/protobuf/generated_message_util.cc', 'google/protobuf/generated_message_table_driven_lite.cc', 'google/protobuf/extension_set.cc', 'google/protobuf/arenastring.cc', 'google/protobuf/arena.cc', 'google/protobuf/compiler/js/embed.cc'] +CC_FILES=['google/protobuf/compiler/zip_writer.cc', 'google/protobuf/compiler/subprocess.cc', 'google/protobuf/compiler/ruby/ruby_generator.cc', 'google/protobuf/compiler/python/python_generator.cc', 'google/protobuf/compiler/plugin.pb.cc', 'google/protobuf/compiler/plugin.cc', 'google/protobuf/compiler/php/php_generator.cc', 'google/protobuf/compiler/objectivec/objectivec_primitive_field.cc', 'google/protobuf/compiler/objectivec/objectivec_oneof.cc', 'google/protobuf/compiler/objectivec/objectivec_message_field.cc', 'google/protobuf/compiler/objectivec/objectivec_message.cc', 'google/protobuf/compiler/objectivec/objectivec_map_field.cc', 'google/protobuf/compiler/objectivec/objectivec_helpers.cc', 'google/protobuf/compiler/objectivec/objectivec_generator.cc', 'google/protobuf/compiler/objectivec/objectivec_file.cc', 'google/protobuf/compiler/objectivec/objectivec_field.cc', 'google/protobuf/compiler/objectivec/objectivec_extension.cc', 'google/protobuf/compiler/objectivec/objectivec_enum_field.cc', 'google/protobuf/compiler/objectivec/objectivec_enum.cc', 'google/protobuf/compiler/js/well_known_types_embed.cc', 'google/protobuf/compiler/js/js_generator.cc', 'google/protobuf/compiler/java/java_string_field_lite.cc', 'google/protobuf/compiler/java/java_string_field.cc', 'google/protobuf/compiler/java/java_shared_code_generator.cc', 'google/protobuf/compiler/java/java_service.cc', 'google/protobuf/compiler/java/java_primitive_field_lite.cc', 'google/protobuf/compiler/java/java_primitive_field.cc', 'google/protobuf/compiler/java/java_name_resolver.cc', 'google/protobuf/compiler/java/java_message_lite.cc', 'google/protobuf/compiler/java/java_message_field_lite.cc', 'google/protobuf/compiler/java/java_message_field.cc', 'google/protobuf/compiler/java/java_message_builder_lite.cc', 'google/protobuf/compiler/java/java_message_builder.cc', 'google/protobuf/compiler/java/java_message.cc', 'google/protobuf/compiler/java/java_map_field_lite.cc', 'google/protobuf/compiler/java/java_map_field.cc', 'google/protobuf/compiler/java/java_lazy_message_field_lite.cc', 'google/protobuf/compiler/java/java_lazy_message_field.cc', 'google/protobuf/compiler/java/java_helpers.cc', 'google/protobuf/compiler/java/java_generator_factory.cc', 'google/protobuf/compiler/java/java_generator.cc', 'google/protobuf/compiler/java/java_file.cc', 'google/protobuf/compiler/java/java_field.cc', 'google/protobuf/compiler/java/java_extension_lite.cc', 'google/protobuf/compiler/java/java_extension.cc', 'google/protobuf/compiler/java/java_enum_lite.cc', 'google/protobuf/compiler/java/java_enum_field_lite.cc', 'google/protobuf/compiler/java/java_enum_field.cc', 'google/protobuf/compiler/java/java_enum.cc', 'google/protobuf/compiler/java/java_doc_comment.cc', 'google/protobuf/compiler/java/java_context.cc', 'google/protobuf/compiler/csharp/csharp_wrapper_field.cc', 'google/protobuf/compiler/csharp/csharp_source_generator_base.cc', 'google/protobuf/compiler/csharp/csharp_repeated_primitive_field.cc', 'google/protobuf/compiler/csharp/csharp_repeated_message_field.cc', 'google/protobuf/compiler/csharp/csharp_repeated_enum_field.cc', 'google/protobuf/compiler/csharp/csharp_reflection_class.cc', 'google/protobuf/compiler/csharp/csharp_primitive_field.cc', 'google/protobuf/compiler/csharp/csharp_message_field.cc', 'google/protobuf/compiler/csharp/csharp_message.cc', 'google/protobuf/compiler/csharp/csharp_map_field.cc', 'google/protobuf/compiler/csharp/csharp_helpers.cc', 'google/protobuf/compiler/csharp/csharp_generator.cc', 'google/protobuf/compiler/csharp/csharp_field_base.cc', 'google/protobuf/compiler/csharp/csharp_enum_field.cc', 'google/protobuf/compiler/csharp/csharp_enum.cc', 'google/protobuf/compiler/csharp/csharp_doc_comment.cc', 'google/protobuf/compiler/cpp/cpp_string_field.cc', 'google/protobuf/compiler/cpp/cpp_service.cc', 'google/protobuf/compiler/cpp/cpp_primitive_field.cc', 'google/protobuf/compiler/cpp/cpp_padding_optimizer.cc', 'google/protobuf/compiler/cpp/cpp_message_field.cc', 'google/protobuf/compiler/cpp/cpp_message.cc', 'google/protobuf/compiler/cpp/cpp_map_field.cc', 'google/protobuf/compiler/cpp/cpp_helpers.cc', 'google/protobuf/compiler/cpp/cpp_generator.cc', 'google/protobuf/compiler/cpp/cpp_file.cc', 'google/protobuf/compiler/cpp/cpp_field.cc', 'google/protobuf/compiler/cpp/cpp_extension.cc', 'google/protobuf/compiler/cpp/cpp_enum_field.cc', 'google/protobuf/compiler/cpp/cpp_enum.cc', 'google/protobuf/compiler/command_line_interface.cc', 'google/protobuf/compiler/code_generator.cc', 'google/protobuf/wrappers.pb.cc', 'google/protobuf/wire_format.cc', 'google/protobuf/util/type_resolver_util.cc', 'google/protobuf/util/time_util.cc', 'google/protobuf/util/message_differencer.cc', 'google/protobuf/util/json_util.cc', 'google/protobuf/util/internal/utility.cc', 'google/protobuf/util/internal/type_info_test_helper.cc', 'google/protobuf/util/internal/type_info.cc', 'google/protobuf/util/internal/protostream_objectwriter.cc', 'google/protobuf/util/internal/protostream_objectsource.cc', 'google/protobuf/util/internal/proto_writer.cc', 'google/protobuf/util/internal/object_writer.cc', 'google/protobuf/util/internal/json_stream_parser.cc', 'google/protobuf/util/internal/json_objectwriter.cc', 'google/protobuf/util/internal/json_escaping.cc', 'google/protobuf/util/internal/field_mask_utility.cc', 'google/protobuf/util/internal/error_listener.cc', 'google/protobuf/util/internal/default_value_objectwriter.cc', 'google/protobuf/util/internal/datapiece.cc', 'google/protobuf/util/field_mask_util.cc', 'google/protobuf/util/field_comparator.cc', 'google/protobuf/util/delimited_message_util.cc', 'google/protobuf/unknown_field_set.cc', 'google/protobuf/type.pb.cc', 'google/protobuf/timestamp.pb.cc', 'google/protobuf/text_format.cc', 'google/protobuf/stubs/substitute.cc', 'google/protobuf/stubs/mathlimits.cc', 'google/protobuf/struct.pb.cc', 'google/protobuf/source_context.pb.cc', 'google/protobuf/service.cc', 'google/protobuf/reflection_ops.cc', 'google/protobuf/message.cc', 'google/protobuf/map_field.cc', 'google/protobuf/io/zero_copy_stream_impl.cc', 'google/protobuf/io/tokenizer.cc', 'google/protobuf/io/strtod.cc', 'google/protobuf/io/printer.cc', 'google/protobuf/io/gzip_stream.cc', 'google/protobuf/generated_message_table_driven.cc', 'google/protobuf/generated_message_reflection.cc', 'google/protobuf/field_mask.pb.cc', 'google/protobuf/extension_set_heavy.cc', 'google/protobuf/empty.pb.cc', 'google/protobuf/dynamic_message.cc', 'google/protobuf/duration.pb.cc', 'google/protobuf/descriptor_database.cc', 'google/protobuf/descriptor.pb.cc', 'google/protobuf/descriptor.cc', 'google/protobuf/compiler/parser.cc', 'google/protobuf/compiler/importer.cc', 'google/protobuf/api.pb.cc', 'google/protobuf/any.pb.cc', 'google/protobuf/any.cc', 'google/protobuf/wire_format_lite.cc', 'google/protobuf/stubs/time.cc', 'google/protobuf/stubs/strutil.cc', 'google/protobuf/stubs/structurally_valid.cc', 'google/protobuf/stubs/stringprintf.cc', 'google/protobuf/stubs/stringpiece.cc', 'google/protobuf/stubs/statusor.cc', 'google/protobuf/stubs/status.cc', 'google/protobuf/stubs/io_win32.cc', 'google/protobuf/stubs/int128.cc', 'google/protobuf/stubs/common.cc', 'google/protobuf/stubs/bytestream.cc', 'google/protobuf/repeated_field.cc', 'google/protobuf/message_lite.cc', 'google/protobuf/io/zero_copy_stream_impl_lite.cc', 'google/protobuf/io/zero_copy_stream.cc', 'google/protobuf/io/coded_stream.cc', 'google/protobuf/implicit_weak_message.cc', 'google/protobuf/generated_message_util.cc', 'google/protobuf/generated_message_table_driven_lite.cc', 'google/protobuf/extension_set.cc', 'google/protobuf/arenastring.cc', 'google/protobuf/arena.cc'] PROTO_FILES=['google/protobuf/wrappers.proto', 'google/protobuf/type.proto', 'google/protobuf/timestamp.proto', 'google/protobuf/struct.proto', 'google/protobuf/source_context.proto', 'google/protobuf/field_mask.proto', 'google/protobuf/empty.proto', 'google/protobuf/duration.proto', 'google/protobuf/descriptor.proto', 'google/protobuf/compiler/plugin.proto', 'google/protobuf/api.proto', 'google/protobuf/any.proto'] CC_INCLUDE='third_party/protobuf/src' PROTO_INCLUDE='third_party/protobuf/src' -PROTOBUF_SUBMODULE_VERSION="b5fbb742af122b565925987e65c08957739976a7" +PROTOBUF_SUBMODULE_VERSION="48cb18e5c419ddd23d9badcfe4e9df7bde1979b2" diff --git a/tools/dockerfile/OWNERS b/tools/dockerfile/OWNERS index db4ab546a6..5f0ad58d61 100644 --- a/tools/dockerfile/OWNERS +++ b/tools/dockerfile/OWNERS @@ -7,5 +7,5 @@ set noparent # for kokoro to be able to access the pre-built images. @jtattermusch -@matt-kwong +@mehrdada @nicolasnoble diff --git a/tools/dockerfile/distribtest/csharp_centos7_x64/Dockerfile b/tools/dockerfile/distribtest/csharp_centos7_x64/Dockerfile index 3e1faafdc0..e32b3cb5e2 100644 --- a/tools/dockerfile/distribtest/csharp_centos7_x64/Dockerfile +++ b/tools/dockerfile/distribtest/csharp_centos7_x64/Dockerfile @@ -22,3 +22,6 @@ RUN yum install -y mono-devel RUN yum install -y nuget RUN yum install -y unzip + +# Make sure the mono certificate store is up-to-date to prevent issues with nuget restore +RUN curl https://curl.haxx.se/ca/cacert.pem > ~/cacert.pem && cert-sync ~/cacert.pem && rm -f ~/cacert.pem diff --git a/tools/dockerfile/distribtest/csharp_jessie_x64/Dockerfile b/tools/dockerfile/distribtest/csharp_jessie_x64/Dockerfile index 03fb7a5343..e95d781dfd 100644 --- a/tools/dockerfile/distribtest/csharp_jessie_x64/Dockerfile +++ b/tools/dockerfile/distribtest/csharp_jessie_x64/Dockerfile @@ -25,3 +25,7 @@ RUN apt-get update && apt-get install -y \ && apt-get clean RUN apt-get update && apt-get install -y unzip && apt-get clean + +# Make sure the mono certificate store is up-to-date to prevent issues with nuget restore +RUN apt-get update && apt-get install -y curl && apt-get clean +RUN curl https://curl.haxx.se/ca/cacert.pem > ~/cacert.pem && cert-sync ~/cacert.pem && rm -f ~/cacert.pem diff --git a/tools/dockerfile/distribtest/csharp_jessie_x86/Dockerfile b/tools/dockerfile/distribtest/csharp_jessie_x86/Dockerfile index f2fa61a691..aec936a5b8 100644 --- a/tools/dockerfile/distribtest/csharp_jessie_x86/Dockerfile +++ b/tools/dockerfile/distribtest/csharp_jessie_x86/Dockerfile @@ -25,3 +25,7 @@ RUN apt-get update && apt-get install -y \ && apt-get clean RUN apt-get update && apt-get install -y unzip && apt-get clean + +# Make sure the mono certificate store is up-to-date to prevent issues with nuget restore +RUN apt-get update && apt-get install -y curl && apt-get clean +RUN curl https://curl.haxx.se/ca/cacert.pem > ~/cacert.pem && cert-sync ~/cacert.pem && rm -f ~/cacert.pem diff --git a/tools/dockerfile/distribtest/csharp_ubuntu1404_x64/Dockerfile b/tools/dockerfile/distribtest/csharp_ubuntu1404_x64/Dockerfile index 3edc31e170..61ca1a08a4 100644 --- a/tools/dockerfile/distribtest/csharp_ubuntu1404_x64/Dockerfile +++ b/tools/dockerfile/distribtest/csharp_ubuntu1404_x64/Dockerfile @@ -38,3 +38,7 @@ RUN mkdir warmup \ && dotnet new \ && cd .. \ && rm -rf warmup + +# Make sure the mono certificate store is up-to-date to prevent issues with nuget restore +RUN apt-get update && apt-get install -y curl && apt-get clean +RUN curl https://curl.haxx.se/ca/cacert.pem > ~/cacert.pem && cert-sync ~/cacert.pem && rm -f ~/cacert.pem diff --git a/tools/dockerfile/distribtest/csharp_ubuntu1604_x64/Dockerfile b/tools/dockerfile/distribtest/csharp_ubuntu1604_x64/Dockerfile index 1a58f9784b..93ee75cfcd 100644 --- a/tools/dockerfile/distribtest/csharp_ubuntu1604_x64/Dockerfile +++ b/tools/dockerfile/distribtest/csharp_ubuntu1604_x64/Dockerfile @@ -25,3 +25,7 @@ RUN apt-get update && apt-get install -y \ && apt-get clean RUN apt-get update && apt-get install -y unzip && apt-get clean + +# Make sure the mono certificate store is up-to-date to prevent issues with nuget restore +RUN apt-get update && apt-get install -y curl && apt-get clean +RUN curl https://curl.haxx.se/ca/cacert.pem > ~/cacert.pem && cert-sync ~/cacert.pem && rm -f ~/cacert.pem diff --git a/tools/dockerfile/grpc_artifact_android_ndk/Dockerfile b/tools/dockerfile/grpc_artifact_android_ndk/Dockerfile index 77b6acfb16..be96f131f9 100644 --- a/tools/dockerfile/grpc_artifact_android_ndk/Dockerfile +++ b/tools/dockerfile/grpc_artifact_android_ndk/Dockerfile @@ -12,9 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Docker file for building gRPC artifacts. +# Docker file for building gRPC artifacts for Android. -# Recent enough cmake (>=3.9) needed by Android SDK FROM debian:sid RUN apt-get update && apt-get install -y debian-keyring && apt-key update @@ -47,20 +46,26 @@ RUN apt-get update && apt-key update && apt-get install -y \ wget \ zip && apt-get clean -# Cmake for cross-compilation -RUN apt-get update && apt-get install -y cmake golang && apt-get clean +# golang needed to build BoringSSL with cmake +RUN apt-get update && apt-get install -y golang && apt-get clean -################## -# Android NDK +# Java required by Android SDK +RUN apt-get update && apt-get -y install openjdk-8-jdk && apt-get clean -# Download and install Android NDK -RUN wget -q https://dl.google.com/android/repository/android-ndk-r16b-linux-x86_64.zip -O android_ndk.zip \ - && unzip -q android_ndk.zip \ - && rm android_ndk.zip \ - && mv ./android-ndk-r16b /opt -ENV ANDROID_NDK_PATH /opt/android-ndk-r16b +# Install Android SDK +ENV ANDROID_SDK_VERSION 4333796 +RUN mkdir -p /opt/android-sdk && cd /opt/android-sdk && \ + wget -q https://dl.google.com/android/repository/sdk-tools-linux-${ANDROID_SDK_VERSION}.zip && \ + unzip -q sdk-tools-linux-${ANDROID_SDK_VERSION}.zip && \ + rm sdk-tools-linux-${ANDROID_SDK_VERSION}.zip +ENV ANDROID_SDK_PATH /opt/android-sdk -RUN apt-get update && apt-get install -y libpthread-stubs0-dev && apt-get clean +# Install Android NDK and cmake using sdkmanager +RUN mkdir -p ~/.android && touch ~/.android/repositories.cfg +RUN yes | ${ANDROID_SDK_PATH}/tools/bin/sdkmanager --licenses # accept all licenses +RUN ${ANDROID_SDK_PATH}/tools/bin/sdkmanager ndk-bundle 'cmake;3.6.4111459' +ENV ANDROID_NDK_PATH ${ANDROID_SDK_PATH}/ndk-bundle +ENV ANDROID_SDK_CMAKE ${ANDROID_SDK_PATH}/cmake/3.6.4111459/bin/cmake RUN mkdir /var/local/jenkins diff --git a/tools/dockerfile/grpc_artifact_protoc/Dockerfile b/tools/dockerfile/grpc_artifact_protoc/Dockerfile index 33e2f2970c..44fde4f99f 100644 --- a/tools/dockerfile/grpc_artifact_protoc/Dockerfile +++ b/tools/dockerfile/grpc_artifact_protoc/Dockerfile @@ -30,24 +30,34 @@ RUN yum install -y git \ glibc-devel \ glibc-devel.i686 -# Install GCC 4.7 to support -static-libstdc++ -RUN wget http://people.centos.org/tru/devtools-1.1/devtools-1.1.repo -P /etc/yum.repos.d -RUN bash -c 'echo "enabled=1" >> /etc/yum.repos.d/devtools-1.1.repo' -RUN bash -c "sed -e 's/\$basearch/i386/g' /etc/yum.repos.d/devtools-1.1.repo > /etc/yum.repos.d/devtools-i386-1.1.repo" -RUN sed -e 's/testing-/testing-i386-/g' -i /etc/yum.repos.d/devtools-i386-1.1.repo +# Install GCC 4.8 +RUN wget http://people.centos.org/tru/devtools-2/devtools-2.repo -P /etc/yum.repos.d +RUN bash -c 'echo "enabled=1" >> /etc/yum.repos.d/devtools-2.repo' +RUN bash -c "sed -e 's/\$basearch/i386/g' /etc/yum.repos.d/devtools-2.repo > /etc/yum.repos.d/devtools-i386-2.repo" +RUN sed -e 's/testing-/testing-i386-/g' -i /etc/yum.repos.d/devtools-i386-2.repo # We'll get and "Rpmdb checksum is invalid: dCDPT(pkg checksums)" error caused by # docker issue when using overlay storage driver, but all the stuff we need # will be installed, so for now we just ignore the error. # https://github.com/docker/docker/issues/10180 -RUN yum install -y devtoolset-1.1 \ - devtoolset-1.1-libstdc++-devel \ - devtoolset-1.1-libstdc++-devel.i686 || true +RUN yum install -y devtoolset-2-build \ + devtoolset-2-toolchain \ + devtoolset-2-binutils \ + devtoolset-2-gcc \ + devtoolset-2-gcc-c++ \ + devtoolset-2-libstdc++-devel \ + devtoolset-2-libstdc++-devel.i686 || true + +# Again, ignore the "Rpmdb checksum is invalid: dCDPT(pkg checksums)" error. +RUN yum install -y ca-certificates || true # renew certs to prevent download error for ius-release.rpm + +# TODO(jtattermusch): gRPC makefile uses "which" to detect the availability of gcc +RUN yum install -y which || true # Update Git to version >1.7 to allow cloning submodules with --reference arg. RUN yum remove -y git && yum clean all RUN yum install -y https://centos6.iuscommunity.org/ius-release.rpm && yum clean all RUN yum install -y git2u && yum clean all -# Start in devtoolset environment that uses GCC 4.7 -CMD ["scl", "enable", "devtoolset-1.1", "bash"] +# Start in devtoolset environment that uses GCC 4.8 +CMD ["scl", "enable", "devtoolset-2", "bash"] diff --git a/tools/doxygen/Doxyfile.c++ b/tools/doxygen/Doxyfile.c++ index 322ab5eb98..688c271fea 100644 --- a/tools/doxygen/Doxyfile.c++ +++ b/tools/doxygen/Doxyfile.c++ @@ -40,7 +40,7 @@ PROJECT_NAME = "GRPC C++" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 1.14.0-dev +PROJECT_NUMBER = 1.15.0-dev # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a @@ -791,6 +791,7 @@ doc/server-reflection.md \ doc/server_reflection_tutorial.md \ doc/server_side_auth.md \ doc/service_config.md \ +doc/ssl-performance.md \ doc/status_ordering.md \ doc/statuscodes.md \ doc/unit_testing.md \ diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal index 7f526b2e2d..ae8ebb1e3c 100644 --- a/tools/doxygen/Doxyfile.c++.internal +++ b/tools/doxygen/Doxyfile.c++.internal @@ -40,7 +40,7 @@ PROJECT_NAME = "GRPC C++" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 1.14.0-dev +PROJECT_NUMBER = 1.15.0-dev # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a @@ -791,6 +791,7 @@ doc/server-reflection.md \ doc/server_reflection_tutorial.md \ doc/server_side_auth.md \ doc/service_config.md \ +doc/ssl-performance.md \ doc/status_ordering.md \ doc/statuscodes.md \ doc/unit_testing.md \ diff --git a/tools/doxygen/Doxyfile.core b/tools/doxygen/Doxyfile.core index 4899eee3ea..aa75bc6828 100644 --- a/tools/doxygen/Doxyfile.core +++ b/tools/doxygen/Doxyfile.core @@ -793,6 +793,7 @@ doc/server-reflection.md \ doc/server_reflection_tutorial.md \ doc/server_side_auth.md \ doc/service_config.md \ +doc/ssl-performance.md \ doc/status_ordering.md \ doc/statuscodes.md \ doc/unit_testing.md \ diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index 9307302aef..75bd67c43d 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -793,6 +793,7 @@ doc/server-reflection.md \ doc/server_reflection_tutorial.md \ doc/server_side_auth.md \ doc/service_config.md \ +doc/ssl-performance.md \ doc/status_ordering.md \ doc/statuscodes.md \ doc/unit_testing.md \ @@ -924,9 +925,12 @@ src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc \ src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.cc \ src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.h \ src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.cc \ +src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_windows.cc \ src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.cc \ src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.h \ src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_fallback.cc \ +src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_posix.cc \ +src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_windows.cc \ src/core/ext/filters/client_channel/resolver/dns/native/README.md \ src/core/ext/filters/client_channel/resolver/dns/native/dns_resolver.cc \ src/core/ext/filters/client_channel/resolver/fake/fake_resolver.cc \ @@ -1356,6 +1360,10 @@ src/core/lib/security/credentials/ssl/ssl_credentials.cc \ src/core/lib/security/credentials/ssl/ssl_credentials.h \ src/core/lib/security/security_connector/alts_security_connector.cc \ src/core/lib/security/security_connector/alts_security_connector.h \ +src/core/lib/security/security_connector/load_system_roots.h \ +src/core/lib/security/security_connector/load_system_roots_fallback.cc \ +src/core/lib/security/security_connector/load_system_roots_linux.cc \ +src/core/lib/security/security_connector/load_system_roots_linux.h \ src/core/lib/security/security_connector/local_security_connector.cc \ src/core/lib/security/security_connector/local_security_connector.h \ src/core/lib/security/security_connector/security_connector.cc \ diff --git a/tools/internal_ci/helper_scripts/prepare_build_macos_interop_rc b/tools/internal_ci/helper_scripts/prepare_build_macos_interop_rc index b0feeef363..43bc9609c7 100644 --- a/tools/internal_ci/helper_scripts/prepare_build_macos_interop_rc +++ b/tools/internal_ci/helper_scripts/prepare_build_macos_interop_rc @@ -17,17 +17,6 @@ # builds. This rc script must be used in the root directory of gRPC # and is expected to be used before prepare_build_macos_rc -export CONFIG=opt - -# Move gRPC repo to directory that Docker for Mac has drive access to -mkdir /Users/kbuilder/workspace -cp -R ./ /Users/kbuilder/workspace/grpc -cd /Users/kbuilder/workspace/grpc - -# Needed for identifying Docker image sha1 -brew update -brew install md5sha1sum - # Set up gRPC-Go and gRPC-Java to test git clone --recursive https://github.com/grpc/grpc-go ./../grpc-go git clone --recursive https://github.com/grpc/grpc-java ./../grpc-java diff --git a/tools/internal_ci/helper_scripts/prepare_build_macos_rc b/tools/internal_ci/helper_scripts/prepare_build_macos_rc index 7cdb2e0cf7..3212e80854 100644 --- a/tools/internal_ci/helper_scripts/prepare_build_macos_rc +++ b/tools/internal_ci/helper_scripts/prepare_build_macos_rc @@ -39,11 +39,17 @@ pip install google-api-python-client==1.6.7 --user python export GOOGLE_APPLICATION_CREDENTIALS=${KOKORO_GFILE_DIR}/GrpcTesting-d0eeee2db331.json # If this is a PR using RUN_TESTS_FLAGS var, then add flags to filter tests -if [ -n "$KOKORO_GITHUB_PULL_REQUEST_NUMBER" ] && [ -n "$RUN_TESTS_FLAGS" ]; then +if [ -n "$KOKORO_GITHUB_PULL_REQUEST_NUMBER" ]; then + set +x brew update brew install jq ghprbTargetBranch=$(curl -s https://api.github.com/repos/grpc/grpc/pulls/$KOKORO_GITHUB_PULL_REQUEST_NUMBER | jq -r .base.ref) export RUN_TESTS_FLAGS="$RUN_TESTS_FLAGS --filter_pr_tests --base_branch origin/$ghprbTargetBranch" + + # TODO(matt-kwong): rename this to GITHUB_OAUTH_TOKEN after Jenkins deprecation + export JENKINS_OAUTH_TOKEN=$(cat ${KOKORO_GFILE_DIR}/oauth_token.txt) + export ghprbPullId=$KOKORO_GITHUB_PULL_REQUEST_NUMBER + set -x fi set +ex # rvm script is very verbose and exits with errorcode @@ -83,3 +89,10 @@ export DOTNET_CLI_TELEMETRY_OPTOUT=true date git submodule update --init + +# Store intermediate build files of ObjC tests into /tmpfs +mkdir /tmpfs/Build-ios-binary-size +ln -s /tmpfs/Build-ios-binary-size src/objective-c/examples/Sample/Build +mkdir /tmpfs/DerivedData +rm -rf ~/Library/Developer/Xcode/DerivedData +ln -s /tmpfs/DerivedData ~/Library/Developer/Xcode/DerivedData diff --git a/tools/internal_ci/helper_scripts/prepare_build_windows.bat b/tools/internal_ci/helper_scripts/prepare_build_windows.bat index 0164e4a561..f987f8a8cb 100644 --- a/tools/internal_ci/helper_scripts/prepare_build_windows.bat +++ b/tools/internal_ci/helper_scripts/prepare_build_windows.bat @@ -14,7 +14,7 @@ @rem make sure msys binaries are preferred over cygwin binaries @rem set path to python 2.7 -set PATH=C:\tools\msys64\usr\bin;C:\Python27;C:\Python37;%PATH% +set PATH=C:\tools\msys64\usr\bin;C:\Python27;%PATH% @rem If this is a PR using RUN_TESTS_FLAGS var, then add flags to filter tests if defined KOKORO_GITHUB_PULL_REQUEST_NUMBER if defined RUN_TESTS_FLAGS ( @@ -34,9 +34,6 @@ netsh interface ip add dnsservers "Local Area Connection 8" 8.8.4.4 index=3 @rem Needed for big_query_utils python -m pip install google-api-python-client -@rem Install Python 3.7 -chocolatey install -y -r python3 --version 3.7 - @rem Disable some unwanted dotnet options set NUGET_XMLDOC_MODE=skip set DOTNET_SKIP_FIRST_TIME_EXPERIENCE=true diff --git a/tools/internal_ci/linux/grpc_android.sh b/tools/internal_ci/linux/grpc_android.sh index 2220145dac..42c7f5fb04 100755 --- a/tools/internal_ci/linux/grpc_android.sh +++ b/tools/internal_ci/linux/grpc_android.sh @@ -25,13 +25,6 @@ git submodule update --init # Build protoc and grpc_cpp_plugin. Codegen is not cross-compiled to Android make HAS_SYSTEM_PROTOBUF=false -# TODO(ericgribkoff) Remove when this commit (already in master) is included in -# next protobuf release -cd third_party/protobuf -git fetch -git cherry-pick 7daa320065f3bea2b54bf983337d1724f153422d -m 1 - - # Build and run interop instrumentation tests on Firebase Test Lab cd "${REPO_ROOT}/src/android/test/interop/" diff --git a/tools/internal_ci/linux/grpc_asan_on_foundry.sh b/tools/internal_ci/linux/grpc_asan_on_foundry.sh index 5099fa0301..a6367ad962 100644 --- a/tools/internal_ci/linux/grpc_asan_on_foundry.sh +++ b/tools/internal_ci/linux/grpc_asan_on_foundry.sh @@ -14,6 +14,6 @@ # limitations under the License. export UPLOAD_TEST_RESULTS=true -EXTRA_FLAGS="--copt=-gmlt --strip=never --copt=-fsanitize=address --linkopt=-fsanitize=address --test_timeout=3600 --runs_per_test_detects_flakes --runs_per_test=2" +EXTRA_FLAGS="--copt=-gmlt --strip=never --copt=-fsanitize=address --linkopt=-fsanitize=address --test_timeout=3600 --cache_test_results=no" github/grpc/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh "${EXTRA_FLAGS}" diff --git a/tools/internal_ci/linux/grpc_bazel_on_foundry_dbg.sh b/tools/internal_ci/linux/grpc_bazel_on_foundry_dbg.sh index 192d9d1b2d..51cb66f5b8 100644 --- a/tools/internal_ci/linux/grpc_bazel_on_foundry_dbg.sh +++ b/tools/internal_ci/linux/grpc_bazel_on_foundry_dbg.sh @@ -14,5 +14,5 @@ # limitations under the License. export UPLOAD_TEST_RESULTS=true -EXTRA_FLAGS="-c dbg --test_timeout=300,450,1200,3600 --runs_per_test_detects_flakes --runs_per_test=2" +EXTRA_FLAGS="-c dbg --test_timeout=300,450,1200,3600 --cache_test_results=no" github/grpc/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh "${EXTRA_FLAGS}" diff --git a/tools/internal_ci/linux/grpc_bazel_on_foundry_opt.sh b/tools/internal_ci/linux/grpc_bazel_on_foundry_opt.sh index 6fb3c77892..cbba9067ad 100644 --- a/tools/internal_ci/linux/grpc_bazel_on_foundry_opt.sh +++ b/tools/internal_ci/linux/grpc_bazel_on_foundry_opt.sh @@ -14,5 +14,5 @@ # limitations under the License. export UPLOAD_TEST_RESULTS=true -EXTRA_FLAGS="-c opt --test_timeout=300,450,1200,3600 --runs_per_test_detects_flakes --runs_per_test=2" +EXTRA_FLAGS="-c opt --test_timeout=300,450,1200,3600 --cache_test_results=no" github/grpc/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh "${EXTRA_FLAGS}" diff --git a/tools/internal_ci/linux/grpc_build_artifacts_extra_release.cfg b/tools/internal_ci/linux/grpc_build_artifacts_extra_release.cfg new file mode 100644 index 0000000000..619e3ea3a9 --- /dev/null +++ b/tools/internal_ci/linux/grpc_build_artifacts_extra_release.cfg @@ -0,0 +1,26 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/linux/grpc_build_artifacts_extra.sh" +timeout_mins: 240 +action { + define_artifacts { + regex: "**/*sponge_log.xml" + regex: "github/grpc/reports/**" + regex: "github/grpc/artifacts/**" + } +} diff --git a/tools/internal_ci/linux/grpc_publish_packages.cfg b/tools/internal_ci/linux/grpc_publish_packages.cfg new file mode 100644 index 0000000000..82d571d642 --- /dev/null +++ b/tools/internal_ci/linux/grpc_publish_packages.cfg @@ -0,0 +1,26 @@ +# Copyright 2018 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/linux/grpc_publish_packages.sh" +timeout_mins: 120 +action { + define_artifacts { + regex: "**/*sponge_log.xml" + regex: "github/grpc/reports/**" + regex: "github/grpc/artifacts/**" + } +} diff --git a/tools/internal_ci/linux/grpc_publish_packages.sh b/tools/internal_ci/linux/grpc_publish_packages.sh new file mode 100755 index 0000000000..14492301cc --- /dev/null +++ b/tools/internal_ci/linux/grpc_publish_packages.sh @@ -0,0 +1,235 @@ +#!/bin/bash +# Copyright 2018 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +shopt -s nullglob + +cd "$(dirname "$0")/../../.." + +GRPC_VERSION=$(grep -e "^ *version: " build.yaml | head -n 1 | sed 's/.*: //') + +INPUT_ARTIFACTS=$KOKORO_GFILE_DIR/github/grpc/artifacts +INDEX_FILENAME=index.xml + +BUILD_ID=${KOKORO_BUILD_ID:-$(uuidgen)} +BUILD_BRANCH_NAME=master +BUILD_GIT_COMMIT=${KOKORO_GIT_COMMIT:-unknown} +BUILD_TIMESTAMP=$(date -Iseconds) +BUILD_RELPATH=$(date "+%Y/%m")/$BUILD_GIT_COMMIT-$BUILD_ID/ + +GCS_ROOT=gs://packages.grpc.io/ +GCS_ARCHIVE_PREFIX=archive/ +GCS_ARCHIVE_ROOT=$GCS_ROOT$GCS_ARCHIVE_PREFIX +GCS_INDEX=$GCS_ROOT$INDEX_FILENAME + +LOCAL_STAGING_TEMPDIR=$(mktemp -d) +LOCAL_BUILD_ROOT=$LOCAL_STAGING_TEMPDIR/$BUILD_RELPATH +LOCAL_BUILD_INDEX=$LOCAL_BUILD_ROOT$INDEX_FILENAME + +mkdir -p "$LOCAL_BUILD_ROOT" + +find "$INPUT_ARTIFACTS" -type f + +# protoc Plugins +PROTOC_PLUGINS_ZIPPED_PACKAGES=$(mktemp -d) +for zip_dir in protoc_windows_{x86,x64} +do + zip -jr "$PROTOC_PLUGINS_ZIPPED_PACKAGES/grpc-$zip_dir-$GRPC_VERSION.zip" "$INPUT_ARTIFACTS/$zip_dir/"* +done +for tar_dir in protoc_{linux,macos}_{x86,x64} +do + chmod +x "$INPUT_ARTIFACTS/$tar_dir"/* + tar -cvzf "$PROTOC_PLUGINS_ZIPPED_PACKAGES/grpc-$tar_dir-$GRPC_VERSION.tar.gz" -C "$INPUT_ARTIFACTS/$tar_dir" . +done + +PROTOC_PACKAGES=( + "$PROTOC_PLUGINS_ZIPPED_PACKAGES"/grpc-protoc_windows_{x86,x64}-"$GRPC_VERSION.zip" + "$PROTOC_PLUGINS_ZIPPED_PACKAGES"/grpc-protoc_{linux,macos}_{x86,x64}-"$GRPC_VERSION.tar.gz" +) + +# C# +UNZIPPED_CSHARP_PACKAGES=$(mktemp -d) +unzip "$INPUT_ARTIFACTS/csharp_nugets_windows_dotnetcli.zip" -d "$UNZIPPED_CSHARP_PACKAGES" +CSHARP_PACKAGES=( + "$UNZIPPED_CSHARP_PACKAGES"/* + "$INPUT_ARTIFACTS"/grpc_unity_package.[0-9]*.zip +) + +# Python +PYTHON_GRPCIO_PACKAGES=( + "$INPUT_ARTIFACTS"/grpcio-[0-9]*.tar.gz + "$INPUT_ARTIFACTS"/grpcio-[0-9]*.whl + "$INPUT_ARTIFACTS"/python_linux_extra_arm*/grpcio-[0-9]*.whl +) +PYTHON_GRPCIO_TOOLS_PACKAGES=( + "$INPUT_ARTIFACTS"/grpcio-tools-[0-9]*.tar.gz + "$INPUT_ARTIFACTS"/grpcio_tools-[0-9]*.whl + "$INPUT_ARTIFACTS"/python_linux_extra_arm*/grpcio_tools-[0-9]*.whl +) +PYTHON_GRPCIO_HEALTH_CHECKING_PACKAGES=( + "$INPUT_ARTIFACTS"/grpcio-health-checking-[0-9]*.tar.gz +) +PYTHON_GRPCIO_REFLECTION_PACKAGES=( + "$INPUT_ARTIFACTS"/grpcio-reflection-[0-9]*.tar.gz +) +PYTHON_GRPCIO_TESTING_PACKAGES=( + "$INPUT_ARTIFACTS"/grpcio-testing-[0-9]*.tar.gz +) + +# PHP +PHP_PACKAGES=( + "$INPUT_ARTIFACTS"/grpc-[0-9]*.tgz +) + +# Ruby +RUBY_PACKAGES=( + "$INPUT_ARTIFACTS"/grpc-[0-9]*.gem + "$INPUT_ARTIFACTS"/grpc-tools-[0-9]*.gem +) + +function add_to_manifest() { + local artifact_type=$1 + local artifact_file=$2 + local artifact_prefix=$3 + local artifact_name + artifact_name=$(basename "$artifact_file") + local artifact_size + artifact_size=$(stat -c%s "$artifact_file") + local artifact_sha256 + artifact_sha256=$(openssl sha256 -r "$artifact_file" | cut -d " " -f 1) + local artifact_target=$LOCAL_BUILD_ROOT/$artifact_type/$artifact_prefix + mkdir -p "$artifact_target" + cp "$artifact_file" "$artifact_target" + cat <<EOF + <artifact name='$artifact_name' + type='$artifact_type' + path='$artifact_type/$artifact_prefix$artifact_name' + size='$artifact_size' + sha256='$artifact_sha256' /> +EOF +} + +{ + cat <<EOF +<?xml version="1.0"?> +<?xml-stylesheet href="/web-assets/build-201807.xsl" type="text/xsl"?> +<build id='$BUILD_ID' timestamp='$BUILD_TIMESTAMP' version="201807"> + <metadata> + <project>gRPC</project> + <repository>https://github.com/grpc/grpc</repository> + <branch>$BUILD_BRANCH_NAME</branch> + <commit>$BUILD_GIT_COMMIT</commit> + </metadata> + <artifacts> +EOF + + for pkg in "${PROTOC_PACKAGES[@]}"; do add_to_manifest protoc "$pkg"; done + for pkg in "${CSHARP_PACKAGES[@]}"; do add_to_manifest csharp "$pkg"; done + for pkg in "${PHP_PACKAGES[@]}"; do add_to_manifest php "$pkg"; done + for pkg in "${PYTHON_GRPCIO_PACKAGES[@]}"; do add_to_manifest python "$pkg" grpcio/; done + for pkg in "${PYTHON_GRPCIO_TOOLS_PACKAGES[@]}"; do add_to_manifest python "$pkg" grpcio-tools/; done + for pkg in "${PYTHON_GRPCIO_HEALTH_CHECKING_PACKAGES[@]}"; do add_to_manifest python "$pkg" grpcio-health-checking/; done + for pkg in "${PYTHON_GRPCIO_REFLECTION_PACKAGES[@]}"; do add_to_manifest python "$pkg" grpcio-reflection/; done + for pkg in "${PYTHON_GRPCIO_TESTING_PACKAGES[@]}"; do add_to_manifest python "$pkg" grpcio-testing/; done + for pkg in "${RUBY_PACKAGES[@]}"; do add_to_manifest ruby "$pkg"; done + + cat <<EOF + </artifacts> +</build> +EOF +}> "$LOCAL_BUILD_INDEX" + +LOCAL_BUILD_INDEX_SIZE=$(stat -c%s "$LOCAL_BUILD_INDEX") +LOCAL_BUILD_INDEX_SHA256=$(openssl sha256 -r "$LOCAL_BUILD_INDEX" | cut -d " " -f 1) + +OLD_INDEX=$(mktemp) +NEW_INDEX=$(mktemp) + +# Download the current /index.xml into $OLD_INDEX +gsutil cp "$GCS_INDEX" "$OLD_INDEX" + +{ + # we want to add an entry as the first child under <builds> tag + # we can get by without a real XML parser by rewriting the header, + # injecting our new tag, and then dumping the rest of the file as is. + cat <<EOF +<?xml version="1.0"?> +<?xml-stylesheet href="/web-assets/home.xsl" type="text/xsl"?> +<packages> + <builds> + <build id='$BUILD_ID' + timestamp='$BUILD_TIMESTAMP' + branch='$BUILD_BRANCH_NAME' + commit='$BUILD_GIT_COMMIT' + path='$GCS_ARCHIVE_PREFIX$BUILD_RELPATH$INDEX_FILENAME' + size='$LOCAL_BUILD_INDEX_SIZE' + sha256='$LOCAL_BUILD_INDEX_SHA256' /> +EOF + tail --lines=+5 "$OLD_INDEX" +}> "$NEW_INDEX" + + +function generate_directory_index() +{ + local target_dir=$1 + local current_directory_name + current_directory_name=$(basename "$target_dir") + cat <<EOF +<!DOCTYPE html> +<html xmlns="http://www.w3.org/1999/xhtml" lang="" xml:lang=""> + <head> + <meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /> + <title>Index of $current_directory_name - packages.grpc.io</title> + <link rel="stylesheet" type="text/css" href="/web-assets/dirindex.css" /> + </head> + <body> + <h1>Index of <a href="#"><code>$current_directory_name</code></a></h1> + <ul> + <li><a href="#">.</a></li> + <li><a href="..">..</a></li> +EOF + +( + cd "$target_dir" + find * -maxdepth 0 -type d -print | sort | while read -r line + do + echo " <li><a href='$line/'>$line/</a></li>" + done + find * -maxdepth 0 -type f -print | sort | while read -r line + do + echo " <li><a href='$line'>$line</a></li>" + done +) + +cat <<EOF + </ul> + </body> +</html> +EOF +} + +# Upload the current build artifacts +gsutil -m cp -r "$LOCAL_STAGING_TEMPDIR/${BUILD_RELPATH%%/*}" "$GCS_ARCHIVE_ROOT" +# Upload directory indicies for subdirectories +( + cd "$LOCAL_BUILD_ROOT" + find * -type d | while read -r directory + do + generate_directory_index "$directory" | gsutil -h 'Content-Type:text/html' cp - "$GCS_ARCHIVE_ROOT$BUILD_RELPATH$directory/$INDEX_FILENAME" + done +) +# Upload the new /index.xml +gsutil -h "Content-Type:application/xml" cp "$NEW_INDEX" "$GCS_INDEX" diff --git a/tools/internal_ci/linux/grpc_tsan_on_foundry.sh b/tools/internal_ci/linux/grpc_tsan_on_foundry.sh index 1396fc4e42..2ba7d469ec 100644 --- a/tools/internal_ci/linux/grpc_tsan_on_foundry.sh +++ b/tools/internal_ci/linux/grpc_tsan_on_foundry.sh @@ -14,5 +14,5 @@ # limitations under the License. export UPLOAD_TEST_RESULTS=true -EXTRA_FLAGS="--copt=-gmlt --strip=never --copt=-fsanitize=thread --linkopt=-fsanitize=thread --test_timeout=3600 --action_env=TSAN_OPTIONS=suppressions=test/core/util/tsan_suppressions.txt:halt_on_error=1:second_deadlock_stack=1" +EXTRA_FLAGS="--copt=-gmlt --strip=never --copt=-fsanitize=thread --linkopt=-fsanitize=thread --test_timeout=3600 --action_env=TSAN_OPTIONS=suppressions=test/core/util/tsan_suppressions.txt:halt_on_error=1:second_deadlock_stack=1 --cache_test_results=no" github/grpc/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh "${EXTRA_FLAGS}" diff --git a/tools/internal_ci/linux/grpc_ubsan_on_foundry.sh b/tools/internal_ci/linux/grpc_ubsan_on_foundry.sh index d9b039f1ec..338b1b6a0d 100644 --- a/tools/internal_ci/linux/grpc_ubsan_on_foundry.sh +++ b/tools/internal_ci/linux/grpc_ubsan_on_foundry.sh @@ -61,8 +61,7 @@ source tools/internal_ci/helper_scripts/prepare_build_linux_rc --extra_execution_platforms=//third_party/toolchains:rbe_ubuntu1604 \ --host_platform=//third_party/toolchains:rbe_ubuntu1604 \ --platforms=//third_party/toolchains:rbe_ubuntu1604 \ - --runs_per_test_detects_flakes \ - --runs_per_test=2 \ + --cache_test_results=no \ -- //test/... || FAILED="true" # Sleep to let ResultStore finish writing results before querying diff --git a/tools/internal_ci/linux/pull_request/grpc_tsan_on_foundry.sh b/tools/internal_ci/linux/pull_request/grpc_tsan_on_foundry.sh new file mode 100644 index 0000000000..edd8f92975 --- /dev/null +++ b/tools/internal_ci/linux/pull_request/grpc_tsan_on_foundry.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +EXTRA_FLAGS="--copt=-gmlt --strip=never --copt=-fsanitize=thread --linkopt=-fsanitize=thread --test_timeout=3600 --action_env=TSAN_OPTIONS=suppressions=test/core/util/tsan_suppressions.txt:halt_on_error=1:second_deadlock_stack=1" +github/grpc/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh "${EXTRA_FLAGS}" diff --git a/tools/internal_ci/macos/grpc_interop_toprod.sh b/tools/internal_ci/macos/grpc_interop_toprod.sh index 5ddabb9bf9..e748a62e76 100755 --- a/tools/internal_ci/macos/grpc_interop_toprod.sh +++ b/tools/internal_ci/macos/grpc_interop_toprod.sh @@ -18,8 +18,8 @@ set -ex # change to grpc repo root cd $(dirname $0)/../../.. -source tools/internal_ci/helper_scripts/prepare_build_macos_interop_rc source tools/internal_ci/helper_scripts/prepare_build_macos_rc +source tools/internal_ci/helper_scripts/prepare_build_macos_interop_rc # using run_interop_tests.py without --use_docker, so we need to build first tools/run_tests/run_tests.py -l c++ -c opt --build_only diff --git a/tools/internal_ci/macos/grpc_ios_binary_size.sh b/tools/internal_ci/macos/grpc_ios_binary_size.sh new file mode 100755 index 0000000000..ea39b0d6e9 --- /dev/null +++ b/tools/internal_ci/macos/grpc_ios_binary_size.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +# Copyright 2018 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This script is invoked by Jenkins and runs a diff on the microbenchmarks +set -ex + +# List of benchmarks that provide good signal for analyzing performance changes in pull requests + +# Enter the gRPC repo root +cd $(dirname $0)/../../.. + +source tools/internal_ci/helper_scripts/prepare_build_macos_rc + +tools/profiling/ios_bin/binary_size.py \ + -d origin/$ghprbTargetBranch diff --git a/tools/internal_ci/macos/pull_request/grpc_ios_binary_size.cfg b/tools/internal_ci/macos/pull_request/grpc_ios_binary_size.cfg new file mode 100644 index 0000000000..8942bc7ba7 --- /dev/null +++ b/tools/internal_ci/macos/pull_request/grpc_ios_binary_size.cfg @@ -0,0 +1,26 @@ +# Copyright 2018 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/macos/grpc_ios_binary_size.sh" +timeout_mins: 60 +gfile_resources: "/bigstore/grpc-testing-secrets/github_credentials/oauth_token.txt" +action { + define_artifacts { + regex: "**/*sponge_log.xml" + regex: "github/grpc/reports/**" + } +} diff --git a/tools/internal_ci/windows/grpc_build_artifacts.bat b/tools/internal_ci/windows/grpc_build_artifacts.bat index 4d528e0128..0e02bdb6ec 100644 --- a/tools/internal_ci/windows/grpc_build_artifacts.bat +++ b/tools/internal_ci/windows/grpc_build_artifacts.bat @@ -18,6 +18,7 @@ rename C:\Python27_32bit Python27_32bits rename C:\Python34_32bit Python34_32bits rename C:\Python35_32bit Python35_32bits rename C:\Python36_32bit Python36_32bits +rename C:\Python37_32bit Python37_32bits @rem enter repo root cd /d %~dp0\..\..\.. diff --git a/tools/interop_matrix/README.md b/tools/interop_matrix/README.md index 40c02a1bc7..db84d9b454 100644 --- a/tools/interop_matrix/README.md +++ b/tools/interop_matrix/README.md @@ -9,7 +9,7 @@ from specific releases/tag, are used to test version compatiblity between gRPC r We have continuous nightly test setup to test gRPC backward compatibility between old clients and latest server. When a gRPC developer creates a new gRPC release, s/he is also responsible to add the just-released gRPC client to the nightly test. The steps are: - Add (or update) an entry in `./client_matrix.py` file to reference the github tag for the release. - Build new client docker image(s). For example, for C and wrapper languages release `v1.9.9`, do - - `tools/interop_matrix/create_matrix_images.py --git_checkout --release=v1.9.9 --language cxx csharp python ruby php` + - `tools/interop_matrix/create_matrix_images.py --git_checkout --release=v1.9.9 --upload_images --language cxx csharp python ruby php` - Verify that the new docker image was built successfully and uploaded to GCR. For example, - `gcloud beta container images list --repository gcr.io/grpc-testing` shows image repos. - `gcloud beta container images list-tags gcr.io/grpc-testing/grpc_interop_java_oracle8` should show an image entry with tag `v1.9.9`. diff --git a/tools/interop_matrix/client_matrix.py b/tools/interop_matrix/client_matrix.py index cc4b5fcf74..bb9222d953 100644 --- a/tools/interop_matrix/client_matrix.py +++ b/tools/interop_matrix/client_matrix.py @@ -96,6 +96,9 @@ LANG_RELEASE_MATRIX = { { 'v1.13.0': None }, + { + 'v1.14.1': None + }, ], 'go': [ { @@ -178,6 +181,9 @@ LANG_RELEASE_MATRIX = { { 'v1.13.1': None }, + { + 'v1.14.0': None + }, ], 'python': [ { @@ -219,6 +225,9 @@ LANG_RELEASE_MATRIX = { { 'v1.13.0': None }, + { + 'v1.14.1': None + }, ], 'node': [ { @@ -252,6 +261,12 @@ LANG_RELEASE_MATRIX = { { 'v1.10.0': None }, + { + 'v1.11.3': None + }, + { + 'v1.12.4': None + }, ], 'ruby': [ { @@ -298,6 +313,9 @@ LANG_RELEASE_MATRIX = { { 'v1.13.0': None }, + { + 'v1.14.1': None + }, ], 'php': [ { @@ -339,6 +357,9 @@ LANG_RELEASE_MATRIX = { { 'v1.13.0': None }, + { + 'v1.14.1': None + }, ], 'csharp': [ { @@ -385,6 +406,9 @@ LANG_RELEASE_MATRIX = { { 'v1.13.0': None }, + { + 'v1.14.1': None + }, ], } diff --git a/tools/package_hosting/404.html b/tools/package_hosting/404.html new file mode 100644 index 0000000000..44d986c4b0 --- /dev/null +++ b/tools/package_hosting/404.html @@ -0,0 +1 @@ +404 Not Found diff --git a/tools/package_hosting/build-201807.xsl b/tools/package_hosting/build-201807.xsl new file mode 100644 index 0000000000..69a190446f --- /dev/null +++ b/tools/package_hosting/build-201807.xsl @@ -0,0 +1,114 @@ +<?xml version="1.0"?> +<xsl:stylesheet version="1.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform"> + +<xsl:template match="//build"> + <html> + <head> + <title>Artifacts for gRPC Build <xsl:value-of select="@id"/> </title> + <link rel="stylesheet" type="text/css" href="/web-assets/style.css" /> + <link rel="apple-touch-icon" href="/web-assets/favicons/apple-touch-icon.png" sizes="180x180" /> + <link rel="icon" type="image/png" href="/web-assets/favicons/android-chrome-192x192.png" sizes="192x192" /> + <link rel="icon" type="image/png" href="/web-assets/favicons/favicon-32x32.png" sizes="32x32" /> + <link rel="icon" type="image/png" href="/web-assets/favicons/favicon-16x16.png" sizes="16x16" /> + <link rel="manifest" href="/web-assets/favicons/manifest.json" /> + <link rel="mask-icon" href="/web-assets/favicons/safari-pinned-tab.svg" color="#2DA6B0" /> + <meta name="msapplication-TileColor" content="#ffffff" /> + <meta name="msapplication-TileImage" content="/web-assets/favicons/mstile-150x150.png" /> + + <meta name="og:title" content="gRPC Package Build"/> + <meta name="og:image" content="https://grpc.io/img/grpc_square_reverse_4x.png"/> + <meta name="og:description" content="gRPC Package Build"/> + </head> + <body bgcolor="#ffffff"> + <div id="topbar"> + <span class="title">Artifacts for gRPC Build <xsl:value-of select="@id"/></span> + </div> + <div id="main"> + <div id="metadata"> + <span class="fieldname">Build: </span> <a href='#'><xsl:value-of select="@id"/></a> + [<a href="https://source.cloud.google.com/results/invocations/{@id}">invocation</a>]<br /> + <span class="fieldname">Timestamp: </span> + <xsl:value-of select="@timestamp"/> <br /> + <span class="fieldname">Branch: </span> + <a href="https://github.com/grpc/grpc/tree/{./metadata/branch[text()]}"> + <xsl:value-of select="./metadata/branch[text()]" /> + </a><br /> + <span class="fieldname">Commit: </span> + <a href="https://github.com/grpc/grpc/tree/{./metadata/commit[text()]}"> + <xsl:value-of select="./metadata/commit[text()]" /><br /></a> + </div> + <xsl:apply-templates select="artifacts" /> + <br /> + <br /> + + <p class="description"><a href="https://grpc.io">gRPC</a> is a <a href="https://www.cncf.io" class="external">Cloud Native Computing Foundation</a> project. <a href="https://policies.google.com/privacy" class="external">Privacy Policy</a>.</p> + <p class="description"> + Copyright © <xsl:value-of select="substring(@timestamp, 1, 4)" /> <a href="https://github.com/grpc/grpc/blob/{./metadata/commit[text()]}/AUTHORS">The gRPC Authors</a></p> + <br /> + <br /> + </div> + </body> + </html> +</xsl:template> + +<xsl:template match="artifacts"> +<h2> gRPC <code>protoc</code> Plugins </h2> +<table> +<xsl:apply-templates select="artifact[@type='protoc']"> + <xsl:sort select="artifact/@name" /> + </xsl:apply-templates> +</table> + +<h2> C# </h2> +<table> +<xsl:apply-templates select="artifact[@type='csharp']"> + <xsl:sort select="artifact/@name" /> + </xsl:apply-templates> +</table> + +<h2> PHP </h2> +<table> +<xsl:apply-templates select="artifact[@type='php']"> + <xsl:sort select="artifact/@name" /> + </xsl:apply-templates> +</table> + +<h2> Python </h2> +<script type="text/javascript"> +// <![CDATA[ +var pythonRepoLink = document.createElement("a"); +pythonRepoLink.href = './python'; +var pythonRepo = pythonRepoLink.href; +document.write("<p><code>" + +"$ pip install --pre --upgrade --force-reinstall --extra-index-url \\<br />" + +" <a href='" + pythonRepo + "'>" + pythonRepo + "</a> \\<br />" + +" grpcio grpcio-{tools,health-checking,reflection,testing}</code></p>"); +// ]]> +</script> +<table> + <xsl:apply-templates select="artifact[@type='python']"> + <xsl:sort select="artifact/@name" /> + </xsl:apply-templates> +</table> + +<h2> Ruby </h2> +<table> +<xsl:apply-templates select="artifact[@type='ruby']"> + <xsl:sort select="artifact/@name" /> + </xsl:apply-templates> +</table> + +</xsl:template> + + +<xsl:template match="artifact"> +<tr> +<td class="name"> <a href="{@path}"><xsl:value-of select="@name" /></a> </td> +<td class="hash"> <xsl:value-of select="@sha256"/> </td> +</tr> +</xsl:template> + +<xsl:template match="metadata"> +</xsl:template> + +</xsl:stylesheet> diff --git a/tools/package_hosting/dirindex.css b/tools/package_hosting/dirindex.css new file mode 100644 index 0000000000..6c4b04ea32 --- /dev/null +++ b/tools/package_hosting/dirindex.css @@ -0,0 +1,16 @@ +ul { + list-style-type: none; +} +a{ + text-decoration: none; +} +a:hover { + text-decoration: underline; +} +ul li a { + font-family: 'SF Mono', 'Menlo', 'Monaco', 'Consolas', 'Courier New', Courier, monospace +} +h1 { + font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Oxygen, Ubuntu, Cantarell, 'Open Sans', 'Helvetica Neue', sans-serif +} + diff --git a/tools/package_hosting/home.xsl b/tools/package_hosting/home.xsl new file mode 100644 index 0000000000..3f79303b1b --- /dev/null +++ b/tools/package_hosting/home.xsl @@ -0,0 +1,86 @@ +<?xml version="1.0"?> +<xsl:stylesheet version="2.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform"> + +<xsl:template match="//packages"> + <html> + <head> + <title>gRPC Packages</title> + <link rel="stylesheet" type="text/css" href="/web-assets/style.css" /> + <link rel="apple-touch-icon" href="/web-assets/favicons/apple-touch-icon.png" sizes="180x180" /> + <link rel="icon" type="image/png" href="/web-assets/favicons/android-chrome-192x192.png" sizes="192x192" /> + <link rel="icon" type="image/png" href="/web-assets/favicons/favicon-32x32.png" sizes="32x32" /> + <link rel="icon" type="image/png" href="/web-assets/favicons/favicon-16x16.png" sizes="16x16" /> + <link rel="manifest" href="/web-assets/favicons/manifest.json" /> + <link rel="mask-icon" href="/web-assets/favicons/safari-pinned-tab.svg" color="#2DA6B0" /> + <meta name="msapplication-TileColor" content="#ffffff" /> + <meta name="msapplication-TileImage" content="/web-assets/favicons/mstile-150x150.png" /> + <meta name="og:title" content="gRPC Packages"/> + <meta name="og:image" content="https://grpc.io/img/grpc_square_reverse_4x.png"/> + <meta name="og:description" content="gRPC Packages"/> + </head> + <body bgcolor="#ffffff"> + <div id="topbar"> + <span class="title">gRPC Packages</span> + </div> + <div id="main"> + <xsl:apply-templates select="releases" /> + <xsl:apply-templates select="builds" /> + <br /> + <br /> + <p class="description"><a href="https://grpc.io">gRPC</a> is a <a href="https://www.cncf.io" class="external">Cloud Native Computing Foundation</a> project. <a href="https://policies.google.com/privacy" class="external">Privacy Policy</a>.</p> + <p class="description">Copyright © 2018 <a href="https://github.com/grpc/grpc/blob/master/AUTHORS">The gRPC Authors</a></p> + </div> + </body> + </html> +</xsl:template> + +<xsl:template match="releases"> + <h2>Official gRPC Releases</h2> + <p>Commits corresponding to <a href="https://github.com/grpc/grpc/releases">official gRPC release points and release candidates</a> are tagged on GitHub.</p> + <p>To maximize usability, gRPC supports the standard way of adding dependencies in your language of choice (if there is one). + In most languages, the gRPC runtime comes in form of a package available in your language's package manager.</p> + <p>For instructions on how to use the language-specific gRPC runtime in your project, please refer to the following:</p> + <ul> + <li><a href="https://github.com/grpc/grpc/blob/master/src/cpp">C++</a>: follow the instructions under the <a href="https://github.com/grpc/grpc/tree/master/src/cpp"><code>src/cpp</code> directory</a></li> + <li><a href="https://github.com/grpc/grpc/blob/master/src/csharp">C#</a>: NuGet package <code>Grpc</code></li> + <li><a href="https://github.com/grpc/grpc-dart">Dart</a>: pub package <code>grpc</code></li> + <li><a href="https://github.com/grpc/grpc-go">Go</a>: <code>go get google.golang.org/grpc</code></li> + <li><a href="https://github.com/grpc/grpc-java">Java</a>: Use JARs from <a href="https://mvnrepository.com/artifact/io.grpc">gRPC Maven Central Repository</a></li> + <li><a href="https://github.com/grpc/grpc-node">Node</a>: <code>npm install grpc</code></li> + <li><a href="https://github.com/grpc/grpc/blob/master/src/objective-c">Objective-C</a>: Add <code>gRPC-ProtoRPC</code> dependency to podspec</li> + <li><a href="https://github.com/grpc/grpc/blob/master/src/php">PHP</a>: <code>pecl install grpc</code></li> + <li><a href="https://github.com/grpc/grpc/blob/master/src/python/grpcio">Python</a>: <code>pip install grpcio</code></li> + <li><a href="https://github.com/grpc/grpc/blob/master/src/ruby">Ruby</a>: <code>gem install grpc</code></li> + <li><a href="https://github.com/grpc/grpc-web">WebJS</a>: follow the <a href="https://github.com/grpc/grpc-web">instructions in <code>grpc-web</code> repository</a></li> + </ul> +</xsl:template> + +<xsl:template match="builds"> + <h2> Daily Builds of <a href="https://github.com/grpc/grpc/tree/master"><code>master</code></a> Branch</h2> + <p>gRPC packages are built on a daily basis at the <code>HEAD</code> of <a href="https://github.com/grpc/grpc/tree/master">the <code>master</code> branch</a> and are archived here.</p> + <p> + <a href="#">The current document</a> (view source) is an XML feed pointing to the packages as they get built and uploaded. + You can subscribe to this feed and fetch, deploy, and test the precompiled packages with your continuous integration infrastructure. + </p> + <p>For stable release packages, please consult the above section and the common package manager for your language.</p> + <table style="border:solid black 1px"> + <tr style="background-color:lightgray"> + <td>Timestamp</td> + <td>Commit</td> + <td>Build ID</td> + </tr> + <xsl:apply-templates select="build[@branch='master']"> + <xsl:sort select="@timestamp" data-type="text" order="descending" /> + </xsl:apply-templates> + </table> +</xsl:template> + +<xsl:template match="build"> + <tr> + <td class="name"><xsl:value-of select="@timestamp" /></td> + <td class="name"> <a href="https://github.com/grpc/grpc/tree/{@commit}"><xsl:value-of select="@commit" /></a></td> + <td class="name"> <a href="{@path}"><xsl:value-of select="@id" /></a></td> + </tr> +</xsl:template> + +</xsl:stylesheet> diff --git a/tools/package_hosting/style.css b/tools/package_hosting/style.css new file mode 100644 index 0000000000..dbd26d50cf --- /dev/null +++ b/tools/package_hosting/style.css @@ -0,0 +1,76 @@ +html, body +{ + margin: 0; + font-family: sans-serif; +} + +a, a:visited, a:link, a:active { + color: #2da6b0; + text-decoration: none; +} + +a:hover { + color: #2da6b0; + text-decoration: underline; +} + +#topbar { + background-color: #2da6b0; + height: 60px; + margin:auto; +} + +#topbar .title { + position: relative; + top: 24px; + left: 24px; + color: white; + font-family: sans-serif; + font-weight: bold; +} + +#main { + max-width:1100px; + margin:auto; +} + +#main h2 { + text-align: left; +} + +#main table { + width:100%; + border-collapse: collapse; + font-size: small; + font-family: 'SF Mono', 'Menlo', 'Monaco', 'Courier New', Courier, monospace; +} +#main table tr td { + border: solid black 1px; + padding: 5px; +} + +#main table tr td.hash { + text-align: right; + border-left: none; + font-size: x-small; +} + +#main table tr td.name { + text-align: left; + border-right: none; +} + +p.description +{ + font-size: smaller; +} + +#metadata { + margin-top: 15px; + padding: 15px; + font-family: 'SF Mono', 'Menlo', 'Monaco', 'Courier New', Courier, monospace; +} + +#metadata span.fieldname { + font-family: sans-serif; +}
\ No newline at end of file diff --git a/tools/package_hosting/upload_web_assets.sh b/tools/package_hosting/upload_web_assets.sh new file mode 100755 index 0000000000..dcf258e38e --- /dev/null +++ b/tools/package_hosting/upload_web_assets.sh @@ -0,0 +1,30 @@ +#!/bin/bash +# Copyright 2018 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +cd "$(dirname "$0")" + +GCS_WEB_ASSETS=gs://packages.grpc.io/web-assets/ + +WEB_ASSETS=( + 404.html + build-201807.xsl + dirindex.css + home.xsl + style.css +) + +gsutil -m cp "${WEB_ASSETS[@]}" "$GCS_WEB_ASSETS" diff --git a/tools/profiling/ios_bin/binary_diff.py b/tools/profiling/ios_bin/binary_size.py index 6d5ae65e46..d4d134fef3 100755 --- a/tools/profiling/ios_bin/binary_diff.py +++ b/tools/profiling/ios_bin/binary_size.py @@ -28,6 +28,9 @@ sys.path.append( os.path.dirname(sys.argv[0]), '..', '..', 'run_tests', 'python_utils')) import comment_on_pr +# Only show diff 1KB or greater +diff_threshold = 1000 + size_labels = ('Core', 'ObjC', 'BoringSSL', 'Protobuf', 'Total') argp = argparse.ArgumentParser( @@ -52,7 +55,7 @@ def dir_size(dir): def get_size(where, frameworks): - build_dir = 'src/objective-c/examples/Sample/Build-%s/' % where + build_dir = 'src/objective-c/examples/Sample/Build/Build-%s/' % where if not frameworks: link_map_filename = 'Build/Intermediates.noindex/Sample.build/Release-iphoneos/Sample.build/Sample-LinkMap-normal-arm64.txt' return parse_link_map(build_dir + link_map_filename) @@ -72,18 +75,20 @@ def get_size(where, frameworks): def build(where, frameworks): + subprocess.check_call(['make', 'clean']) shutil.rmtree( - 'src/objective-c/examples/Sample/Build-%s' % where, ignore_errors=True) + 'src/objective-c/examples/Sample/Build/Build-%s' % where, + ignore_errors=True) subprocess.check_call( 'CONFIG=opt EXAMPLE_PATH=src/objective-c/examples/Sample SCHEME=Sample FRAMEWORKS=%s ./build_one_example.sh' % ('YES' if frameworks else 'NO'), shell=True, cwd='src/objective-c/tests') - os.rename('src/objective-c/examples/Sample/Build', - 'src/objective-c/examples/Sample/Build-%s' % where) + os.rename('src/objective-c/examples/Sample/Build/Build', + 'src/objective-c/examples/Sample/Build/Build-%s' % where) -text = '' +text = 'Objective-C binary sizes\n' for frameworks in [False, True]: build('new', frameworks) new_size = get_size('new', frameworks) @@ -104,22 +109,40 @@ for frameworks in [False, True]: subprocess.check_call(['git', 'checkout', where_am_i]) subprocess.check_call(['git', 'submodule', 'update']) - text += ('****************FRAMEWORKS*****************\n' - if frameworks else '******************STATIC*******************\n') + text += ('***************FRAMEWORKS****************\n' + if frameworks else '*****************STATIC******************\n') row_format = "{:>10}{:>15}{:>15}" + '\n' text += row_format.format('New size', '', 'Old size') - for i in range(0, len(size_labels)): - if old_size == None: - diff_sign = ' ' - elif new_size[i] == old_size[i]: - diff_sign = ' (=)' - elif new_size[i] > old_size[i]: + if old_size == None: + for i in range(0, len(size_labels)): + text += ('\n' + if i == len(size_labels) - 1 else '') + row_format.format( + '{:,}'.format(new_size[i]), size_labels[i], '') + else: + has_diff = False + for i in range(0, len(size_labels) - 1): + if abs(new_size[i] - old_size[i]) < diff_threshold: + continue + if new_size[i] > old_size[i]: + diff_sign = ' (>)' + else: + diff_sign = ' (<)' + has_diff = True + text += row_format.format('{:,}'.format(new_size[i]), + size_labels[i] + diff_sign, '{:,}'.format( + old_size[i])) + i = len(size_labels) - 1 + if new_size[i] > old_size[i]: diff_sign = ' (>)' - else: + elif new_size[i] < old_size[i]: diff_sign = ' (<)' - text += ('\n' if i == len(size_labels) - 1 else '') + row_format.format( + else: + diff_sign = ' (=)' + text += ('\n' if has_diff else '') + row_format.format( '{:,}'.format(new_size[i]), size_labels[i] + diff_sign, - '{:,}'.format(old_size[i]) if old_size != None else '') + '{:,}'.format(old_size[i])) + if not has_diff: + text += '\n No significant differences in binary sizes\n' text += '\n' print text diff --git a/tools/run_tests/artifacts/artifact_targets.py b/tools/run_tests/artifacts/artifact_targets.py index edde3eae28..bdeb258e1f 100644 --- a/tools/run_tests/artifacts/artifact_targets.py +++ b/tools/run_tests/artifacts/artifact_targets.py @@ -212,11 +212,15 @@ class RubyArtifact: class CSharpExtArtifact: """Builds C# native extension library""" - def __init__(self, platform, arch): + def __init__(self, platform, arch, arch_abi=None): self.name = 'csharp_ext_%s_%s' % (platform, arch) self.platform = platform self.arch = arch + self.arch_abi = arch_abi self.labels = ['artifact', 'csharp', platform, arch] + if arch_abi: + self.name += '_%s' % arch_abi + self.labels.append(arch_abi) def pre_build_jobspecs(self): return [] @@ -227,7 +231,14 @@ class CSharpExtArtifact: self.name, 'tools/dockerfile/grpc_artifact_android_ndk', 'tools/run_tests/artifacts/build_artifact_csharp_android.sh', - environ={}) + environ={ + 'ANDROID_ABI': self.arch_abi + }) + elif self.arch == 'ios': + return create_jobspec( + self.name, + ['tools/run_tests/artifacts/build_artifact_csharp_ios.sh'], + use_workspace=True) elif self.platform == 'windows': cmake_arch_option = 'Win32' if self.arch == 'x86' else self.arch return create_jobspec( @@ -279,15 +290,9 @@ class PHPArtifact: return [] def build_jobspec(self): - if self.platform == 'linux': - return create_docker_jobspec( - self.name, 'tools/dockerfile/grpc_artifact_linux_{}'.format( - self.arch), - 'tools/run_tests/artifacts/build_artifact_php.sh') - else: - return create_jobspec( - self.name, ['tools/run_tests/artifacts/build_artifact_php.sh'], - use_workspace=True) + return create_docker_jobspec( + self.name, 'tools/dockerfile/grpc_artifact_linux_{}'.format( + self.arch), 'tools/run_tests/artifacts/build_artifact_php.sh') class ProtocArtifact: @@ -348,7 +353,10 @@ def targets(): for Cls in (CSharpExtArtifact, ProtocArtifact) for platform in ('linux', 'macos', 'windows') for arch in ('x86', 'x64') ] + [ - CSharpExtArtifact('linux', 'android'), + CSharpExtArtifact('linux', 'android', arch_abi='arm64-v8a'), + CSharpExtArtifact('linux', 'android', arch_abi='armeabi-v7a'), + CSharpExtArtifact('linux', 'android', arch_abi='x86'), + CSharpExtArtifact('macos', 'ios'), PythonArtifact('linux', 'x86', 'cp27-cp27m'), PythonArtifact('linux', 'x86', 'cp27-cp27mu'), PythonArtifact('linux', 'x86', 'cp34-cp34m'), @@ -378,6 +386,7 @@ def targets(): PythonArtifact('windows', 'x86', 'Python34_32bits'), PythonArtifact('windows', 'x86', 'Python35_32bits'), PythonArtifact('windows', 'x86', 'Python36_32bits'), + PythonArtifact('windows', 'x86', 'Python37_32bits'), PythonArtifact('windows', 'x64', 'Python27'), PythonArtifact('windows', 'x64', 'Python34'), PythonArtifact('windows', 'x64', 'Python35'), @@ -385,6 +394,5 @@ def targets(): PythonArtifact('windows', 'x64', 'Python37'), RubyArtifact('linux', 'x64'), RubyArtifact('macos', 'x64'), - PHPArtifact('linux', 'x64'), - PHPArtifact('macos', 'x64') + PHPArtifact('linux', 'x64') ]) diff --git a/tools/run_tests/artifacts/build_artifact_csharp_android.sh b/tools/run_tests/artifacts/build_artifact_csharp_android.sh index ba598e76a4..067eb30edc 100755 --- a/tools/run_tests/artifacts/build_artifact_csharp_android.sh +++ b/tools/run_tests/artifacts/build_artifact_csharp_android.sh @@ -17,6 +17,7 @@ set -ex cd "$(dirname "$0")/../../.." +# ANDROID_ABI is set by the job definition in artifact_targets.py src/csharp/experimental/build_native_ext_for_android.sh mkdir -p "${ARTIFACTS_OUT}" diff --git a/tools/run_tests/artifacts/build_artifact_csharp_ios.sh b/tools/run_tests/artifacts/build_artifact_csharp_ios.sh new file mode 100755 index 0000000000..c902a45db7 --- /dev/null +++ b/tools/run_tests/artifacts/build_artifact_csharp_ios.sh @@ -0,0 +1,23 @@ +#!/bin/bash +# Copyright 2018 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +cd "$(dirname "$0")/../../.." + +src/csharp/experimental/build_native_ext_for_ios.sh + +mkdir -p "${ARTIFACTS_OUT}" +cp libs/ios/libgrpc_csharp_ext.a libs/ios/libgrpc.a "${ARTIFACTS_OUT}" diff --git a/tools/run_tests/artifacts/build_artifact_protoc.sh b/tools/run_tests/artifacts/build_artifact_protoc.sh index b531fc9a5d..6d433f2dad 100755 --- a/tools/run_tests/artifacts/build_artifact_protoc.sh +++ b/tools/run_tests/artifacts/build_artifact_protoc.sh @@ -13,8 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Use devtoolset environment that has GCC 4.7 before set -ex -source scl_source enable devtoolset-1.1 +# Use devtoolset environment that has GCC 4.8 before set -ex +source scl_source enable devtoolset-2 set -ex diff --git a/tools/run_tests/artifacts/build_package_php.sh b/tools/run_tests/artifacts/build_package_php.sh index 9a8f25a6f8..e263e09a35 100755 --- a/tools/run_tests/artifacts/build_package_php.sh +++ b/tools/run_tests/artifacts/build_package_php.sh @@ -20,7 +20,4 @@ cd "$(dirname "$0")/../../.." # All the PHP packages have been built in the artifact phase already # and we only collect them here to deliver them to the distribtest phase. mkdir -p artifacts/ -# Jenkins flow (deprecated) -cp -r "${EXTERNAL_GIT_ROOT}"/platform={windows,linux,macos}/artifacts/php_*/* artifacts/ || true -# Kokoro flow cp -r "${EXTERNAL_GIT_ROOT}"/input_artifacts/php_*/* artifacts/ || true diff --git a/tools/run_tests/artifacts/build_package_python.sh b/tools/run_tests/artifacts/build_package_python.sh index 15627881e8..29801a5b86 100755 --- a/tools/run_tests/artifacts/build_package_python.sh +++ b/tools/run_tests/artifacts/build_package_python.sh @@ -21,9 +21,6 @@ mkdir -p artifacts/ # All the python packages have been built in the artifact phase already # and we only collect them here to deliver them to the distribtest phase. -# Jenkins flow (deprecated) -cp -r "${EXTERNAL_GIT_ROOT}"/platform={windows,linux,macos}/artifacts/python_*/* artifacts/ || true -# Kokoro flow cp -r "${EXTERNAL_GIT_ROOT}"/input_artifacts/python_*/* artifacts/ || true # TODO: all the artifact builder configurations generate a grpcio-VERSION.tar.gz diff --git a/tools/run_tests/artifacts/build_package_ruby.sh b/tools/run_tests/artifacts/build_package_ruby.sh index 4f74f08276..bd62ec1ff4 100755 --- a/tools/run_tests/artifacts/build_package_ruby.sh +++ b/tools/run_tests/artifacts/build_package_ruby.sh @@ -23,9 +23,6 @@ mkdir -p artifacts/ # All the ruby packages have been built in the artifact phase already # and we only collect them here to deliver them to the distribtest phase. -# Jenkins flow (deprecated) -cp -r "${EXTERNAL_GIT_ROOT}"/platform={windows,linux,macos}/artifacts/ruby_native_gem_*/* artifacts/ || true -# Kokoro flow cp -r "${EXTERNAL_GIT_ROOT}"/input_artifacts/ruby_native_gem_*/* artifacts/ || true well_known_protos=( any api compiler/plugin descriptor duration empty field_mask source_context struct timestamp type wrappers ) @@ -44,12 +41,7 @@ for arch in {x86,x64}; do ;; esac for plat in {windows,linux,macos}; do - if [ "${KOKORO_JOB_NAME}" != "" ] - then - input_dir="${EXTERNAL_GIT_ROOT}/input_artifacts/protoc_${plat}_${arch}" - else - input_dir="${EXTERNAL_GIT_ROOT}/platform=${plat}/artifacts/protoc_${plat}_${arch}" - fi + input_dir="${EXTERNAL_GIT_ROOT}/input_artifacts/protoc_${plat}_${arch}" output_dir="$base/src/ruby/tools/bin/${ruby_arch}-${plat}" mkdir -p "$output_dir"/google/protobuf mkdir -p "$output_dir"/google/protobuf/compiler # needed for plugin.proto diff --git a/tools/run_tests/artifacts/package_targets.py b/tools/run_tests/artifacts/package_targets.py index abf1b5ebed..70f1fc8a4b 100644 --- a/tools/run_tests/artifacts/package_targets.py +++ b/tools/run_tests/artifacts/package_targets.py @@ -56,7 +56,8 @@ def create_jobspec(name, cwd=None, shell=False, flake_retries=0, - timeout_retries=0): + timeout_retries=0, + cpu_cost=1.0): """Creates jobspec.""" jobspec = jobset.JobSpec( cmdline=cmdline, @@ -66,31 +67,36 @@ def create_jobspec(name, timeout_seconds=10 * 60, flake_retries=flake_retries, timeout_retries=timeout_retries, + cpu_cost=cpu_cost, shell=shell) return jobspec class CSharpPackage: - """Builds C# nuget packages.""" - - def __init__(self, linux=False): - self.linux = linux - self.labels = ['package', 'csharp'] - if linux: - self.name = 'csharp_package_dotnetcli_linux' - self.labels += ['linux'] + """Builds C# packages.""" + + def __init__(self, unity=False): + self.unity = unity + self.labels = ['package', 'csharp', 'windows'] + if unity: + self.name = 'csharp_package_unity_windows' + self.labels += ['unity'] else: - self.name = 'csharp_package_dotnetcli_windows' - self.labels += ['windows'] + self.name = 'csharp_package_nuget_windows' + self.labels += ['nuget'] def pre_build_jobspecs(self): return [] def build_jobspec(self): - if self.linux: - return create_docker_jobspec( - self.name, 'tools/dockerfile/test/csharp_jessie_x64', - 'src/csharp/build_packages_dotnetcli.sh') + if self.unity: + # use very high CPU cost to avoid running nuget package build + # and unity build concurrently + return create_jobspec( + self.name, ['build_unitypackage.bat'], + cwd='src\\csharp', + cpu_cost=1e6, + shell=True) else: return create_jobspec( self.name, ['build_packages_dotnetcli.bat'], @@ -153,7 +159,7 @@ def targets(): """Gets list of supported targets""" return [ CSharpPackage(), - CSharpPackage(linux=True), + CSharpPackage(unity=True), RubyPackage(), PythonPackage(), PHPPackage() diff --git a/tools/run_tests/dockerize/build_and_run_docker.sh b/tools/run_tests/dockerize/build_and_run_docker.sh index 4ef74085f9..3f01fbc7b7 100755 --- a/tools/run_tests/dockerize/build_and_run_docker.sh +++ b/tools/run_tests/dockerize/build_and_run_docker.sh @@ -73,6 +73,10 @@ docker run \ # Copy output artifacts if [ "$OUTPUT_DIR" != "" ] then + # Create the artifact directory in advance to avoid a race in "docker cp" if tasks + # that were running in parallel finish at the same time. + # see https://github.com/grpc/grpc/issues/16155 + mkdir -p "$git_root/$OUTPUT_DIR" docker cp "$CONTAINER_NAME:/var/local/git/grpc/$OUTPUT_DIR" "$git_root" || FAILED="true" fi diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json index ef6141dfb1..b9f4de6171 100644 --- a/tools/run_tests/generated/sources_and_headers.json +++ b/tools/run_tests/generated/sources_and_headers.json @@ -1059,6 +1059,23 @@ "headers": [], "is_filegroup": false, "language": "c", + "name": "grpc_ipv6_loopback_available_test", + "src": [ + "test/core/iomgr/grpc_ipv6_loopback_available_test.cc" + ], + "third_party": false, + "type": "target" + }, + { + "deps": [ + "gpr", + "gpr_test_util", + "grpc", + "grpc_test_util" + ], + "headers": [], + "is_filegroup": false, + "language": "c", "name": "grpc_json_token_test", "src": [ "test/core/security/json_token_test.cc" @@ -3141,10 +3158,10 @@ "gpr_test_util", "grpc", "grpc++", - "grpc++_channelz_proto", "grpc++_test", "grpc++_test_util", - "grpc_test_util" + "grpc_test_util", + "grpcpp_channelz_proto" ], "headers": [], "is_filegroup": false, @@ -3182,10 +3199,31 @@ "gpr_test_util", "grpc", "grpc++", - "grpc++_channelz_proto", + "grpc++_test_util", + "grpc_test_util", + "grpcpp_channelz", + "grpcpp_channelz_proto" + ], + "headers": [], + "is_filegroup": false, + "language": "c++", + "name": "channelz_service_test", + "src": [ + "test/cpp/end2end/channelz_service_test.cc" + ], + "third_party": false, + "type": "target" + }, + { + "deps": [ + "gpr", + "gpr_test_util", + "grpc", + "grpc++", "grpc++_test", "grpc++_test_util", - "grpc_test_util" + "grpc_test_util", + "grpcpp_channelz_proto" ], "headers": [], "is_filegroup": false, @@ -3688,6 +3726,23 @@ }, { "deps": [ + "gpr", + "gpr_test_util", + "grpc", + "grpc_test_util" + ], + "headers": [], + "is_filegroup": false, + "language": "c++", + "name": "grpc_linux_system_roots_test", + "src": [ + "test/core/security/linux_system_roots_test.cc" + ], + "third_party": false, + "type": "target" + }, + { + "deps": [ "grpc_plugin_support" ], "headers": [], @@ -7549,6 +7604,28 @@ "deps": [ "grpc", "grpc++", + "grpcpp_channelz_proto" + ], + "headers": [ + "include/grpcpp/ext/channelz_service_plugin.h", + "src/cpp/server/channelz/channelz_service.h" + ], + "is_filegroup": false, + "language": "c++", + "name": "grpcpp_channelz", + "src": [ + "include/grpcpp/ext/channelz_service_plugin.h", + "src/cpp/server/channelz/channelz_service.cc", + "src/cpp/server/channelz/channelz_service.h", + "src/cpp/server/channelz/channelz_service_plugin.cc" + ], + "third_party": false, + "type": "lib" + }, + { + "deps": [ + "grpc", + "grpc++", "grpc++_test_config", "grpc++_test_util", "grpc_test_util" @@ -7735,6 +7812,7 @@ "test/cpp/qps/histogram.h", "test/cpp/qps/interarrival.h", "test/cpp/qps/parse_json.h", + "test/cpp/qps/qps_server_builder.h", "test/cpp/qps/qps_worker.h", "test/cpp/qps/report.h", "test/cpp/qps/server.h", @@ -7756,6 +7834,8 @@ "test/cpp/qps/interarrival.h", "test/cpp/qps/parse_json.cc", "test/cpp/qps/parse_json.h", + "test/cpp/qps/qps_server_builder.cc", + "test/cpp/qps/qps_server_builder.h", "test/cpp/qps/qps_worker.cc", "test/cpp/qps/qps_worker.h", "test/cpp/qps/report.cc", @@ -9864,6 +9944,7 @@ "src/core/lib/iomgr/endpoint_cfstream.h", "src/core/lib/iomgr/error_cfstream.cc", "src/core/lib/iomgr/error_cfstream.h", + "src/core/lib/iomgr/iomgr_posix_cfstream.cc", "src/core/lib/iomgr/tcp_client_cfstream.cc" ], "third_party": false, @@ -10231,9 +10312,12 @@ "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.cc", "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.h", "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.cc", + "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_windows.cc", "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.cc", "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.h", - "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_fallback.cc" + "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_fallback.cc", + "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_posix.cc", + "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_windows.cc" ], "third_party": false, "type": "filegroup" @@ -10315,6 +10399,8 @@ "src/core/lib/security/credentials/plugin/plugin_credentials.h", "src/core/lib/security/credentials/ssl/ssl_credentials.h", "src/core/lib/security/security_connector/alts_security_connector.h", + "src/core/lib/security/security_connector/load_system_roots.h", + "src/core/lib/security/security_connector/load_system_roots_linux.h", "src/core/lib/security/security_connector/local_security_connector.h", "src/core/lib/security/security_connector/security_connector.h", "src/core/lib/security/transport/auth_filters.h", @@ -10363,6 +10449,10 @@ "src/core/lib/security/credentials/ssl/ssl_credentials.h", "src/core/lib/security/security_connector/alts_security_connector.cc", "src/core/lib/security/security_connector/alts_security_connector.h", + "src/core/lib/security/security_connector/load_system_roots.h", + "src/core/lib/security/security_connector/load_system_roots_fallback.cc", + "src/core/lib/security/security_connector/load_system_roots_linux.cc", + "src/core/lib/security/security_connector/load_system_roots_linux.h", "src/core/lib/security/security_connector/local_security_connector.cc", "src/core/lib/security/security_connector/local_security_connector.h", "src/core/lib/security/security_connector/security_connector.cc", @@ -10912,20 +11002,6 @@ "type": "filegroup" }, { - "deps": [], - "headers": [ - "src/proto/grpc/channelz/channelz.grpc.pb.h", - "src/proto/grpc/channelz/channelz.pb.h", - "src/proto/grpc/channelz/channelz_mock.grpc.pb.h" - ], - "is_filegroup": true, - "language": "c++", - "name": "grpc++_channelz_proto", - "src": [], - "third_party": false, - "type": "filegroup" - }, - { "deps": [ "grpc_codegen" ], @@ -11400,5 +11476,19 @@ ], "third_party": false, "type": "filegroup" + }, + { + "deps": [], + "headers": [ + "src/proto/grpc/channelz/channelz.grpc.pb.h", + "src/proto/grpc/channelz/channelz.pb.h", + "src/proto/grpc/channelz/channelz_mock.grpc.pb.h" + ], + "is_filegroup": true, + "language": "c++", + "name": "grpcpp_channelz_proto", + "src": [], + "third_party": false, + "type": "filegroup" } ] diff --git a/tools/run_tests/generated/tests.json b/tools/run_tests/generated/tests.json index 3abca8efd8..26e43e350c 100644 --- a/tools/run_tests/generated/tests.json +++ b/tools/run_tests/generated/tests.json @@ -1339,6 +1339,32 @@ "ci_platforms": [ "linux", "mac", + "posix", + "windows" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [ + "uv" + ], + "flaky": false, + "gtest": false, + "language": "c", + "name": "grpc_ipv6_loopback_available_test", + "platforms": [ + "linux", + "mac", + "posix", + "windows" + ], + "uses_polling": true + }, + { + "args": [], + "benchmark": false, + "ci_platforms": [ + "linux", + "mac", "posix" ], "cpu_cost": 1.0, @@ -3790,6 +3816,30 @@ "flaky": false, "gtest": true, "language": "c++", + "name": "channelz_service_test", + "platforms": [ + "linux", + "mac", + "posix", + "windows" + ], + "uses_polling": true + }, + { + "args": [], + "benchmark": false, + "ci_platforms": [ + "linux", + "mac", + "posix", + "windows" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "gtest": true, + "language": "c++", "name": "channelz_test", "platforms": [ "linux", @@ -4318,6 +4368,30 @@ "flaky": false, "gtest": true, "language": "c++", + "name": "grpc_linux_system_roots_test", + "platforms": [ + "linux", + "mac", + "posix", + "windows" + ], + "uses_polling": true + }, + { + "args": [], + "benchmark": false, + "ci_platforms": [ + "linux", + "mac", + "posix", + "windows" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "gtest": true, + "language": "c++", "name": "grpc_tool_test", "platforms": [ "linux", @@ -5706,7 +5780,8 @@ "ci_platforms": [ "linux", "mac", - "posix" + "posix", + "windows" ], "cpu_cost": 1.0, "exclude_configs": [], @@ -5718,7 +5793,8 @@ "platforms": [ "linux", "mac", - "posix" + "posix", + "windows" ], "uses_polling": true }, @@ -5728,7 +5804,8 @@ "ci_platforms": [ "linux", "mac", - "posix" + "posix", + "windows" ], "cpu_cost": 1.0, "exclude_configs": [], @@ -5740,7 +5817,8 @@ "platforms": [ "linux", "mac", - "posix" + "posix", + "windows" ], "uses_polling": true }, @@ -5750,7 +5828,8 @@ "ci_platforms": [ "linux", "mac", - "posix" + "posix", + "windows" ], "cpu_cost": 1.0, "exclude_configs": [], @@ -5762,7 +5841,8 @@ "platforms": [ "linux", "mac", - "posix" + "posix", + "windows" ], "uses_polling": true }, diff --git a/tools/run_tests/performance/OWNERS b/tools/run_tests/performance/OWNERS index fc1d6eec9c..9cf8c13111 100644 --- a/tools/run_tests/performance/OWNERS +++ b/tools/run_tests/performance/OWNERS @@ -5,5 +5,5 @@ set noparent # to update the BigQuery schema @ncteisen -@matt-kwong +@apolcyn @jtattermusch diff --git a/tools/run_tests/performance/README.md b/tools/run_tests/performance/README.md index 2fc1a27c9b..791270ab38 100644 --- a/tools/run_tests/performance/README.md +++ b/tools/run_tests/performance/README.md @@ -104,3 +104,31 @@ Example memory profile of grpc-go server, with `go tools pprof`: ``` $ go tool pprof --text --alloc_space http://localhost:<pprof_port>/debug/heap ``` + +### Configuration environment variables: + +* QPS_WORKER_CHANNEL_CONNECT_TIMEOUT + + Consuming process: qps_worker + + Type: integer (number of seconds) + + This can be used to configure the amount of time that benchmark + clients wait for channels to the benchmark server to become ready. + This is useful in certain benchmark environments in which the + server can take a long time to become ready. Note: if setting + this to a high value, then the scenario config under test should + probably also have a large "warmup_seconds". + +* QPS_WORKERS + + Consuming process: qps_json_driver + + Type: comma separated list of host:port + + Set this to a comma separated list of QPS worker processes/machines. + Each scenario in a scenario config has specifies a certain number + of servers, `num_servers`, and the driver will start + "benchmark servers"'s on the first `num_server` `host:port` pairs in + the comma separated list. The rest will be told to run as clients + against the benchmark server. diff --git a/tools/run_tests/run_interop_tests.py b/tools/run_tests/run_interop_tests.py index aa58107ced..22055d58e8 100755 --- a/tools/run_tests/run_interop_tests.py +++ b/tools/run_tests/run_interop_tests.py @@ -637,13 +637,13 @@ _LANGUAGES_WITH_HTTP2_CLIENTS_FOR_HTTP2_SERVER_TEST_CASES = [ 'java', 'go', 'python', 'c++' ] -#TODO: Add c++ when c++ ALTS interop client is ready. _LANGUAGES_FOR_ALTS_TEST_CASES = ['java', 'go', 'c++'] -#TODO: Add c++ when c++ ALTS interop server is ready. _SERVERS_FOR_ALTS_TEST_CASES = ['java', 'go', 'c++'] -_TRANSPORT_SECURITY_OPTIONS = ['tls', 'alts', 'insecure'] +_TRANSPORT_SECURITY_OPTIONS = [ + 'tls', 'alts', 'google_default_credentials', 'insecure' +] DOCKER_WORKDIR_ROOT = '/var/local/git/grpc' @@ -724,6 +724,9 @@ def auth_options(language, test_case, service_account_key_file=None): key_file_arg = '--service_account_key_file=%s' % service_account_key_file default_account_arg = '--default_service_account=830293263384-compute@developer.gserviceaccount.com' + # TODO: When using google_default_credentials outside of cloud-to-prod, the environment variable + # 'GOOGLE_APPLICATION_CREDENTIALS' needs to be set for the test case + # 'jwt_token_creds' to work. if test_case in ['jwt_token_creds', 'per_rpc_creds', 'oauth2_auth_token']: if language in [ 'csharp', 'csharpcoreclr', 'node', 'php', 'php7', 'python', @@ -763,15 +766,25 @@ def cloud_to_prod_jobspec(language, docker_image=None, auth=False, manual_cmd_log=None, - service_account_key_file=None): + service_account_key_file=None, + transport_security='tls'): """Creates jobspec for cloud-to-prod interop test""" container_name = None cmdargs = [ '--server_host=%s' % server_host, '--server_host_override=%s' % server_host, '--server_port=443', - '--use_tls=true', '--test_case=%s' % test_case ] + if transport_security == 'tls': + transport_security_options += ['--use_tls=true'] + elif transport_security == 'google_default_credentials' and language == 'c++': + transport_security_options += [ + '--custom_credentials_type=google_default_credentials' + ] + else: + print('Invalid transport security option.') + sys.exit(1) + cmdargs = cmdargs + transport_security_options environ = dict(language.cloud_to_prod_env(), **language.global_env()) if auth: auth_cmdargs, auth_env = auth_options(language, test_case, @@ -1285,14 +1298,16 @@ try: jobs = [] if args.cloud_to_prod: - if args.transport_security != 'tls': - print('TLS is always enabled for cloud_to_prod scenarios.') + if args.transport_security not in ['tls', 'google_default_credentials']: + print( + 'TLS or google default credential is always enabled for cloud_to_prod scenarios.' + ) for server_host_nickname in args.prod_servers: for language in languages: for test_case in _TEST_CASES: if not test_case in language.unimplemented_test_cases(): if not test_case in _SKIP_ADVANCED + _SKIP_COMPRESSION: - test_job = cloud_to_prod_jobspec( + tls_test_job = cloud_to_prod_jobspec( language, test_case, server_host_nickname, @@ -1300,8 +1315,23 @@ try: docker_image=docker_images.get(str(language)), manual_cmd_log=client_manual_cmd_log, service_account_key_file=args. - service_account_key_file) - jobs.append(test_job) + service_account_key_file, + transport_security='tls') + jobs.append(tls_test_job) + if language == 'c++': + google_default_creds_test_job = cloud_to_prod_jobspec( + language, + test_case, + server_host_nickname, + prod_servers[server_host_nickname], + docker_image=docker_images.get( + str(language)), + manual_cmd_log=client_manual_cmd_log, + service_account_key_file=args. + service_account_key_file, + transport_security= + 'google_default_credentials') + jobs.append(google_default_creds_test_job) if args.http2_interop: for test_case in _HTTP2_TEST_CASES: @@ -1312,12 +1342,15 @@ try: prod_servers[server_host_nickname], docker_image=docker_images.get(str(http2Interop)), manual_cmd_log=client_manual_cmd_log, - service_account_key_file=args.service_account_key_file) + service_account_key_file=args.service_account_key_file, + transport_security=args.transport_security) jobs.append(test_job) if args.cloud_to_prod_auth: - if args.transport_security != 'tls': - print('TLS is always enabled for cloud_to_prod scenarios.') + if args.transport_security not in ['tls', 'google_default_credentials']: + print( + 'TLS or google default credential is always enabled for cloud_to_prod scenarios.' + ) for server_host_nickname in args.prod_servers: for language in languages: for test_case in _AUTH_TEST_CASES: @@ -1325,7 +1358,7 @@ try: not compute_engine_creds_required( language, test_case)): if not test_case in language.unimplemented_test_cases(): - test_job = cloud_to_prod_jobspec( + tls_test_job = cloud_to_prod_jobspec( language, test_case, server_host_nickname, @@ -1334,8 +1367,23 @@ try: auth=True, manual_cmd_log=client_manual_cmd_log, service_account_key_file=args. - service_account_key_file) - jobs.append(test_job) + service_account_key_file, + transport_security='tls') + jobs.append(tls_test_job) + if language == 'c++': + google_default_creds_test_job = cloud_to_prod_jobspec( + language, + test_case, + server_host_nickname, + prod_servers[server_host_nickname], + docker_image=docker_images.get( + str(language)), + manual_cmd_log=client_manual_cmd_log, + service_account_key_file=args. + service_account_key_file, + transport_security= + 'google_default_credentials') + jobs.append(google_default_creds_test_job) for server in args.override_server: server_name = server[0] diff --git a/tools/run_tests/sanity/check_submodules.sh b/tools/run_tests/sanity/check_submodules.sh index 4fe1aa8450..2c7c140716 100755 --- a/tools/run_tests/sanity/check_submodules.sh +++ b/tools/run_tests/sanity/check_submodules.sh @@ -29,14 +29,14 @@ cat << EOF | awk '{ print $1 }' | sort > "$want_submodules" cc4bed2d74f7c8717e31f9579214ab52a9c9c610 third_party/abseil-cpp (cc4bed2) 5b7683f49e1e9223cf9927b24f6fd3d6bd82e3f8 third_party/benchmark (v1.2.0) 73594cde8c9a52a102c4341c244c833aa61b9c06 third_party/bloaty (remotes/origin/wide-14-g73594cd) - 70ef9596bbcc11353b9bb8d4e91478694dd21439 third_party/boringssl (fips-20170615-704-g70ef959) - dcd3e6e6ecddf059adb48fca45bc7346a108bdd9 third_party/boringssl-with-bazel (version_for_cocoapods_10.0-369-gdcd3e6e) + b29b21a81b32ec273f118f589f46d56ad3332420 third_party/boringssl (remotes/origin/chromium-stable) + 8149b351bf797bd80e063787886b7618f508e451 third_party/boringssl-with-bazel (version_for_cocoapods_10.0-434-g8149b351) 3be1924221e1326df520f8498d704a5c4c8d0cce third_party/cares/cares (cares-1_13_0) 30dbc81fb5ffdc98ea9b14b1918bfe4e8779b26e third_party/gflags (v2.2.0-5-g30dbc81) ec44c6c1675c25b9827aacd08c02433cccde7780 third_party/googletest (release-1.8.0) 6599cac0965be8e5a835ab7a5684bbef033d5ad0 third_party/libcxx (heads/release_60) 9245d481eb3e890f708ff2d7dadf2a10c04748ba third_party/libcxxabi (heads/release_60) - b5fbb742af122b565925987e65c08957739976a7 third_party/protobuf (v3.3.1-641-gb5fbb74) + 48cb18e5c419ddd23d9badcfe4e9df7bde1979b2 third_party/protobuf (v3.6.0.1-37-g48cb18e5) cacf7f1d4e3d44d871b605da3b647f07d718623f third_party/zlib (v1.2.11) EOF |