diff options
Diffstat (limited to 'tools')
24 files changed, 588 insertions, 144 deletions
diff --git a/tools/distrib/python/grpcio_tools/protoc_lib_deps.py b/tools/distrib/python/grpcio_tools/protoc_lib_deps.py index cd4effa7ae..569328e57e 100644 --- a/tools/distrib/python/grpcio_tools/protoc_lib_deps.py +++ b/tools/distrib/python/grpcio_tools/protoc_lib_deps.py @@ -29,7 +29,7 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # AUTO-GENERATED BY make_grpcio_tools.py! -CC_FILES=['google/protobuf/compiler/zip_writer.cc', 'google/protobuf/compiler/subprocess.cc', 'google/protobuf/compiler/ruby/ruby_generator.cc', 'google/protobuf/compiler/python/python_generator.cc', 'google/protobuf/compiler/plugin.pb.cc', 'google/protobuf/compiler/plugin.cc', 'google/protobuf/compiler/objectivec/objectivec_primitive_field.cc', 'google/protobuf/compiler/objectivec/objectivec_oneof.cc', 'google/protobuf/compiler/objectivec/objectivec_message_field.cc', 'google/protobuf/compiler/objectivec/objectivec_message.cc', 'google/protobuf/compiler/objectivec/objectivec_map_field.cc', 'google/protobuf/compiler/objectivec/objectivec_helpers.cc', 'google/protobuf/compiler/objectivec/objectivec_generator.cc', 'google/protobuf/compiler/objectivec/objectivec_file.cc', 'google/protobuf/compiler/objectivec/objectivec_field.cc', 'google/protobuf/compiler/objectivec/objectivec_extension.cc', 'google/protobuf/compiler/objectivec/objectivec_enum_field.cc', 'google/protobuf/compiler/objectivec/objectivec_enum.cc', 'google/protobuf/compiler/js/js_generator.cc', 'google/protobuf/compiler/javanano/javanano_primitive_field.cc', 'google/protobuf/compiler/javanano/javanano_message_field.cc', 'google/protobuf/compiler/javanano/javanano_message.cc', 'google/protobuf/compiler/javanano/javanano_map_field.cc', 'google/protobuf/compiler/javanano/javanano_helpers.cc', 'google/protobuf/compiler/javanano/javanano_generator.cc', 'google/protobuf/compiler/javanano/javanano_file.cc', 'google/protobuf/compiler/javanano/javanano_field.cc', 'google/protobuf/compiler/javanano/javanano_extension.cc', 'google/protobuf/compiler/javanano/javanano_enum_field.cc', 'google/protobuf/compiler/javanano/javanano_enum.cc', 'google/protobuf/compiler/java/java_string_field_lite.cc', 'google/protobuf/compiler/java/java_string_field.cc', 'google/protobuf/compiler/java/java_shared_code_generator.cc', 'google/protobuf/compiler/java/java_service.cc', 'google/protobuf/compiler/java/java_primitive_field_lite.cc', 'google/protobuf/compiler/java/java_primitive_field.cc', 'google/protobuf/compiler/java/java_name_resolver.cc', 'google/protobuf/compiler/java/java_message_lite.cc', 'google/protobuf/compiler/java/java_message_field_lite.cc', 'google/protobuf/compiler/java/java_message_field.cc', 'google/protobuf/compiler/java/java_message_builder_lite.cc', 'google/protobuf/compiler/java/java_message_builder.cc', 'google/protobuf/compiler/java/java_message.cc', 'google/protobuf/compiler/java/java_map_field_lite.cc', 'google/protobuf/compiler/java/java_map_field.cc', 'google/protobuf/compiler/java/java_lazy_message_field_lite.cc', 'google/protobuf/compiler/java/java_lazy_message_field.cc', 'google/protobuf/compiler/java/java_helpers.cc', 'google/protobuf/compiler/java/java_generator_factory.cc', 'google/protobuf/compiler/java/java_generator.cc', 'google/protobuf/compiler/java/java_file.cc', 'google/protobuf/compiler/java/java_field.cc', 'google/protobuf/compiler/java/java_extension_lite.cc', 'google/protobuf/compiler/java/java_extension.cc', 'google/protobuf/compiler/java/java_enum_lite.cc', 'google/protobuf/compiler/java/java_enum_field_lite.cc', 'google/protobuf/compiler/java/java_enum_field.cc', 'google/protobuf/compiler/java/java_enum.cc', 'google/protobuf/compiler/java/java_doc_comment.cc', 'google/protobuf/compiler/java/java_context.cc', 'google/protobuf/compiler/csharp/csharp_wrapper_field.cc', 'google/protobuf/compiler/csharp/csharp_source_generator_base.cc', 'google/protobuf/compiler/csharp/csharp_repeated_primitive_field.cc', 'google/protobuf/compiler/csharp/csharp_repeated_message_field.cc', 'google/protobuf/compiler/csharp/csharp_repeated_enum_field.cc', 'google/protobuf/compiler/csharp/csharp_reflection_class.cc', 'google/protobuf/compiler/csharp/csharp_primitive_field.cc', 'google/protobuf/compiler/csharp/csharp_message_field.cc', 'google/protobuf/compiler/csharp/csharp_message.cc', 'google/protobuf/compiler/csharp/csharp_map_field.cc', 'google/protobuf/compiler/csharp/csharp_helpers.cc', 'google/protobuf/compiler/csharp/csharp_generator.cc', 'google/protobuf/compiler/csharp/csharp_field_base.cc', 'google/protobuf/compiler/csharp/csharp_enum_field.cc', 'google/protobuf/compiler/csharp/csharp_enum.cc', 'google/protobuf/compiler/csharp/csharp_doc_comment.cc', 'google/protobuf/compiler/cpp/cpp_string_field.cc', 'google/protobuf/compiler/cpp/cpp_service.cc', 'google/protobuf/compiler/cpp/cpp_primitive_field.cc', 'google/protobuf/compiler/cpp/cpp_message_field.cc', 'google/protobuf/compiler/cpp/cpp_message.cc', 'google/protobuf/compiler/cpp/cpp_map_field.cc', 'google/protobuf/compiler/cpp/cpp_helpers.cc', 'google/protobuf/compiler/cpp/cpp_generator.cc', 'google/protobuf/compiler/cpp/cpp_file.cc', 'google/protobuf/compiler/cpp/cpp_field.cc', 'google/protobuf/compiler/cpp/cpp_extension.cc', 'google/protobuf/compiler/cpp/cpp_enum_field.cc', 'google/protobuf/compiler/cpp/cpp_enum.cc', 'google/protobuf/compiler/command_line_interface.cc', 'google/protobuf/compiler/code_generator.cc', 'google/protobuf/wrappers.pb.cc', 'google/protobuf/wire_format.cc', 'google/protobuf/util/type_resolver_util.cc', 'google/protobuf/util/time_util.cc', 'google/protobuf/util/message_differencer.cc', 'google/protobuf/util/json_util.cc', 'google/protobuf/util/internal/utility.cc', 'google/protobuf/util/internal/type_info_test_helper.cc', 'google/protobuf/util/internal/type_info.cc', 'google/protobuf/util/internal/protostream_objectwriter.cc', 'google/protobuf/util/internal/protostream_objectsource.cc', 'google/protobuf/util/internal/proto_writer.cc', 'google/protobuf/util/internal/object_writer.cc', 'google/protobuf/util/internal/json_stream_parser.cc', 'google/protobuf/util/internal/json_objectwriter.cc', 'google/protobuf/util/internal/json_escaping.cc', 'google/protobuf/util/internal/field_mask_utility.cc', 'google/protobuf/util/internal/error_listener.cc', 'google/protobuf/util/internal/default_value_objectwriter.cc', 'google/protobuf/util/internal/datapiece.cc', 'google/protobuf/util/field_mask_util.cc', 'google/protobuf/util/field_comparator.cc', 'google/protobuf/unknown_field_set.cc', 'google/protobuf/type.pb.cc', 'google/protobuf/timestamp.pb.cc', 'google/protobuf/text_format.cc', 'google/protobuf/stubs/substitute.cc', 'google/protobuf/stubs/mathlimits.cc', 'google/protobuf/struct.pb.cc', 'google/protobuf/source_context.pb.cc', 'google/protobuf/service.cc', 'google/protobuf/reflection_ops.cc', 'google/protobuf/message.cc', 'google/protobuf/map_field.cc', 'google/protobuf/io/zero_copy_stream_impl.cc', 'google/protobuf/io/tokenizer.cc', 'google/protobuf/io/strtod.cc', 'google/protobuf/io/printer.cc', 'google/protobuf/io/gzip_stream.cc', 'google/protobuf/generated_message_reflection.cc', 'google/protobuf/field_mask.pb.cc', 'google/protobuf/extension_set_heavy.cc', 'google/protobuf/empty.pb.cc', 'google/protobuf/dynamic_message.cc', 'google/protobuf/duration.pb.cc', 'google/protobuf/descriptor_database.cc', 'google/protobuf/descriptor.pb.cc', 'google/protobuf/descriptor.cc', 'google/protobuf/compiler/parser.cc', 'google/protobuf/compiler/importer.cc', 'google/protobuf/api.pb.cc', 'google/protobuf/any.pb.cc', 'google/protobuf/any.cc', 'google/protobuf/wire_format_lite.cc', 'google/protobuf/stubs/time.cc', 'google/protobuf/stubs/strutil.cc', 'google/protobuf/stubs/structurally_valid.cc', 'google/protobuf/stubs/stringprintf.cc', 'google/protobuf/stubs/stringpiece.cc', 'google/protobuf/stubs/statusor.cc', 'google/protobuf/stubs/status.cc', 'google/protobuf/stubs/once.cc', 'google/protobuf/stubs/int128.cc', 'google/protobuf/stubs/common.cc', 'google/protobuf/stubs/bytestream.cc', 'google/protobuf/stubs/atomicops_internals_x86_msvc.cc', 'google/protobuf/stubs/atomicops_internals_x86_gcc.cc', 'google/protobuf/repeated_field.cc', 'google/protobuf/message_lite.cc', 'google/protobuf/io/zero_copy_stream_impl_lite.cc', 'google/protobuf/io/zero_copy_stream.cc', 'google/protobuf/io/coded_stream.cc', 'google/protobuf/generated_message_util.cc', 'google/protobuf/extension_set.cc', 'google/protobuf/arenastring.cc', 'google/protobuf/arena.cc'] +CC_FILES=['google/protobuf/compiler/zip_writer.cc', 'google/protobuf/compiler/subprocess.cc', 'google/protobuf/compiler/ruby/ruby_generator.cc', 'google/protobuf/compiler/python/python_generator.cc', 'google/protobuf/compiler/plugin.pb.cc', 'google/protobuf/compiler/plugin.cc', 'google/protobuf/compiler/php/php_generator.cc', 'google/protobuf/compiler/objectivec/objectivec_primitive_field.cc', 'google/protobuf/compiler/objectivec/objectivec_oneof.cc', 'google/protobuf/compiler/objectivec/objectivec_message_field.cc', 'google/protobuf/compiler/objectivec/objectivec_message.cc', 'google/protobuf/compiler/objectivec/objectivec_map_field.cc', 'google/protobuf/compiler/objectivec/objectivec_helpers.cc', 'google/protobuf/compiler/objectivec/objectivec_generator.cc', 'google/protobuf/compiler/objectivec/objectivec_file.cc', 'google/protobuf/compiler/objectivec/objectivec_field.cc', 'google/protobuf/compiler/objectivec/objectivec_extension.cc', 'google/protobuf/compiler/objectivec/objectivec_enum_field.cc', 'google/protobuf/compiler/objectivec/objectivec_enum.cc', 'google/protobuf/compiler/js/js_generator.cc', 'google/protobuf/compiler/javanano/javanano_primitive_field.cc', 'google/protobuf/compiler/javanano/javanano_message_field.cc', 'google/protobuf/compiler/javanano/javanano_message.cc', 'google/protobuf/compiler/javanano/javanano_map_field.cc', 'google/protobuf/compiler/javanano/javanano_helpers.cc', 'google/protobuf/compiler/javanano/javanano_generator.cc', 'google/protobuf/compiler/javanano/javanano_file.cc', 'google/protobuf/compiler/javanano/javanano_field.cc', 'google/protobuf/compiler/javanano/javanano_extension.cc', 'google/protobuf/compiler/javanano/javanano_enum_field.cc', 'google/protobuf/compiler/javanano/javanano_enum.cc', 'google/protobuf/compiler/java/java_string_field_lite.cc', 'google/protobuf/compiler/java/java_string_field.cc', 'google/protobuf/compiler/java/java_shared_code_generator.cc', 'google/protobuf/compiler/java/java_service.cc', 'google/protobuf/compiler/java/java_primitive_field_lite.cc', 'google/protobuf/compiler/java/java_primitive_field.cc', 'google/protobuf/compiler/java/java_name_resolver.cc', 'google/protobuf/compiler/java/java_message_lite.cc', 'google/protobuf/compiler/java/java_message_field_lite.cc', 'google/protobuf/compiler/java/java_message_field.cc', 'google/protobuf/compiler/java/java_message_builder_lite.cc', 'google/protobuf/compiler/java/java_message_builder.cc', 'google/protobuf/compiler/java/java_message.cc', 'google/protobuf/compiler/java/java_map_field_lite.cc', 'google/protobuf/compiler/java/java_map_field.cc', 'google/protobuf/compiler/java/java_lazy_message_field_lite.cc', 'google/protobuf/compiler/java/java_lazy_message_field.cc', 'google/protobuf/compiler/java/java_helpers.cc', 'google/protobuf/compiler/java/java_generator_factory.cc', 'google/protobuf/compiler/java/java_generator.cc', 'google/protobuf/compiler/java/java_file.cc', 'google/protobuf/compiler/java/java_field.cc', 'google/protobuf/compiler/java/java_extension_lite.cc', 'google/protobuf/compiler/java/java_extension.cc', 'google/protobuf/compiler/java/java_enum_lite.cc', 'google/protobuf/compiler/java/java_enum_field_lite.cc', 'google/protobuf/compiler/java/java_enum_field.cc', 'google/protobuf/compiler/java/java_enum.cc', 'google/protobuf/compiler/java/java_doc_comment.cc', 'google/protobuf/compiler/java/java_context.cc', 'google/protobuf/compiler/csharp/csharp_wrapper_field.cc', 'google/protobuf/compiler/csharp/csharp_source_generator_base.cc', 'google/protobuf/compiler/csharp/csharp_repeated_primitive_field.cc', 'google/protobuf/compiler/csharp/csharp_repeated_message_field.cc', 'google/protobuf/compiler/csharp/csharp_repeated_enum_field.cc', 'google/protobuf/compiler/csharp/csharp_reflection_class.cc', 'google/protobuf/compiler/csharp/csharp_primitive_field.cc', 'google/protobuf/compiler/csharp/csharp_message_field.cc', 'google/protobuf/compiler/csharp/csharp_message.cc', 'google/protobuf/compiler/csharp/csharp_map_field.cc', 'google/protobuf/compiler/csharp/csharp_helpers.cc', 'google/protobuf/compiler/csharp/csharp_generator.cc', 'google/protobuf/compiler/csharp/csharp_field_base.cc', 'google/protobuf/compiler/csharp/csharp_enum_field.cc', 'google/protobuf/compiler/csharp/csharp_enum.cc', 'google/protobuf/compiler/csharp/csharp_doc_comment.cc', 'google/protobuf/compiler/cpp/cpp_string_field.cc', 'google/protobuf/compiler/cpp/cpp_service.cc', 'google/protobuf/compiler/cpp/cpp_primitive_field.cc', 'google/protobuf/compiler/cpp/cpp_message_field.cc', 'google/protobuf/compiler/cpp/cpp_message.cc', 'google/protobuf/compiler/cpp/cpp_map_field.cc', 'google/protobuf/compiler/cpp/cpp_helpers.cc', 'google/protobuf/compiler/cpp/cpp_generator.cc', 'google/protobuf/compiler/cpp/cpp_file.cc', 'google/protobuf/compiler/cpp/cpp_field.cc', 'google/protobuf/compiler/cpp/cpp_extension.cc', 'google/protobuf/compiler/cpp/cpp_enum_field.cc', 'google/protobuf/compiler/cpp/cpp_enum.cc', 'google/protobuf/compiler/command_line_interface.cc', 'google/protobuf/compiler/code_generator.cc', 'google/protobuf/wrappers.pb.cc', 'google/protobuf/wire_format.cc', 'google/protobuf/util/type_resolver_util.cc', 'google/protobuf/util/time_util.cc', 'google/protobuf/util/message_differencer.cc', 'google/protobuf/util/json_util.cc', 'google/protobuf/util/internal/utility.cc', 'google/protobuf/util/internal/type_info_test_helper.cc', 'google/protobuf/util/internal/type_info.cc', 'google/protobuf/util/internal/protostream_objectwriter.cc', 'google/protobuf/util/internal/protostream_objectsource.cc', 'google/protobuf/util/internal/proto_writer.cc', 'google/protobuf/util/internal/object_writer.cc', 'google/protobuf/util/internal/json_stream_parser.cc', 'google/protobuf/util/internal/json_objectwriter.cc', 'google/protobuf/util/internal/json_escaping.cc', 'google/protobuf/util/internal/field_mask_utility.cc', 'google/protobuf/util/internal/error_listener.cc', 'google/protobuf/util/internal/default_value_objectwriter.cc', 'google/protobuf/util/internal/datapiece.cc', 'google/protobuf/util/field_mask_util.cc', 'google/protobuf/util/field_comparator.cc', 'google/protobuf/unknown_field_set.cc', 'google/protobuf/type.pb.cc', 'google/protobuf/timestamp.pb.cc', 'google/protobuf/text_format.cc', 'google/protobuf/stubs/substitute.cc', 'google/protobuf/stubs/mathlimits.cc', 'google/protobuf/struct.pb.cc', 'google/protobuf/source_context.pb.cc', 'google/protobuf/service.cc', 'google/protobuf/reflection_ops.cc', 'google/protobuf/message.cc', 'google/protobuf/map_field.cc', 'google/protobuf/io/zero_copy_stream_impl.cc', 'google/protobuf/io/tokenizer.cc', 'google/protobuf/io/strtod.cc', 'google/protobuf/io/printer.cc', 'google/protobuf/io/gzip_stream.cc', 'google/protobuf/generated_message_reflection.cc', 'google/protobuf/field_mask.pb.cc', 'google/protobuf/extension_set_heavy.cc', 'google/protobuf/empty.pb.cc', 'google/protobuf/dynamic_message.cc', 'google/protobuf/duration.pb.cc', 'google/protobuf/descriptor_database.cc', 'google/protobuf/descriptor.pb.cc', 'google/protobuf/descriptor.cc', 'google/protobuf/compiler/parser.cc', 'google/protobuf/compiler/importer.cc', 'google/protobuf/api.pb.cc', 'google/protobuf/any.pb.cc', 'google/protobuf/any.cc', 'google/protobuf/wire_format_lite.cc', 'google/protobuf/stubs/time.cc', 'google/protobuf/stubs/strutil.cc', 'google/protobuf/stubs/structurally_valid.cc', 'google/protobuf/stubs/stringprintf.cc', 'google/protobuf/stubs/stringpiece.cc', 'google/protobuf/stubs/statusor.cc', 'google/protobuf/stubs/status.cc', 'google/protobuf/stubs/once.cc', 'google/protobuf/stubs/int128.cc', 'google/protobuf/stubs/common.cc', 'google/protobuf/stubs/bytestream.cc', 'google/protobuf/stubs/atomicops_internals_x86_msvc.cc', 'google/protobuf/stubs/atomicops_internals_x86_gcc.cc', 'google/protobuf/repeated_field.cc', 'google/protobuf/message_lite.cc', 'google/protobuf/io/zero_copy_stream_impl_lite.cc', 'google/protobuf/io/zero_copy_stream.cc', 'google/protobuf/io/coded_stream.cc', 'google/protobuf/generated_message_util.cc', 'google/protobuf/extension_set.cc', 'google/protobuf/arenastring.cc', 'google/protobuf/arena.cc'] PROTO_FILES=['google/protobuf/wrappers.proto', 'google/protobuf/type.proto', 'google/protobuf/timestamp.proto', 'google/protobuf/struct.proto', 'google/protobuf/source_context.proto', 'google/protobuf/field_mask.proto', 'google/protobuf/empty.proto', 'google/protobuf/duration.proto', 'google/protobuf/descriptor.proto', 'google/protobuf/compiler/plugin.proto', 'google/protobuf/api.proto', 'google/protobuf/any.proto'] CC_INCLUDE='third_party/protobuf/src' diff --git a/tools/dockerfile/interoptest/grpc_interop_go/build_interop.sh b/tools/dockerfile/interoptest/grpc_interop_go/build_interop.sh index 1fd088322c..858ee0a68c 100755 --- a/tools/dockerfile/interoptest/grpc_interop_go/build_interop.sh +++ b/tools/dockerfile/interoptest/grpc_interop_go/build_interop.sh @@ -36,19 +36,12 @@ set -e # to test instead of using "go get" to download from Github directly. git clone --recursive /var/local/jenkins/grpc-go src/google.golang.org/grpc +# Get all gRPC Go dependencies +(cd src/google.golang.org/grpc && go get -t .) + # copy service account keys if available cp -r /var/local/jenkins/service_account $HOME || true -# Get dependencies from GitHub -# NOTE: once grpc-go dependencies change, this needs to be updated manually -# but we don't expect this to happen any time soon. -go get github.com/golang/protobuf/proto -go get golang.org/x/net/context -go get golang.org/x/net/trace -go get golang.org/x/oauth2 -go get golang.org/x/oauth2/google -go get google.golang.org/cloud - # Build the interop client and server (cd src/google.golang.org/grpc/interop/client && go install) (cd src/google.golang.org/grpc/interop/server && go install) diff --git a/tools/dockerfile/interoptest/grpc_interop_php/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_php/Dockerfile index 0d6171c170..d0e1a96553 100644 --- a/tools/dockerfile/interoptest/grpc_interop_php/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_php/Dockerfile @@ -63,21 +63,6 @@ RUN apt-get update && apt-get install -y \ # Build profiling RUN apt-get update && apt-get install -y time && apt-get clean -#================== -# Ruby dependencies - -# Install rvm -RUN gpg --keyserver hkp://keys.gnupg.net --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 -RUN \curl -sSL https://get.rvm.io | bash -s stable - -# Install Ruby 2.1 -RUN /bin/bash -l -c "rvm install ruby-2.1" -RUN /bin/bash -l -c "rvm use --default ruby-2.1" -RUN /bin/bash -l -c "echo 'gem: --no-ri --no-rdoc' > ~/.gemrc" -RUN /bin/bash -l -c "echo 'export PATH=/usr/local/rvm/bin:$PATH' >> ~/.bashrc" -RUN /bin/bash -l -c "echo 'rvm --default use ruby-2.1' >> ~/.bashrc" -RUN /bin/bash -l -c "gem install bundler --no-ri --no-rdoc" - #================= # PHP dependencies @@ -97,25 +82,10 @@ RUN ln -s /usr/bin/ccache /usr/local/bin/clang++ RUN mkdir /var/local/jenkins -# ronn: a ruby tool used to convert markdown to man pages, used during the -# install of Protobuf extensions -# -# rake: a ruby version of make used to build the PHP Protobuf extension -RUN /bin/bash -l -c "rvm all do gem install ronn rake" - # Install composer RUN curl -sS https://getcomposer.org/installer | php RUN mv composer.phar /usr/local/bin/composer -# Download the patched PHP protobuf so that PHP gRPC clients can be generated -# from proto3 schemas. -RUN git clone https://github.com/stanley-cheung/Protobuf-PHP.git /var/local/git/protobuf-php - -RUN /bin/bash -l -c "rvm use ruby-2.1 \ - && cd /var/local/git/protobuf-php \ - && rvm all do rake pear:package version=1.0 \ - && pear install Protobuf-1.0.tgz" - # Define the default command. CMD ["bash"] diff --git a/tools/dockerfile/interoptest/grpc_interop_php/build_interop.sh b/tools/dockerfile/interoptest/grpc_interop_php/build_interop.sh index a84a450221..624d587786 100755 --- a/tools/dockerfile/interoptest/grpc_interop_php/build_interop.sh +++ b/tools/dockerfile/interoptest/grpc_interop_php/build_interop.sh @@ -38,7 +38,6 @@ git clone --recursive /var/local/jenkins/grpc /var/local/git/grpc cp -r /var/local/jenkins/service_account $HOME || true cd /var/local/git/grpc -rvm --default use ruby-2.1 # gRPC core and protobuf need to be installed make install @@ -47,6 +46,6 @@ make install (cd third_party/protobuf && make install) -(cd src/php && composer install) +(cd src/php && php -d extension=ext/grpc/modules/grpc.so /usr/local/bin/composer install) -(cd src/php && protoc-gen-php -i tests/interop/ -o tests/interop/ tests/interop/test.proto) +(cd src/php && ./bin/generate_proto_php.sh) diff --git a/tools/dockerfile/interoptest/grpc_interop_php7/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_php7/Dockerfile index be8f25f8ff..d8cbe7093c 100644 --- a/tools/dockerfile/interoptest/grpc_interop_php7/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_php7/Dockerfile @@ -75,21 +75,6 @@ RUN cd /var/local/git/php-src \ && make \ && make install -#================== -# Ruby dependencies - -# Install rvm -RUN gpg --keyserver hkp://keys.gnupg.net --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 -RUN \curl -sSL https://get.rvm.io | bash -s stable - -# Install Ruby 2.1 -RUN /bin/bash -l -c "rvm install ruby-2.1" -RUN /bin/bash -l -c "rvm use --default ruby-2.1" -RUN /bin/bash -l -c "echo 'gem: --no-ri --no-rdoc' > ~/.gemrc" -RUN /bin/bash -l -c "echo 'export PATH=/usr/local/rvm/bin:$PATH' >> ~/.bashrc" -RUN /bin/bash -l -c "echo 'rvm --default use ruby-2.1' >> ~/.bashrc" -RUN /bin/bash -l -c "gem install bundler --no-ri --no-rdoc" - # Prepare ccache RUN ln -s /usr/bin/ccache /usr/local/bin/gcc RUN ln -s /usr/bin/ccache /usr/local/bin/g++ @@ -101,25 +86,10 @@ RUN ln -s /usr/bin/ccache /usr/local/bin/clang++ RUN mkdir /var/local/jenkins -# ronn: a ruby tool used to convert markdown to man pages, used during the -# install of Protobuf extensions -# -# rake: a ruby version of make used to build the PHP Protobuf extension -RUN /bin/bash -l -c "rvm all do gem install ronn rake" - # Install composer RUN curl -sS https://getcomposer.org/installer | php RUN mv composer.phar /usr/local/bin/composer -# Download the patched PHP protobuf so that PHP gRPC clients can be generated -# from proto3 schemas. -RUN git clone https://github.com/stanley-cheung/Protobuf-PHP.git /var/local/git/protobuf-php - -RUN /bin/bash -l -c "rvm use ruby-2.1 \ - && cd /var/local/git/protobuf-php \ - && rvm all do rake pear:package version=1.0 \ - && pear install Protobuf-1.0.tgz" - # Define the default command. CMD ["bash"] diff --git a/tools/dockerfile/interoptest/grpc_interop_php7/build_interop.sh b/tools/dockerfile/interoptest/grpc_interop_php7/build_interop.sh index 261dded282..87cb0fe4b2 100755 --- a/tools/dockerfile/interoptest/grpc_interop_php7/build_interop.sh +++ b/tools/dockerfile/interoptest/grpc_interop_php7/build_interop.sh @@ -38,7 +38,6 @@ git clone --recursive /var/local/jenkins/grpc /var/local/git/grpc cp -r /var/local/jenkins/service_account $HOME || true cd /var/local/git/grpc -rvm --default use ruby-2.1 # gRPC core and protobuf need to be installed make install @@ -47,6 +46,6 @@ make install (cd third_party/protobuf && make install) -(cd src/php && composer install) +(cd src/php && php -d extension=ext/grpc/modules/grpc.so /usr/local/bin/composer install) -(cd src/php && protoc-gen-php -i tests/interop/ -o tests/interop/ tests/interop/test.proto) +(cd src/php && ./bin/generate_proto_php.sh) diff --git a/tools/dockerfile/stress_test/grpc_interop_stress_php/Dockerfile b/tools/dockerfile/stress_test/grpc_interop_stress_php/Dockerfile index 0716be5a9d..0fe9c151e5 100644 --- a/tools/dockerfile/stress_test/grpc_interop_stress_php/Dockerfile +++ b/tools/dockerfile/stress_test/grpc_interop_stress_php/Dockerfile @@ -117,24 +117,9 @@ RUN ln -s /usr/bin/ccache /usr/local/bin/clang++ RUN mkdir /var/local/jenkins -# ronn: a ruby tool used to convert markdown to man pages, used during the -# install of Protobuf extensions -# -# rake: a ruby version of make used to build the PHP Protobuf extension -RUN /bin/bash -l -c "rvm all do gem install ronn rake" - # Install composer RUN curl -sS https://getcomposer.org/installer | php RUN mv composer.phar /usr/local/bin/composer -# Download the patched PHP protobuf so that PHP gRPC clients can be generated -# from proto3 schemas. -RUN git clone https://github.com/stanley-cheung/Protobuf-PHP.git /var/local/git/protobuf-php - -RUN /bin/bash -l -c "rvm use ruby-2.1 \ - && cd /var/local/git/protobuf-php \ - && rvm all do rake pear:package version=1.0 \ - && pear install Protobuf-1.0.tgz" - # Define the default command. CMD ["bash"] diff --git a/tools/dockerfile/stress_test/grpc_interop_stress_php/build_interop_stress.sh b/tools/dockerfile/stress_test/grpc_interop_stress_php/build_interop_stress.sh index 87262f1d62..a671d1501f 100755 --- a/tools/dockerfile/stress_test/grpc_interop_stress_php/build_interop_stress.sh +++ b/tools/dockerfile/stress_test/grpc_interop_stress_php/build_interop_stress.sh @@ -38,7 +38,6 @@ git clone --recursive /var/local/jenkins/grpc /var/local/git/grpc cp -r /var/local/jenkins/service_account $HOME || true cd /var/local/git/grpc -rvm --default use ruby-2.1 make install-certs @@ -49,6 +48,6 @@ make install (cd third_party/protobuf && make install) -(cd src/php && composer install) +(cd src/php && php -d extension=ext/grpc/modules/grpc.so /usr/local/bin/composer install) -(cd src/php && protoc-gen-php -i tests/interop/ -o tests/interop/ tests/interop/test.proto) +(cd src/php && ./bin/generate_proto_php.sh) diff --git a/tools/gce/create_linux_worker.sh b/tools/gce/create_linux_worker.sh index 7bf8b24081..013127c585 100755 --- a/tools/gce/create_linux_worker.sh +++ b/tools/gce/create_linux_worker.sh @@ -42,7 +42,7 @@ INSTANCE_NAME="${1:-grpc-jenkins-worker1}" gcloud compute instances create $INSTANCE_NAME \ --project="$CLOUD_PROJECT" \ --zone "$ZONE" \ - --machine-type n1-standard-8 \ + --machine-type n1-highmem-8 \ --image=ubuntu-1510 \ --image-project=grpc-testing \ --boot-disk-size 1000 diff --git a/tools/gce/linux_worker_init.sh b/tools/gce/linux_worker_init.sh index afcf7a52d9..9230acdca6 100755 --- a/tools/gce/linux_worker_init.sh +++ b/tools/gce/linux_worker_init.sh @@ -34,6 +34,14 @@ set -ex +# Create some swap space +sudo dd if=/dev/zero of=/swap bs=1024 count=10485760 +sudo chmod 600 /swap +sudo mkswap /swap +sudo sed -i '$ a\/swap none swap sw 0 0' /etc/fstab +sudo swapon -a + +# Typical apt-get maintenance sudo apt-get update # Install JRE diff --git a/tools/jenkins/reboot_worker.sh b/tools/jenkins/reboot_worker.sh new file mode 100755 index 0000000000..285e699b9b --- /dev/null +++ b/tools/jenkins/reboot_worker.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +# Copyright 2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# Reboots Jenkins worker +# +# NOTE: No empty lines should appear in this file before igncr is set! +set -ex -o igncr || set -ex + +# Give 5 seconds to finish the current job, then kill the jenkins slave process +# to avoid running any other jobs on the worker and restart the worker. +nohup sh -c 'sleep 5; killall java; sudo reboot' & diff --git a/tools/jenkins/run_full_performance.sh b/tools/jenkins/run_full_performance.sh index 3feda866f2..7631db8844 100755 --- a/tools/jenkins/run_full_performance.sh +++ b/tools/jenkins/run_full_performance.sh @@ -38,7 +38,7 @@ cd $(dirname $0)/../.. tools/run_tests/run_performance_tests.py \ -l c++ csharp node ruby java python go \ --netperf \ - --category all \ + --category scalable \ --bq_result_table performance_test.performance_experiment \ --remote_worker_host grpc-performance-server-8core grpc-performance-client-8core grpc-performance-client2-8core \ || EXIT_CODE=1 @@ -53,4 +53,3 @@ tools/run_tests/run_performance_tests.py \ || EXIT_CODE=1 exit $EXIT_CODE - diff --git a/tools/jenkins/run_jenkins_matrix.sh b/tools/jenkins/run_jenkins_matrix.sh new file mode 100755 index 0000000000..b3783e6958 --- /dev/null +++ b/tools/jenkins/run_jenkins_matrix.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +# Copyright 2015, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# This script is invoked by Jenkins and triggers a test run, bypassing +# all args to the test script. +# +# Setting up rvm environment BEFORE we set -ex. +[[ -s /etc/profile.d/rvm.sh ]] && . /etc/profile.d/rvm.sh +# To prevent cygwin bash complaining about empty lines ending with \r +# we set the igncr option. The option doesn't exist on Linux, so we fallback +# to just 'set -ex' there. +# NOTE: No empty lines should appear in this file before igncr is set! +set -ex -o igncr || set -ex + +python tools/run_tests/run_tests_matrix.py $@ diff --git a/tools/run_tests/dockerize/build_docker_and_run_tests.sh b/tools/run_tests/dockerize/build_docker_and_run_tests.sh index c2ea6f2c6e..c3219c533d 100755 --- a/tools/run_tests/dockerize/build_docker_and_run_tests.sh +++ b/tools/run_tests/dockerize/build_docker_and_run_tests.sh @@ -44,9 +44,6 @@ mkdir -p /tmp/ccache # its cache location now that --download-cache is deprecated). mkdir -p /tmp/xdg-cache-home -# Create a local branch so the child Docker script won't complain -git branch -f jenkins-docker - # Inputs # DOCKERFILE_DIR - Directory in which Dockerfile file is located. # DOCKER_RUN_SCRIPT - Script to run under docker (relative to grpc repo root) @@ -64,6 +61,7 @@ CONTAINER_NAME="run_tests_$(uuidgen)" docker_instance_git_root=/var/local/jenkins/grpc # Run tests inside docker +DOCKER_EXIT_CODE=0 docker run \ -e "RUN_TESTS_COMMAND=$RUN_TESTS_COMMAND" \ -e "config=$config" \ @@ -84,16 +82,16 @@ docker run \ -w /var/local/git/grpc \ --name=$CONTAINER_NAME \ $DOCKER_IMAGE_NAME \ - bash -l "/var/local/jenkins/grpc/$DOCKER_RUN_SCRIPT" || DOCKER_FAILED="true" + bash -l "/var/local/jenkins/grpc/$DOCKER_RUN_SCRIPT" || DOCKER_EXIT_CODE=$? -docker cp "$CONTAINER_NAME:/var/local/git/grpc/reports.zip" $git_root || true -unzip -o $git_root/reports.zip -d $git_root || true -rm -f reports.zip +# use unique name for reports.zip to prevent clash between concurrent +# run_tests.py runs +TEMP_REPORTS_ZIP=`mktemp` +docker cp "$CONTAINER_NAME:/var/local/git/grpc/reports.zip" ${TEMP_REPORTS_ZIP} || true +unzip -o ${TEMP_REPORTS_ZIP} -d $git_root || true +rm -f ${TEMP_REPORTS_ZIP} # remove the container, possibly killing it first docker rm -f $CONTAINER_NAME || true -if [ "$DOCKER_FAILED" != "" ] && [ "$XML_REPORT" == "" ] -then - exit 1 -fi +exit $DOCKER_EXIT_CODE diff --git a/tools/run_tests/dockerize/docker_run_tests.sh b/tools/run_tests/dockerize/docker_run_tests.sh index 8c6143d24f..ef02d26625 100755 --- a/tools/run_tests/dockerize/docker_run_tests.sh +++ b/tools/run_tests/dockerize/docker_run_tests.sh @@ -63,6 +63,7 @@ echo '</body></html>' >> index.html cd .. zip -r reports.zip reports -find . -name report.xml | xargs zip reports.zip +find . -name report.xml | xargs -r zip reports.zip +find . -name 'report_*.xml' | xargs -r zip reports.zip exit $exit_code diff --git a/tools/run_tests/performance/scenario_config.py b/tools/run_tests/performance/scenario_config.py index fa401fdaaf..725ca299bd 100644 --- a/tools/run_tests/performance/scenario_config.py +++ b/tools/run_tests/performance/scenario_config.py @@ -190,7 +190,7 @@ class CXXLanguage: # TODO(ctiller): add 70% load latency test for secure in [True, False]: secstr = 'secure' if secure else 'insecure' - smoketest_categories = [SMOKETEST] if secure else [] + smoketest_categories = ([SMOKETEST] if secure else []) + [SCALABLE] yield _ping_pong_scenario( 'cpp_generic_async_streaming_ping_pong_%s' % secstr, @@ -270,7 +270,7 @@ class CSharpLanguage: 'csharp_generic_async_streaming_ping_pong', rpc_type='STREAMING', client_type='ASYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER', use_generic_payload=True, - categories=[SMOKETEST]) + categories=[SMOKETEST, SCALABLE]) yield _ping_pong_scenario( 'csharp_protobuf_async_streaming_ping_pong', rpc_type='STREAMING', @@ -279,7 +279,7 @@ class CSharpLanguage: yield _ping_pong_scenario( 'csharp_protobuf_async_unary_ping_pong', rpc_type='UNARY', client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', - categories=[SMOKETEST]) + categories=[SMOKETEST, SCALABLE]) yield _ping_pong_scenario( 'csharp_protobuf_sync_to_async_unary_ping_pong', rpc_type='UNARY', @@ -301,7 +301,7 @@ class CSharpLanguage: 'csharp_to_cpp_protobuf_sync_unary_ping_pong', rpc_type='UNARY', client_type='SYNC_CLIENT', server_type='SYNC_SERVER', server_language='c++', server_core_limit=1, async_server_threads=1, - categories=[SMOKETEST]) + categories=[SMOKETEST, SCALABLE]) yield _ping_pong_scenario( 'csharp_to_cpp_protobuf_async_streaming_ping_pong', rpc_type='STREAMING', @@ -358,13 +358,13 @@ class NodeLanguage: yield _ping_pong_scenario( 'node_protobuf_unary_ping_pong', rpc_type='UNARY', client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', - categories=[SMOKETEST]) + categories=[SCALABLE, SMOKETEST]) yield _ping_pong_scenario( 'node_protobuf_async_unary_qps_unconstrained', rpc_type='UNARY', client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', unconstrained_client='async', - categories=[SMOKETEST]) + categories=[SCALABLE, SMOKETEST]) # TODO(jtattermusch): make this scenario work #yield _ping_pong_scenario( @@ -403,7 +403,7 @@ class PythonLanguage: 'python_generic_sync_streaming_ping_pong', rpc_type='STREAMING', client_type='SYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER', use_generic_payload=True, - categories=[SMOKETEST]) + categories=[SMOKETEST, SCALABLE]) yield _ping_pong_scenario( 'python_protobuf_sync_streaming_ping_pong', rpc_type='STREAMING', @@ -416,7 +416,7 @@ class PythonLanguage: yield _ping_pong_scenario( 'python_protobuf_sync_unary_ping_pong', rpc_type='UNARY', client_type='SYNC_CLIENT', server_type='ASYNC_SERVER', - categories=[SMOKETEST]) + categories=[SMOKETEST, SCALABLE]) yield _ping_pong_scenario( 'python_protobuf_sync_unary_qps_unconstrained', rpc_type='UNARY', @@ -432,7 +432,7 @@ class PythonLanguage: 'python_to_cpp_protobuf_sync_unary_ping_pong', rpc_type='UNARY', client_type='SYNC_CLIENT', server_type='ASYNC_SERVER', server_language='c++', server_core_limit=1, async_server_threads=1, - categories=[SMOKETEST]) + categories=[SMOKETEST, SCALABLE]) yield _ping_pong_scenario( 'python_to_cpp_protobuf_sync_streaming_ping_pong', rpc_type='STREAMING', @@ -458,12 +458,12 @@ class RubyLanguage: yield _ping_pong_scenario( 'ruby_protobuf_sync_streaming_ping_pong', rpc_type='STREAMING', client_type='SYNC_CLIENT', server_type='SYNC_SERVER', - categories=[SMOKETEST]) + categories=[SMOKETEST, SCALABLE]) yield _ping_pong_scenario( 'ruby_protobuf_unary_ping_pong', rpc_type='UNARY', client_type='SYNC_CLIENT', server_type='SYNC_SERVER', - categories=[SMOKETEST]) + categories=[SMOKETEST, SCALABLE]) yield _ping_pong_scenario( 'ruby_protobuf_sync_unary_qps_unconstrained', rpc_type='UNARY', @@ -504,7 +504,7 @@ class JavaLanguage: def scenarios(self): for secure in [True, False]: secstr = 'secure' if secure else 'insecure' - smoketest_categories = [SMOKETEST] if secure else [] + smoketest_categories = ([SMOKETEST] if secure else []) + [SCALABLE] yield _ping_pong_scenario( 'java_generic_async_streaming_ping_pong_%s' % secstr, rpc_type='STREAMING', @@ -581,7 +581,7 @@ class GoLanguage: def scenarios(self): for secure in [True, False]: secstr = 'secure' if secure else 'insecure' - smoketest_categories = [SMOKETEST] if secure else [] + smoketest_categories = ([SMOKETEST] if secure else []) + [SCALABLE] # ASYNC_GENERIC_SERVER for Go actually uses a sync streaming server, # but that's mostly because of lack of better name of the enum value. diff --git a/tools/run_tests/run_performance_tests.py b/tools/run_tests/run_performance_tests.py index 3f028f177b..92149115fe 100755 --- a/tools/run_tests/run_performance_tests.py +++ b/tools/run_tests/run_performance_tests.py @@ -280,7 +280,7 @@ def create_qpsworkers(languages, worker_hosts): for worker_idx, worker in enumerate(workers)] -Scenario = collections.namedtuple('Scenario', 'jobspec workers') +Scenario = collections.namedtuple('Scenario', 'jobspec workers name') def create_scenarios(languages, workers_by_lang, remote_host=None, regex='.*', @@ -307,7 +307,7 @@ def create_scenarios(languages, workers_by_lang, remote_host=None, regex='.*', create_netperf_jobspec(server_host=netperf_server, client_host=netperf_client, bq_result_table=bq_result_table), - _NO_WORKERS)) + _NO_WORKERS, 'netperf')) for language in languages: for scenario_json in language.scenarios(): @@ -347,7 +347,8 @@ def create_scenarios(languages, workers_by_lang, remote_host=None, regex='.*', [w.host_and_port for w in workers], remote_host=remote_host, bq_result_table=bq_result_table), - workers) + workers, + scenario_json['name']) scenarios.append(scenario) return scenarios @@ -382,6 +383,11 @@ argp.add_argument('--remote_worker_host', nargs='+', default=[], help='Worker hosts where to start QPS workers.') +argp.add_argument('--dry_run', + default=False, + action='store_const', + const=True, + help='Just list scenarios to be run, but don\'t run them.') argp.add_argument('-r', '--regex', default='.*', type=str, help='Regex to select scenarios to run.') argp.add_argument('--bq_result_table', default=None, type=str, @@ -412,16 +418,18 @@ if args.remote_worker_host: if args.remote_driver_host: remote_hosts.add(args.remote_driver_host) -if remote_hosts: - archive_repo(languages=[str(l) for l in languages]) - prepare_remote_hosts(remote_hosts, prepare_local=True) -else: - prepare_remote_hosts([], prepare_local=True) +if not args.dry_run: + if remote_hosts: + archive_repo(languages=[str(l) for l in languages]) + prepare_remote_hosts(remote_hosts, prepare_local=True) + else: + prepare_remote_hosts([], prepare_local=True) build_local = False if not args.remote_driver_host: build_local = True -build_on_remote_hosts(remote_hosts, languages=[str(l) for l in languages], build_local=build_local) +if not args.dry_run: + build_on_remote_hosts(remote_hosts, languages=[str(l) for l in languages], build_local=build_local) qpsworker_jobs = create_qpsworkers(languages, args.remote_worker_host) @@ -443,11 +451,14 @@ if not scenarios: raise Exception('No scenarios to run') for scenario in scenarios: - try: - for worker in scenario.workers: - worker.start() - jobset.run([scenario.jobspec, - create_quit_jobspec(scenario.workers, remote_host=args.remote_driver_host)], - newline_on_success=True, maxjobs=1) - finally: - finish_qps_workers(scenario.workers) + if args.dry_run: + print(scenario.name) + else: + try: + for worker in scenario.workers: + worker.start() + jobset.run([scenario.jobspec, + create_quit_jobspec(scenario.workers, remote_host=args.remote_driver_host)], + newline_on_success=True, maxjobs=1) + finally: + finish_qps_workers(scenario.workers) diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index 168331602c..a6f3d405dc 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -1423,7 +1423,7 @@ else: exit_code = 0 if BuildAndRunError.BUILD in errors: exit_code |= 1 - if BuildAndRunError.TEST in errors and not args.travis: + if BuildAndRunError.TEST in errors: exit_code |= 2 if BuildAndRunError.POST_TEST in errors: exit_code |= 4 diff --git a/tools/run_tests/run_tests_in_workspace.sh b/tools/run_tests/run_tests_in_workspace.sh new file mode 100755 index 0000000000..98ef3566db --- /dev/null +++ b/tools/run_tests/run_tests_in_workspace.sh @@ -0,0 +1,46 @@ +#!/bin/bash +# Copyright 2015, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# Create a workspace in a subdirectory to allow running multiple builds in isolation. +# WORKSPACE_NAME env variable needs to contain name of the workspace to create. +# All cmdline args will be passed to run_tests.py script (executed in the +# newly created workspace) +set -ex + +cd $(dirname $0)/../.. + +rm -rf "${WORKSPACE_NAME}" +# TODO(jtattermusch): clone --recursive fetches the submodules from github. +# Try avoiding that to save time and network capacity. +git clone --recursive . "${WORKSPACE_NAME}" + +echo "Running run_tests.py in workspace ${WORKSPACE_NAME}" +python "${WORKSPACE_NAME}/tools/run_tests/run_tests.py" $@ + diff --git a/tools/run_tests/run_tests_matrix.py b/tools/run_tests/run_tests_matrix.py new file mode 100755 index 0000000000..a94f9cfef5 --- /dev/null +++ b/tools/run_tests/run_tests_matrix.py @@ -0,0 +1,282 @@ +#!/usr/bin/env python2.7 +# Copyright 2015, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Run test matrix.""" + +import argparse +import jobset +import multiprocessing +import os +import report_utils +import sys + +_ROOT = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '../..')) +os.chdir(_ROOT) + +# Set the timeout high to allow enough time for sanitizers and pre-building +# clang docker. +_RUNTESTS_TIMEOUT = 2*60*60 + +# Number of jobs assigned to each run_tests.py instance +_INNER_JOBS = 2 + + +def _docker_jobspec(name, runtests_args=[]): + """Run a single instance of run_tests.py in a docker container""" + test_job = jobset.JobSpec( + cmdline=['python', 'tools/run_tests/run_tests.py', + '--use_docker', + '-t', + '-j', str(_INNER_JOBS), + '-x', 'report_%s.xml' % name] + runtests_args, + shortname='run_tests_%s' % name, + timeout_seconds=_RUNTESTS_TIMEOUT) + return test_job + + +def _workspace_jobspec(name, runtests_args=[], workspace_name=None): + """Run a single instance of run_tests.py in a separate workspace""" + if not workspace_name: + workspace_name = 'workspace_%s' % name + env = {'WORKSPACE_NAME': workspace_name} + test_job = jobset.JobSpec( + cmdline=['tools/run_tests/run_tests_in_workspace.sh', + '-t', + '-j', str(_INNER_JOBS), + '-x', '../report_%s.xml' % name] + runtests_args, + environ=env, + shortname='run_tests_%s' % name, + timeout_seconds=_RUNTESTS_TIMEOUT) + return test_job + + +def _generate_jobs(languages, configs, platforms, + arch=None, compiler=None, + labels=[], extra_args=[]): + result = [] + for language in languages: + for platform in platforms: + for config in configs: + name = '%s_%s_%s' % (language, platform, config) + runtests_args = ['-l', language, + '-c', config] + if arch or compiler: + name += '_%s_%s' % (arch, compiler) + runtests_args += ['--arch', arch, + '--compiler', compiler] + + runtests_args += extra_args + if platform == 'linux': + job = _docker_jobspec(name=name, runtests_args=runtests_args) + else: + job = _workspace_jobspec(name=name, runtests_args=runtests_args) + + job.labels = [platform, config, language] + labels + result.append(job) + return result + + +def _create_test_jobs(extra_args=[]): + test_jobs = [] + # supported on linux only + test_jobs += _generate_jobs(languages=['sanity', 'php7'], + configs=['dbg', 'opt'], + platforms=['linux'], + labels=['basictests'], + extra_args=extra_args) + + # supported on all platforms. + test_jobs += _generate_jobs(languages=['c', 'csharp', 'node', 'python'], + configs=['dbg', 'opt'], + platforms=['linux', 'macos', 'windows'], + labels=['basictests'], + extra_args=extra_args) + + # supported on linux and mac. + test_jobs += _generate_jobs(languages=['c++', 'ruby', 'php'], + configs=['dbg', 'opt'], + platforms=['linux', 'macos'], + labels=['basictests'], + extra_args=extra_args) + + # supported on mac only. + test_jobs += _generate_jobs(languages=['objc'], + configs=['dbg', 'opt'], + platforms=['macos'], + labels=['basictests'], + extra_args=extra_args) + + # sanitizers + test_jobs += _generate_jobs(languages=['c'], + configs=['msan', 'asan', 'tsan'], + platforms=['linux'], + labels=['sanitizers'], + extra_args=extra_args) + test_jobs += _generate_jobs(languages=['c++'], + configs=['asan', 'tsan'], + platforms=['linux'], + labels=['sanitizers'], + extra_args=extra_args) + return test_jobs + + +def _create_portability_test_jobs(extra_args=[]): + test_jobs = [] + # portability C x86 + test_jobs += _generate_jobs(languages=['c'], + configs=['dbg'], + platforms=['linux'], + arch='x86', + compiler='default', + labels=['portability'], + extra_args=extra_args) + + # portability C and C++ on x64 + for compiler in ['gcc4.4', 'gcc4.6', 'gcc5.3', + 'clang3.5', 'clang3.6', 'clang3.7']: + test_jobs += _generate_jobs(languages=['c', 'c++'], + configs=['dbg'], + platforms=['linux'], + arch='x64', + compiler=compiler, + labels=['portability'], + extra_args=extra_args) + + # portability C on Windows + for arch in ['x86', 'x64']: + for compiler in ['vs2013', 'vs2015']: + test_jobs += _generate_jobs(languages=['c'], + configs=['dbg'], + platforms=['windows'], + arch=arch, + compiler=compiler, + labels=['portability'], + extra_args=extra_args) + + test_jobs += _generate_jobs(languages=['python'], + configs=['dbg'], + platforms=['linux'], + arch='default', + compiler='python3.4', + labels=['portability'], + extra_args=extra_args) + + test_jobs += _generate_jobs(languages=['csharp'], + configs=['dbg'], + platforms=['linux'], + arch='default', + compiler='coreclr', + labels=['portability'], + extra_args=extra_args) + return test_jobs + + +def _allowed_labels(): + """Returns a list of existing job labels.""" + all_labels = set() + for job in _create_test_jobs() + _create_portability_test_jobs(): + for label in job.labels: + all_labels.add(label) + return sorted(all_labels) + + +argp = argparse.ArgumentParser(description='Run a matrix of run_tests.py tests.') +argp.add_argument('-j', '--jobs', + default=multiprocessing.cpu_count()/_INNER_JOBS, + type=int, + help='Number of concurrent run_tests.py instances.') +argp.add_argument('-f', '--filter', + choices=_allowed_labels(), + nargs='+', + default=[], + help='Filter targets to run by label with AND semantics.') +argp.add_argument('--build_only', + default=False, + action='store_const', + const=True, + help='Pass --build_only flag to run_tests.py instances.') +argp.add_argument('--force_default_poller', default=False, action='store_const', const=True, + help='Pass --force_default_poller to run_tests.py instances.') +argp.add_argument('--dry_run', + default=False, + action='store_const', + const=True, + help='Only print what would be run.') +args = argp.parse_args() + +extra_args = [] +if args.build_only: + extra_args.append('--build_only') +if args.force_default_poller: + extra_args.append('--force_default_poller') + +all_jobs = _create_test_jobs(extra_args=extra_args) + _create_portability_test_jobs(extra_args=extra_args) + +jobs = [] +for job in all_jobs: + if not args.filter or all(filter in job.labels for filter in args.filter): + jobs.append(job) + +if not jobs: + jobset.message('FAILED', 'No test suites match given criteria.', + do_newline=True) + sys.exit(1) + +print('IMPORTANT: The changes you are testing need to be locally committed') +print('because only the committed changes in the current branch will be') +print('copied to the docker environment or into subworkspaces.') + +print +print 'Will run these tests:' +for job in jobs: + if args.dry_run: + print ' %s: "%s"' % (job.shortname, ' '.join(job.cmdline)) + else: + print ' %s' % job.shortname +print + +if args.dry_run: + print '--dry_run was used, exiting' + sys.exit(1) + +jobset.message('START', 'Running test matrix.', do_newline=True) +num_failures, resultset = jobset.run(jobs, + newline_on_success=True, + travis=True, + maxjobs=args.jobs) +report_utils.render_junit_xml_report(resultset, 'report.xml') + +if num_failures == 0: + jobset.message('SUCCESS', 'All run_tests.py instance finished successfully.', + do_newline=True) +else: + jobset.message('FAILED', 'Some run_tests.py instance have failed.', + do_newline=True) + sys.exit(1) diff --git a/tools/run_tests/sanity/check_submodules.sh b/tools/run_tests/sanity/check_submodules.sh index a33f5b0cc3..6410646cd1 100755 --- a/tools/run_tests/sanity/check_submodules.sh +++ b/tools/run_tests/sanity/check_submodules.sh @@ -44,7 +44,7 @@ cat << EOF | awk '{ print $1 }' | sort > $want_submodules c880e42ba1c8032d4cdde2aba0541d8a9d9fa2e9 third_party/boringssl (version_for_cocoapods_2.0-100-gc880e42) 05b155ff59114735ec8cd089f669c4c3d8f59029 third_party/gflags (v2.1.0-45-g05b155f) c99458533a9b4c743ed51537e25989ea55944908 third_party/googletest (release-1.7.0) - 1a586735085e817b1f52e53feec92ce418049f69 third_party/protobuf (v3.0.2) + a428e42072765993ff674fda72863c9f1aa2d268 third_party/protobuf (v3.1.0) 50893291621658f355bc5b4d450a8d06a563053d third_party/zlib (v1.2.8) bcad91771b7f0bff28a1cac1981d7ef2b9bcef3c third_party/thrift EOF diff --git a/tools/run_tests/sources_and_headers.json b/tools/run_tests/sources_and_headers.json index 03bdcd4264..c05d194e19 100644 --- a/tools/run_tests/sources_and_headers.json +++ b/tools/run_tests/sources_and_headers.json @@ -346,6 +346,23 @@ "headers": [], "is_filegroup": false, "language": "c", + "name": "connection_refused_test", + "src": [ + "test/core/end2end/connection_refused_test.c" + ], + "third_party": false, + "type": "target" + }, + { + "deps": [ + "gpr", + "gpr_test_util", + "grpc", + "grpc_test_util" + ], + "headers": [], + "is_filegroup": false, + "language": "c", "name": "dns_resolver_connectivity_test", "src": [ "test/core/client_config/resolvers/dns_resolver_connectivity_test.c" @@ -2489,6 +2506,20 @@ "headers": [], "is_filegroup": false, "language": "c++", + "name": "grpc_php_plugin", + "src": [ + "src/compiler/php_plugin.cc" + ], + "third_party": false, + "type": "target" + }, + { + "deps": [ + "grpc_plugin_support" + ], + "headers": [], + "is_filegroup": false, + "language": "c++", "name": "grpc_python_plugin", "src": [ "src/compiler/python_plugin.cc" @@ -4979,8 +5010,7 @@ { "deps": [ "grpc++", - "grpc++_reflection", - "grpc++_test_config" + "grpc++_reflection" ], "headers": [ "test/cpp/util/cli_call.h", @@ -5027,6 +5057,8 @@ "src/compiler/node_generator_helpers.h", "src/compiler/objective_c_generator.h", "src/compiler/objective_c_generator_helpers.h", + "src/compiler/php_generator.h", + "src/compiler/php_generator_helpers.h", "src/compiler/python_generator.h", "src/compiler/ruby_generator.h", "src/compiler/ruby_generator_helpers-inl.h", @@ -5051,6 +5083,9 @@ "src/compiler/objective_c_generator.cc", "src/compiler/objective_c_generator.h", "src/compiler/objective_c_generator_helpers.h", + "src/compiler/php_generator.cc", + "src/compiler/php_generator.h", + "src/compiler/php_generator_helpers.h", "src/compiler/python_generator.cc", "src/compiler/python_generator.h", "src/compiler/ruby_generator.cc", @@ -6881,6 +6916,7 @@ ], "headers": [ "test/core/end2end/cq_verifier.h", + "test/core/end2end/fake_resolver.h", "test/core/end2end/fixtures/http_proxy.h", "test/core/end2end/fixtures/proxy.h", "test/core/iomgr/endpoint_tests.h", @@ -6899,6 +6935,8 @@ "src": [ "test/core/end2end/cq_verifier.c", "test/core/end2end/cq_verifier.h", + "test/core/end2end/fake_resolver.c", + "test/core/end2end/fake_resolver.h", "test/core/end2end/fixtures/http_proxy.c", "test/core/end2end/fixtures/http_proxy.h", "test/core/end2end/fixtures/proxy.c", diff --git a/tools/run_tests/tests.json b/tools/run_tests/tests.json index 355fea5d5f..c3395067c9 100644 --- a/tools/run_tests/tests.json +++ b/tools/run_tests/tests.json @@ -392,6 +392,27 @@ "flaky": false, "gtest": false, "language": "c", + "name": "connection_refused_test", + "platforms": [ + "linux", + "mac", + "posix", + "windows" + ] + }, + { + "args": [], + "ci_platforms": [ + "linux", + "mac", + "posix", + "windows" + ], + "cpu_cost": 0.1, + "exclude_configs": [], + "flaky": false, + "gtest": false, + "language": "c", "name": "dns_resolver_connectivity_test", "platforms": [ "linux", @@ -31461,6 +31482,27 @@ { "args": [ "--scenarios_json", + "{\"scenarios\": [{\"name\": \"cpp_generic_async_streaming_ping_pong_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 1, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 1}]}" + ], + "boringssl": true, + "ci_platforms": [ + "linux" + ], + "cpu_cost": 2, + "defaults": "boringssl", + "exclude_configs": [], + "flaky": false, + "language": "c++", + "name": "json_run_localhost", + "platforms": [ + "linux" + ], + "shortname": "json_run_localhost:cpp_generic_async_streaming_ping_pong_secure", + "timeout_seconds": 180 + }, + { + "args": [ + "--scenarios_json", "{\"scenarios\": [{\"name\": \"cpp_generic_async_streaming_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"core_limit\": 0, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}]}" ], "boringssl": true, @@ -31671,6 +31713,27 @@ { "args": [ "--scenarios_json", + "{\"scenarios\": [{\"name\": \"cpp_generic_async_streaming_ping_pong_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 1, \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 1}]}" + ], + "boringssl": true, + "ci_platforms": [ + "linux" + ], + "cpu_cost": 2, + "defaults": "boringssl", + "exclude_configs": [], + "flaky": false, + "language": "c++", + "name": "json_run_localhost", + "platforms": [ + "linux" + ], + "shortname": "json_run_localhost:cpp_generic_async_streaming_ping_pong_insecure", + "timeout_seconds": 180 + }, + { + "args": [ + "--scenarios_json", "{\"scenarios\": [{\"name\": \"cpp_generic_async_streaming_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"core_limit\": 0, \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}]}" ], "boringssl": true, diff --git a/tools/tsan_suppressions.txt b/tools/tsan_suppressions.txt index 09e68cd233..765fa098f9 100644 --- a/tools/tsan_suppressions.txt +++ b/tools/tsan_suppressions.txt @@ -6,3 +6,6 @@ race:cleanse_ctr race:ssleay_rand_add race:ssleay_rand_bytes race:__sleep_for +# protobuf has an idempotent write race in ByteSize +# https://github.com/google/protobuf/issues/2169 +race:ByteSize |