diff options
| -rw-r--r-- | .gitmodules | 3 | ||||
| -rw-r--r-- | INSTALL.md | 5 | ||||
| -rw-r--r-- | bazel/generate_cc.bzl | 16 | ||||
| -rw-r--r-- | examples/BUILD (renamed from examples/protos/BUILD) | 22 | ||||
| -rw-r--r-- | examples/cpp/helloworld/BUILD | 42 | ||||
| -rw-r--r-- | include/grpc++/impl/codegen/thrift_serializer.h | 217 | ||||
| -rw-r--r-- | include/grpc++/impl/codegen/thrift_utils.h | 83 | ||||
| -rw-r--r-- | src/core/ext/filters/http/message_compress/message_compress_filter.c | 22 | ||||
| -rw-r--r-- | test/cpp/util/BUILD | 30 | ||||
| m--------- | third_party/thrift | 0 | ||||
| -rwxr-xr-x | tools/gce/create_linux_performance_worker.sh | 4 | ||||
| -rwxr-xr-x | tools/gce/linux_performance_worker_init.sh | 50 | ||||
| -rwxr-xr-x | tools/jenkins/run_bazel_basic_in_docker.sh | 2 | ||||
| -rwxr-xr-x | tools/run_tests/python_utils/jobset.py | 5 | ||||
| -rwxr-xr-x | tools/run_tests/run_interop_tests.py | 10 |
15 files changed, 115 insertions, 396 deletions
diff --git a/.gitmodules b/.gitmodules index 0f003693e4..144fd080ac 100644 --- a/.gitmodules +++ b/.gitmodules @@ -14,9 +14,6 @@ [submodule "third_party/boringssl"] path = third_party/boringssl url = https://github.com/google/boringssl.git -[submodule "third_party/thrift"] - path = third_party/thrift - url = https://github.com/apache/thrift.git [submodule "third_party/benchmark"] path = third_party/benchmark url = https://github.com/google/benchmark diff --git a/INSTALL.md b/INSTALL.md index 29f0060c81..6cfa1b6cba 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -46,6 +46,11 @@ packages, which you can get from [Homebrew](https://brew.sh): $ brew install autoconf automake libtool shtool ``` +If you plan to build from source and run tests, install the following as well: +```sh + $ brew install gflags +``` + ## Protoc By default gRPC uses [protocol buffers](https://github.com/google/protobuf), diff --git a/bazel/generate_cc.bzl b/bazel/generate_cc.bzl index 8f0f94f563..d05509fc15 100644 --- a/bazel/generate_cc.bzl +++ b/bazel/generate_cc.bzl @@ -9,18 +9,18 @@ def generate_cc_impl(ctx): protos = [f for src in ctx.attr.srcs for f in src.proto.direct_sources] includes = [f for src in ctx.attr.srcs for f in src.proto.transitive_imports] outs = [] + # label_len is length of the path from WORKSPACE root to the location of this build file + label_len = len(ctx.label.package) + 1 if ctx.executable.plugin: - outs += [proto.basename[:-len(".proto")] + ".grpc.pb.h" for proto in protos] - outs += [proto.basename[:-len(".proto")] + ".grpc.pb.cc" for proto in protos] + outs += [proto.path[label_len:-len(".proto")] + ".grpc.pb.h" for proto in protos] + outs += [proto.path[label_len:-len(".proto")] + ".grpc.pb.cc" for proto in protos] if ctx.attr.generate_mock: - outs += [proto.basename[:-len(".proto")] + "_mock.grpc.pb.h" for proto in protos] + outs += [proto.path[label_len:-len(".proto")] + "_mock.grpc.pb.h" for proto in protos] else: - outs += [proto.basename[:-len(".proto")] + ".pb.h" for proto in protos] - outs += [proto.basename[:-len(".proto")] + ".pb.cc" for proto in protos] + outs += [proto.path[label_len:-len(".proto")] + ".pb.h" for proto in protos] + outs += [proto.path[label_len:-len(".proto")] + ".pb.cc" for proto in protos] out_files = [ctx.new_file(out) for out in outs] - # The following should be replaced with ctx.configuration.buildout - # whenever this is added to Skylark. - dir_out = out_files[0].dirname[:-len(protos[0].dirname)] + dir_out = str(ctx.genfiles_dir.path) arguments = [] if ctx.executable.plugin: diff --git a/examples/protos/BUILD b/examples/BUILD index 2ffdf64f9a..382713e5e4 100644 --- a/examples/protos/BUILD +++ b/examples/BUILD @@ -33,20 +33,34 @@ load("//bazel:grpc_build_system.bzl", "grpc_proto_library") grpc_proto_library( name = "auth_sample", - srcs = ["auth_sample.proto"], + srcs = ["protos/auth_sample.proto"], ) grpc_proto_library( name = "hellostreamingworld", - srcs = ["hellostreamingworld.proto"], + srcs = ["protos/hellostreamingworld.proto"], ) grpc_proto_library( name = "helloworld", - srcs = ["helloworld.proto"], + srcs = ["protos/helloworld.proto"], ) grpc_proto_library( name = "route_guide", - srcs = ["route_guide.proto"], + srcs = ["protos/route_guide.proto"], +) + +cc_binary( + name = "greeter_client", + srcs = ["cpp/helloworld/greeter_client.cc"], + deps = ["helloworld"], + defines = ["BAZEL_BUILD"], +) + +cc_binary( + name = "greeter_server", + srcs = ["cpp/helloworld/greeter_server.cc"], + deps = ["helloworld"], + defines = ["BAZEL_BUILD"], ) diff --git a/examples/cpp/helloworld/BUILD b/examples/cpp/helloworld/BUILD deleted file mode 100644 index b9c3f5dfbe..0000000000 --- a/examples/cpp/helloworld/BUILD +++ /dev/null @@ -1,42 +0,0 @@ -# Copyright 2017, Google Inc. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -cc_binary( - name = "greeter_client", - srcs = ["greeter_client.cc"], - deps = ["//examples/protos:helloworld"], - defines = ["BAZEL_BUILD"], -) - -cc_binary( - name = "greeter_server", - srcs = ["greeter_server.cc"], - deps = ["//examples/protos:helloworld"], - defines = ["BAZEL_BUILD"], -) diff --git a/include/grpc++/impl/codegen/thrift_serializer.h b/include/grpc++/impl/codegen/thrift_serializer.h deleted file mode 100644 index 86bc7105c0..0000000000 --- a/include/grpc++/impl/codegen/thrift_serializer.h +++ /dev/null @@ -1,217 +0,0 @@ -/* - * - * Copyright 2016, Google Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Google Inc. nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - */ - -#ifndef GRPCXX_IMPL_CODEGEN_THRIFT_SERIALIZER_H -#define GRPCXX_IMPL_CODEGEN_THRIFT_SERIALIZER_H - -#include <grpc/impl/codegen/byte_buffer_reader.h> -#include <grpc/impl/codegen/slice.h> -#include <thrift/protocol/TBinaryProtocol.h> -#include <thrift/protocol/TCompactProtocol.h> -#include <thrift/protocol/TProtocolException.h> -#include <thrift/transport/TBufferTransports.h> -#include <thrift/transport/TTransportUtils.h> -#include <boost/make_shared.hpp> -#include <memory> -#include <stdexcept> -#include <string> - -namespace apache { -namespace thrift { -namespace util { - -using apache::thrift::protocol::TBinaryProtocolT; -using apache::thrift::protocol::TCompactProtocolT; -using apache::thrift::protocol::TMessageType; -using apache::thrift::protocol::TNetworkBigEndian; -using apache::thrift::transport::TMemoryBuffer; -using apache::thrift::transport::TBufferBase; -using apache::thrift::transport::TTransport; - -template <typename Dummy, typename Protocol> -class ThriftSerializer { - public: - ThriftSerializer() - : prepared_(false), - last_deserialized_(false), - serialize_version_(false) {} - - virtual ~ThriftSerializer() {} - - // Serialize the passed type into the internal buffer - // and returns a pointer to internal buffer and its size - template <typename T> - void Serialize(const T& fields, const uint8_t** serialized_buffer, - size_t* serialized_len) { - // prepare or reset buffer - if (!prepared_ || last_deserialized_) { - prepare(); - } else { - buffer_->resetBuffer(); - } - last_deserialized_ = false; - - // if required serialize protocol version - if (serialize_version_) { - protocol_->writeMessageBegin("", TMessageType(0), 0); - } - - // serialize fields into buffer - fields.write(protocol_.get()); - - // write the end of message - if (serialize_version_) { - protocol_->writeMessageEnd(); - } - - uint8_t* byte_buffer; - uint32_t byte_buffer_size; - buffer_->getBuffer(&byte_buffer, &byte_buffer_size); - *serialized_buffer = byte_buffer; - *serialized_len = byte_buffer_size; - } - - // Serialize the passed type into the byte buffer - template <typename T> - void Serialize(const T& fields, grpc_byte_buffer** bp) { - const uint8_t* byte_buffer; - size_t byte_buffer_size; - - Serialize(fields, &byte_buffer, &byte_buffer_size); - - grpc_slice slice = grpc_slice_from_copied_buffer( - reinterpret_cast<const char*>(byte_buffer), byte_buffer_size); - - *bp = grpc_raw_byte_buffer_create(&slice, 1); - - grpc_slice_unref(slice); - } - - // Deserialize the passed char array into the passed type, returns the number - // of bytes that have been consumed from the passed string. - template <typename T> - uint32_t Deserialize(uint8_t* serialized_buffer, size_t length, T* fields) { - // prepare buffer if necessary - if (!prepared_) { - prepare(); - } - last_deserialized_ = true; - - // reset buffer transport - buffer_->resetBuffer(serialized_buffer, length); - - // read the protocol version if necessary - if (serialize_version_) { - std::string name = ""; - TMessageType mt = static_cast<TMessageType>(0); - int32_t seq_id = 0; - protocol_->readMessageBegin(name, mt, seq_id); - } - - // deserialize buffer into fields - uint32_t len = fields->read(protocol_.get()); - - // read the end of message - if (serialize_version_) { - protocol_->readMessageEnd(); - } - - return len; - } - - // Deserialize the passed byte buffer to passed type, returns the number - // of bytes consumed from byte buffer - template <typename T> - uint32_t Deserialize(grpc_byte_buffer* buffer, T* msg) { - grpc_byte_buffer_reader reader; - grpc_byte_buffer_reader_init(&reader, buffer); - - grpc_slice slice = grpc_byte_buffer_reader_readall(&reader); - - uint32_t len = - Deserialize(GRPC_SLICE_START_PTR(slice), GRPC_SLICE_LENGTH(slice), msg); - - grpc_slice_unref(slice); - - grpc_byte_buffer_reader_destroy(&reader); - - return len; - } - - // set serialization version flag - void SetSerializeVersion(bool value) { serialize_version_ = value; } - - // Set the container size limit to deserialize - // This function should be called after buffer_ is initialized - void SetContainerSizeLimit(int32_t container_limit) { - if (!prepared_) { - prepare(); - } - protocol_->setContainerSizeLimit(container_limit); - } - - // Set the string size limit to deserialize - // This function should be called after buffer_ is initialized - void SetStringSizeLimit(int32_t string_limit) { - if (!prepared_) { - prepare(); - } - protocol_->setStringSizeLimit(string_limit); - } - - private: - bool prepared_; - bool last_deserialized_; - boost::shared_ptr<TMemoryBuffer> buffer_; - std::shared_ptr<Protocol> protocol_; - bool serialize_version_; - - void prepare() { - buffer_ = boost::make_shared<TMemoryBuffer>(); - // create a protocol for the memory buffer transport - protocol_ = std::make_shared<Protocol>(buffer_); - prepared_ = true; - } - -}; // ThriftSerializer - -typedef ThriftSerializer<void, TBinaryProtocolT<TBufferBase, TNetworkBigEndian>> - ThriftSerializerBinary; -typedef ThriftSerializer<void, TCompactProtocolT<TBufferBase>> - ThriftSerializerCompact; - -} // namespace util -} // namespace thrift -} // namespace apache - -#endif // GRPCXX_IMPL_CODEGEN_THRIFT_SERIALIZER_H diff --git a/include/grpc++/impl/codegen/thrift_utils.h b/include/grpc++/impl/codegen/thrift_utils.h deleted file mode 100644 index 742d739703..0000000000 --- a/include/grpc++/impl/codegen/thrift_utils.h +++ /dev/null @@ -1,83 +0,0 @@ -/* - * - * Copyright 2016, Google Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Google Inc. nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - */ - -#ifndef GRPCXX_IMPL_CODEGEN_THRIFT_UTILS_H -#define GRPCXX_IMPL_CODEGEN_THRIFT_UTILS_H - -#include <grpc++/impl/codegen/config.h> -#include <grpc++/impl/codegen/core_codegen_interface.h> -#include <grpc++/impl/codegen/serialization_traits.h> -#include <grpc++/impl/codegen/status.h> -#include <grpc++/impl/codegen/status_code_enum.h> -#include <grpc++/impl/codegen/thrift_serializer.h> -#include <grpc/impl/codegen/byte_buffer_reader.h> -#include <grpc/impl/codegen/slice.h> -#include <cstdint> -#include <cstdlib> - -namespace grpc { - -using apache::thrift::util::ThriftSerializerCompact; - -template <class T> -class SerializationTraits<T, typename std::enable_if<std::is_base_of< - apache::thrift::TBase, T>::value>::type> { - public: - static Status Serialize(const T& msg, grpc_byte_buffer** bp, - bool* own_buffer) { - *own_buffer = true; - - ThriftSerializerCompact serializer; - serializer.Serialize(msg, bp); - - return Status(StatusCode::OK, "ok"); - } - - static Status Deserialize(grpc_byte_buffer* buffer, T* msg, - int max_receive_message_size) { - if (!buffer) { - return Status(StatusCode::INTERNAL, "No payload"); - } - - ThriftSerializerCompact deserializer; - deserializer.Deserialize(buffer, msg); - - grpc_byte_buffer_destroy(buffer); - - return Status(StatusCode::OK, "ok"); - } -}; - -} // namespace grpc - -#endif // GRPCXX_IMPL_CODEGEN_THRIFT_UTILS_H diff --git a/src/core/ext/filters/http/message_compress/message_compress_filter.c b/src/core/ext/filters/http/message_compress/message_compress_filter.c index 4f5f41e9b0..1da8cf69cb 100644 --- a/src/core/ext/filters/http/message_compress/message_compress_filter.c +++ b/src/core/ext/filters/http/message_compress/message_compress_filter.c @@ -277,13 +277,10 @@ static void compress_start_transport_stream_op_batch( GPR_TIMER_BEGIN("compress_start_transport_stream_op_batch", 0); if (op->cancel_stream) { - gpr_atm cur; GRPC_ERROR_REF(op->payload->cancel_stream.cancel_error); - do { - cur = gpr_atm_acq_load(&calld->send_initial_metadata_state); - } while (!gpr_atm_rel_cas( - &calld->send_initial_metadata_state, cur, - CANCELLED_BIT | (gpr_atm)op->payload->cancel_stream.cancel_error)); + gpr_atm cur = gpr_atm_full_xchg( + &calld->send_initial_metadata_state, + CANCELLED_BIT | (gpr_atm)op->payload->cancel_stream.cancel_error); switch (cur) { case HAS_COMPRESSION_ALGORITHM: case NO_COMPRESSION_ALGORITHM: @@ -311,13 +308,18 @@ static void compress_start_transport_stream_op_batch( grpc_transport_stream_op_batch_finish_with_failure(exec_ctx, op, error); return; } - gpr_atm cur = gpr_atm_acq_load(&calld->send_initial_metadata_state); + gpr_atm cur; + retry_send_im: + cur = gpr_atm_acq_load(&calld->send_initial_metadata_state); GPR_ASSERT(cur != HAS_COMPRESSION_ALGORITHM && cur != NO_COMPRESSION_ALGORITHM); if ((cur & CANCELLED_BIT) == 0) { - gpr_atm_rel_store(&calld->send_initial_metadata_state, - has_compression_algorithm ? HAS_COMPRESSION_ALGORITHM - : NO_COMPRESSION_ALGORITHM); + if (!gpr_atm_rel_cas(&calld->send_initial_metadata_state, cur, + has_compression_algorithm + ? HAS_COMPRESSION_ALGORITHM + : NO_COMPRESSION_ALGORITHM)) { + goto retry_send_im; + } if (cur != INITIAL_METADATA_UNSEEN) { grpc_call_next_op(exec_ctx, elem, (grpc_transport_stream_op_batch *)cur); diff --git a/test/cpp/util/BUILD b/test/cpp/util/BUILD index c83f89eb90..9dde22b4d1 100644 --- a/test/cpp/util/BUILD +++ b/test/cpp/util/BUILD @@ -34,8 +34,8 @@ licenses(["notice"]) # 3-clause BSD cc_binary( name = "testso.so", srcs = [], - deps = ["//:grpc++_unsecure"], linkshared = 1, + deps = ["//:grpc++_unsecure"], ) cc_library( @@ -104,5 +104,29 @@ cc_test( ], ) - - +cc_binary( + name = "grpc_cli", + srcs = [ + "cli_call.cc", + "cli_call.h", + "cli_credentials.cc", + "cli_credentials.h", + "config_grpc_cli.h", + "grpc_cli.cc", + "grpc_tool.cc", + "grpc_tool.h", + "proto_file_parser.cc", + "proto_file_parser.h", + "proto_reflection_descriptor_database.cc", + "proto_reflection_descriptor_database.h", + "service_describer.cc", + "service_describer.h", + "test_config.h", + "test_config_cc.cc", + ], + deps = [ + "//:grpc++", + "//external:gflags", + "//src/proto/grpc/reflection/v1alpha:reflection_proto", + ], +) diff --git a/third_party/thrift b/third_party/thrift deleted file mode 160000 -Subproject bcad91771b7f0bff28a1cac1981d7ef2b9bcef3 diff --git a/tools/gce/create_linux_performance_worker.sh b/tools/gce/create_linux_performance_worker.sh index 2c8cf0b96b..68710e13b0 100755 --- a/tools/gce/create_linux_performance_worker.sh +++ b/tools/gce/create_linux_performance_worker.sh @@ -42,14 +42,14 @@ CLOUD_PROJECT=grpc-testing ZONE=us-central1-b # this zone allows 32core machines INSTANCE_NAME="${1:-grpc-performance-server1}" -MACHINE_TYPE=n1-standard-8 +MACHINE_TYPE=n1-standard-32 gcloud compute instances create $INSTANCE_NAME \ --project="$CLOUD_PROJECT" \ --zone "$ZONE" \ --machine-type $MACHINE_TYPE \ --image-project ubuntu-os-cloud \ - --image-family ubuntu-1604-lts \ + --image-family ubuntu-1610 \ --boot-disk-size 300 \ --scopes https://www.googleapis.com/auth/bigquery diff --git a/tools/gce/linux_performance_worker_init.sh b/tools/gce/linux_performance_worker_init.sh index 17f36fb4ff..78cdd31f0b 100755 --- a/tools/gce/linux_performance_worker_init.sh +++ b/tools/gce/linux_performance_worker_init.sh @@ -55,7 +55,10 @@ sudo apt-get install -y \ libc6 \ libc6-dbg \ libc6-dev \ + libcurl4-openssl-dev \ libgtest-dev \ + libreadline-dev \ + libssl-dev \ libtool \ make \ strace \ @@ -71,7 +74,8 @@ sudo apt-get install -y \ telnet \ unzip \ wget \ - zip + zip \ + zlib1g-dev # perftools sudo apt-get install -y google-perftools libgoogle-perftools-dev @@ -87,14 +91,15 @@ sudo pip install tabulate sudo pip install google-api-python-client sudo pip install virtualenv -# TODO(jtattermusch): For some reason, building gRPC Python depends on python3.4 -# being installed, but python3.4 is not available on Ubuntu 16.04. -# Temporarily fixing this by adding a PPA with python3.4, but we should -# really remove this hack once possible. -sudo add-apt-repository -y ppa:fkrull/deadsnakes -sudo apt-get update -sudo apt-get install -y python3.4 python3.4-dev -python3.4 -m pip install virtualenv +# Building gRPC Python depends on python3.4 being installed, but python3.4 +# is not available on Ubuntu 16.10, so install from source +curl -O https://www.python.org/ftp/python/3.4.6/Python-3.4.6.tgz +tar xzvf Python-3.4.6.tgz +cd Python-3.4.6 +./configure --enable-shared --prefix=/usr/local LDFLAGS="-Wl,--rpath=/usr/local/lib" +sudo make altinstall +cd .. +rm Python-3.4.6.tgz curl -O https://bootstrap.pypa.io/get-pip.py sudo pypy get-pip.py @@ -117,18 +122,25 @@ sudo apt-get update sudo apt-get install -y mono-devel nuget # C# .NET Core dependencies (https://www.microsoft.com/net/core#ubuntu) -sudo sh -c 'echo "deb [arch=amd64] https://apt-mo.trafficmanager.net/repos/dotnet-release/ xenial main" > /etc/apt/sources.list.d/dotnetdev.list' +sudo sh -c 'echo "deb [arch=amd64] https://apt-mo.trafficmanager.net/repos/dotnet-release/ yakkety main" > /etc/apt/sources.list.d/dotnetdev.list' sudo apt-key adv --keyserver apt-mo.trafficmanager.net --recv-keys 417A0893 sudo apt-get update -sudo apt-get install -y dotnet-dev-1.0.0-preview2-003131 +sudo apt-get install -y dotnet-dev-1.0.0-preview2.1-003155 sudo apt-get install -y dotnet-dev-1.0.1 # Ruby dependencies -gpg --keyserver hkp://keys.gnupg.net --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 -curl -sSL https://get.rvm.io | bash -s stable --ruby +git clone https://github.com/rbenv/rbenv.git ~/.rbenv +export PATH="$HOME/.rbenv/bin:$PATH" +eval "$(rbenv init -)" + +git clone https://github.com/rbenv/ruby-build.git ~/.rbenv/plugins/ruby-build +export PATH="$HOME/.rbenv/plugins/ruby-build/bin:$PATH" + +rbenv install 2.4.0 +rbenv global 2.4.0 +ruby -v # Install bundler (prerequisite for gRPC Ruby) -source ~/.rvm/scripts/rvm gem install bundler # Java dependencies - nothing as we already have Java JDK 8 @@ -163,15 +175,7 @@ echo 4096 | sudo tee /proc/sys/kernel/perf_event_mlock_kb git clone -v https://github.com/brendangregg/FlameGraph ~/FlameGraph # Install scipy and numpy for benchmarking scripts -sudo apt-get install python-scipy python-numpy - -# Update Linux kernel to 4.9 -wget \ - kernel.ubuntu.com/~kernel-ppa/mainline/v4.9.20/linux-headers-4.9.20-040920_4.9.20-040920.201703310531_all.deb \ - kernel.ubuntu.com/~kernel-ppa/mainline/v4.9.20/linux-headers-4.9.20-040920-generic_4.9.20-040920.201703310531_amd64.deb \ - kernel.ubuntu.com/~kernel-ppa/mainline/v4.9.20/linux-image-4.9.20-040920-generic_4.9.20-040920.201703310531_amd64.deb -sudo dpkg -i linux-headers-4.9*.deb linux-image-4.9*.deb -rm linux-* +sudo apt-get install -y python-scipy python-numpy # Add pubkey of jenkins@grpc-jenkins-master to authorized keys of jenkins@ # This needs to happen as the last step to prevent Jenkins master from connecting diff --git a/tools/jenkins/run_bazel_basic_in_docker.sh b/tools/jenkins/run_bazel_basic_in_docker.sh index b1d498a07d..5013f80b1d 100755 --- a/tools/jenkins/run_bazel_basic_in_docker.sh +++ b/tools/jenkins/run_bazel_basic_in_docker.sh @@ -39,4 +39,4 @@ git clone /var/local/jenkins/grpc /var/local/git/grpc && git submodule update --init --reference /var/local/jenkins/grpc/${name} \ ${name}') cd /var/local/git/grpc -bazel build --spawn_strategy=standalone --genrule_strategy=standalone :all test/... examples/cpp/... +bazel build --spawn_strategy=standalone --genrule_strategy=standalone :all test/... examples/... diff --git a/tools/run_tests/python_utils/jobset.py b/tools/run_tests/python_utils/jobset.py index 460f359cf3..d01e82a76b 100755 --- a/tools/run_tests/python_utils/jobset.py +++ b/tools/run_tests/python_utils/jobset.py @@ -208,6 +208,11 @@ class JobSpec(object): def __repr__(self): return 'JobSpec(shortname=%s, cmdline=%s)' % (self.shortname, self.cmdline) + def __str__(self): + return '%s: %s %s' % (self.shortname, + ' '.join('%s=%s' % kv for kv in self.environ.items()), + ' '.join(self.cmdline)) + class JobResult(object): def __init__(self): diff --git a/tools/run_tests/run_interop_tests.py b/tools/run_tests/run_interop_tests.py index 44e93eb9cc..867d9e6f7b 100755 --- a/tools/run_tests/run_interop_tests.py +++ b/tools/run_tests/run_interop_tests.py @@ -888,6 +888,10 @@ argp.add_argument('-t', '--travis', default=False, action='store_const', const=True) +argp.add_argument('-v', '--verbose', + default=False, + action='store_const', + const=True) argp.add_argument('--use_docker', default=False, action='store_const', @@ -989,6 +993,9 @@ if args.use_docker: if build_jobs: jobset.message('START', 'Building interop docker images.', do_newline=True) + if args.verbose: + print('Jobs to run: \n%s\n' % '\n'.join(str(j) for j in build_jobs)) + num_failures, _ = jobset.run( build_jobs, newline_on_success=True, maxjobs=args.jobs) if num_failures == 0: @@ -1164,6 +1171,9 @@ try: if args.manual_run: print('All tests will skipped --manual_run option is active.') + if args.verbose: + print('Jobs to run: \n%s\n' % '\n'.join(str(job) for job in jobs)) + num_failures, resultset = jobset.run(jobs, newline_on_success=True, maxjobs=args.jobs, skip_jobs=args.manual_run) |