diff options
96 files changed, 1502 insertions, 350 deletions
diff --git a/.github/mergeable.yml b/.github/mergeable.yml new file mode 100644 index 0000000000..f0180b9979 --- /dev/null +++ b/.github/mergeable.yml @@ -0,0 +1,6 @@ +mergeable: + pull_requests: + label: + must_include: + regex: "release notes:yes|release notes:no" + message: "Add release notes yes/no label. For yes, add lang label" @@ -64,11 +64,11 @@ config_setting( ) # This should be updated along with build.yaml -g_stands_for = "gladiolus" +g_stands_for = "glider" core_version = "6.0.0-dev" -version = "1.14.0-dev" +version = "1.15.0-dev" GPR_PUBLIC_HDRS = [ "include/grpc/support/alloc.h", diff --git a/CMakeLists.txt b/CMakeLists.txt index 75f3a2f3cd..84e9c08cb5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -24,7 +24,7 @@ cmake_minimum_required(VERSION 2.8) set(PACKAGE_NAME "grpc") -set(PACKAGE_VERSION "1.14.0-dev") +set(PACKAGE_VERSION "1.15.0-dev") set(PACKAGE_STRING "${PACKAGE_NAME} ${PACKAGE_VERSION}") set(PACKAGE_TARNAME "${PACKAGE_NAME}-${PACKAGE_VERSION}") set(PACKAGE_BUGREPORT "https://github.com/grpc/grpc/issues/") @@ -5164,6 +5164,7 @@ add_library(qps test/cpp/qps/client_sync.cc test/cpp/qps/driver.cc test/cpp/qps/parse_json.cc + test/cpp/qps/qps_server_builder.cc test/cpp/qps/qps_worker.cc test/cpp/qps/report.cc test/cpp/qps/server_async.cc @@ -437,8 +437,8 @@ Q = @ endif CORE_VERSION = 6.0.0-dev -CPP_VERSION = 1.14.0-dev -CSHARP_VERSION = 1.14.0-dev +CPP_VERSION = 1.15.0-dev +CSHARP_VERSION = 1.15.0-dev CPPFLAGS_NO_ARCH += $(addprefix -I, $(INCLUDES)) $(addprefix -D, $(DEFINES)) CPPFLAGS += $(CPPFLAGS_NO_ARCH) $(ARCH_FLAGS) @@ -1381,6 +1381,8 @@ static_c: pc_c pc_c_unsecure cache.mk $(LIBDIR)/$(CONFIG)/libaddress_sorting.a static_cxx: pc_cxx pc_cxx_unsecure cache.mk $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc++_cronet.a $(LIBDIR)/$(CONFIG)/libgrpc++_error_details.a $(LIBDIR)/$(CONFIG)/libgrpc++_reflection.a $(LIBDIR)/$(CONFIG)/libgrpc++_unsecure.a $(LIBDIR)/$(CONFIG)/libgrpcpp_channelz.a +static_csharp: static_c $(LIBDIR)/$(CONFIG)/libgrpc_csharp_ext.a + shared: shared_c shared_cxx shared_c: pc_c pc_c_unsecure cache.mk $(LIBDIR)/$(CONFIG)/$(SHARED_PREFIX)address_sorting$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/$(SHARED_PREFIX)gpr$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/$(SHARED_PREFIX)grpc$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/$(SHARED_PREFIX)grpc_cronet$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/$(SHARED_PREFIX)grpc_unsecure$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) @@ -7438,6 +7440,7 @@ LIBQPS_SRC = \ test/cpp/qps/client_sync.cc \ test/cpp/qps/driver.cc \ test/cpp/qps/parse_json.cc \ + test/cpp/qps/qps_server_builder.cc \ test/cpp/qps/qps_worker.cc \ test/cpp/qps/report.cc \ test/cpp/qps/server_async.cc \ @@ -7493,6 +7496,7 @@ $(OBJDIR)/$(CONFIG)/test/cpp/qps/client_async.o: $(GENDIR)/src/proto/grpc/testin $(OBJDIR)/$(CONFIG)/test/cpp/qps/client_sync.o: $(GENDIR)/src/proto/grpc/testing/messages.pb.cc $(GENDIR)/src/proto/grpc/testing/messages.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/payloads.pb.cc $(GENDIR)/src/proto/grpc/testing/payloads.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/stats.pb.cc $(GENDIR)/src/proto/grpc/testing/stats.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/control.pb.cc $(GENDIR)/src/proto/grpc/testing/control.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/benchmark_service.pb.cc $(GENDIR)/src/proto/grpc/testing/benchmark_service.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/report_qps_scenario_service.pb.cc $(GENDIR)/src/proto/grpc/testing/report_qps_scenario_service.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/worker_service.pb.cc $(GENDIR)/src/proto/grpc/testing/worker_service.grpc.pb.cc $(OBJDIR)/$(CONFIG)/test/cpp/qps/driver.o: $(GENDIR)/src/proto/grpc/testing/messages.pb.cc $(GENDIR)/src/proto/grpc/testing/messages.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/payloads.pb.cc $(GENDIR)/src/proto/grpc/testing/payloads.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/stats.pb.cc $(GENDIR)/src/proto/grpc/testing/stats.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/control.pb.cc $(GENDIR)/src/proto/grpc/testing/control.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/benchmark_service.pb.cc $(GENDIR)/src/proto/grpc/testing/benchmark_service.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/report_qps_scenario_service.pb.cc $(GENDIR)/src/proto/grpc/testing/report_qps_scenario_service.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/worker_service.pb.cc $(GENDIR)/src/proto/grpc/testing/worker_service.grpc.pb.cc $(OBJDIR)/$(CONFIG)/test/cpp/qps/parse_json.o: $(GENDIR)/src/proto/grpc/testing/messages.pb.cc $(GENDIR)/src/proto/grpc/testing/messages.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/payloads.pb.cc $(GENDIR)/src/proto/grpc/testing/payloads.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/stats.pb.cc $(GENDIR)/src/proto/grpc/testing/stats.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/control.pb.cc $(GENDIR)/src/proto/grpc/testing/control.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/benchmark_service.pb.cc $(GENDIR)/src/proto/grpc/testing/benchmark_service.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/report_qps_scenario_service.pb.cc $(GENDIR)/src/proto/grpc/testing/report_qps_scenario_service.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/worker_service.pb.cc $(GENDIR)/src/proto/grpc/testing/worker_service.grpc.pb.cc +$(OBJDIR)/$(CONFIG)/test/cpp/qps/qps_server_builder.o: $(GENDIR)/src/proto/grpc/testing/messages.pb.cc $(GENDIR)/src/proto/grpc/testing/messages.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/payloads.pb.cc $(GENDIR)/src/proto/grpc/testing/payloads.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/stats.pb.cc $(GENDIR)/src/proto/grpc/testing/stats.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/control.pb.cc $(GENDIR)/src/proto/grpc/testing/control.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/benchmark_service.pb.cc $(GENDIR)/src/proto/grpc/testing/benchmark_service.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/report_qps_scenario_service.pb.cc $(GENDIR)/src/proto/grpc/testing/report_qps_scenario_service.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/worker_service.pb.cc $(GENDIR)/src/proto/grpc/testing/worker_service.grpc.pb.cc $(OBJDIR)/$(CONFIG)/test/cpp/qps/qps_worker.o: $(GENDIR)/src/proto/grpc/testing/messages.pb.cc $(GENDIR)/src/proto/grpc/testing/messages.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/payloads.pb.cc $(GENDIR)/src/proto/grpc/testing/payloads.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/stats.pb.cc $(GENDIR)/src/proto/grpc/testing/stats.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/control.pb.cc $(GENDIR)/src/proto/grpc/testing/control.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/benchmark_service.pb.cc $(GENDIR)/src/proto/grpc/testing/benchmark_service.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/report_qps_scenario_service.pb.cc $(GENDIR)/src/proto/grpc/testing/report_qps_scenario_service.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/worker_service.pb.cc $(GENDIR)/src/proto/grpc/testing/worker_service.grpc.pb.cc $(OBJDIR)/$(CONFIG)/test/cpp/qps/report.o: $(GENDIR)/src/proto/grpc/testing/messages.pb.cc $(GENDIR)/src/proto/grpc/testing/messages.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/payloads.pb.cc $(GENDIR)/src/proto/grpc/testing/payloads.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/stats.pb.cc $(GENDIR)/src/proto/grpc/testing/stats.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/control.pb.cc $(GENDIR)/src/proto/grpc/testing/control.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/benchmark_service.pb.cc $(GENDIR)/src/proto/grpc/testing/benchmark_service.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/report_qps_scenario_service.pb.cc $(GENDIR)/src/proto/grpc/testing/report_qps_scenario_service.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/worker_service.pb.cc $(GENDIR)/src/proto/grpc/testing/worker_service.grpc.pb.cc $(OBJDIR)/$(CONFIG)/test/cpp/qps/server_async.o: $(GENDIR)/src/proto/grpc/testing/messages.pb.cc $(GENDIR)/src/proto/grpc/testing/messages.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/payloads.pb.cc $(GENDIR)/src/proto/grpc/testing/payloads.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/stats.pb.cc $(GENDIR)/src/proto/grpc/testing/stats.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/control.pb.cc $(GENDIR)/src/proto/grpc/testing/control.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/benchmark_service.pb.cc $(GENDIR)/src/proto/grpc/testing/benchmark_service.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/report_qps_scenario_service.pb.cc $(GENDIR)/src/proto/grpc/testing/report_qps_scenario_service.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/worker_service.pb.cc $(GENDIR)/src/proto/grpc/testing/worker_service.grpc.pb.cc @@ -24691,6 +24695,7 @@ test/cpp/qps/client_async.cc: $(OPENSSL_DEP) test/cpp/qps/client_sync.cc: $(OPENSSL_DEP) test/cpp/qps/driver.cc: $(OPENSSL_DEP) test/cpp/qps/parse_json.cc: $(OPENSSL_DEP) +test/cpp/qps/qps_server_builder.cc: $(OPENSSL_DEP) test/cpp/qps/qps_worker.cc: $(OPENSSL_DEP) test/cpp/qps/report.cc: $(OPENSSL_DEP) test/cpp/qps/server_async.cc: $(OPENSSL_DEP) @@ -37,6 +37,8 @@ For instructions on how to use the language-specific gRPC runtime in your projec You can find per-language quickstart guides and tutorials in [Documentation section on grpc.io website](https://grpc.io/docs/). The code examples are available in the [examples](examples) directory. +Precompiled bleeding-edge package builds of gRPC `master` branch's `HEAD` are uploaded daily to [packages.grpc.io](https://packages.grpc.io). + # To start developing gRPC Contributions are welcome! diff --git a/build.yaml b/build.yaml index 2ff98f12bc..30389ec114 100644 --- a/build.yaml +++ b/build.yaml @@ -13,8 +13,8 @@ settings: '#09': Per-language overrides are possible with (eg) ruby_version tag here '#10': See the expand_version.py for all the quirks here core_version: 6.0.0-dev - g_stands_for: gladiolus - version: 1.14.0-dev + g_stands_for: glider + version: 1.15.0-dev filegroups: - name: alts_proto headers: @@ -1963,6 +1963,7 @@ libs: - test/cpp/qps/histogram.h - test/cpp/qps/interarrival.h - test/cpp/qps/parse_json.h + - test/cpp/qps/qps_server_builder.h - test/cpp/qps/qps_worker.h - test/cpp/qps/report.h - test/cpp/qps/server.h @@ -1981,6 +1982,7 @@ libs: - test/cpp/qps/client_sync.cc - test/cpp/qps/driver.cc - test/cpp/qps/parse_json.cc + - test/cpp/qps/qps_server_builder.cc - test/cpp/qps/qps_worker.cc - test/cpp/qps/report.cc - test/cpp/qps/server_async.cc diff --git a/doc/g_stands_for.md b/doc/g_stands_for.md index f86bd76f32..b3ed9bd713 100644 --- a/doc/g_stands_for.md +++ b/doc/g_stands_for.md @@ -13,4 +13,5 @@ - 1.11 'g' stands for ['gorgeous'](https://github.com/grpc/grpc/tree/v1.11.x) - 1.12 'g' stands for ['glorious'](https://github.com/grpc/grpc/tree/v1.12.x) - 1.13 'g' stands for ['gloriosa'](https://github.com/grpc/grpc/tree/v1.13.x) -- 1.14 'g' stands for ['gladiolus'](https://github.com/grpc/grpc/tree/master) +- 1.14 'g' stands for ['gladiolus'](https://github.com/grpc/grpc/tree/v1.14.x) +- 1.15 'g' stands for ['glider'](https://github.com/grpc/grpc/tree/master) diff --git a/doc/interop-test-descriptions.md b/doc/interop-test-descriptions.md index 9781925533..3c33189196 100644 --- a/doc/interop-test-descriptions.md +++ b/doc/interop-test-descriptions.md @@ -784,6 +784,32 @@ Client asserts: * received status message is the same as the sent message for both Procedure steps 1 and 2 +### special_status_message + +This test verifies Unicode and whitespace is correctly processed in status +message. "\t" is horizontal tab. "\r" is carriage return. "\n" is line feed. + +Server features: +* [UnaryCall][] +* [Echo Status][] + +Procedure: + 1. Client calls UnaryCall with: + + ``` + { + response_status:{ + code: 2 + message: "\t\ntest with whitespace\r\nand Unicode BMP ☺ and non-BMP 😈\t\n" + } + } + ``` + +Client asserts: +* received status code is the same as the sent code for Procedure step 1 +* received status message is the same as the sent message for Procedure step 1, + including all whitespace characters + ### unimplemented_method This test verifies that calling an unimplemented RPC method returns the diff --git a/doc/statuscodes.md b/doc/statuscodes.md index f2df9e00de..06fbe5c8fe 100644 --- a/doc/statuscodes.md +++ b/doc/statuscodes.md @@ -38,6 +38,7 @@ situations in which they are generated. | Error parsing response proto | INTERNAL | Client| | Error parsing request proto | INTERNAL | Server| | Sent or received message was larger than configured limit | RESOURCE_EXHAUSTED | Both | +| Keepalive watchdog times out | INTERNAL | Both | The following status codes are never generated by the library: - INVALID_ARGUMENT @@ -47,3 +48,5 @@ The following status codes are never generated by the library: - ABORTED - OUT_OF_RANGE - DATA_LOSS + +Applications that may wish to [retry](https://github.com/grpc/proposal/blob/master/A6-client-retries.md) failed RPCs must decide which status codes on which to retry. As shown in the table above, the gRPC library can generate the same status code for different cases. Server applications can also return those same status codes. Therefore, there is no fixed list of status codes on which it is appropriate to retry in all applications. As a result, individual applications must make their own determination as to which status codes should cause an RPC to be retried. diff --git a/gRPC-C++.podspec b/gRPC-C++.podspec index 57d58cc440..1d9237bf62 100644 --- a/gRPC-C++.podspec +++ b/gRPC-C++.podspec @@ -23,7 +23,7 @@ Pod::Spec.new do |s| s.name = 'gRPC-C++' # TODO (mxyan): use version that match gRPC version when pod is stabilized - # version = '1.14.0-dev' + # version = '1.15.0-dev' version = '0.0.3' s.version = version s.summary = 'gRPC C++ library' @@ -31,7 +31,7 @@ Pod::Spec.new do |s| s.license = 'Apache License, Version 2.0' s.authors = { 'The gRPC contributors' => 'grpc-packages@google.com' } - grpc_version = '1.14.0-dev' + grpc_version = '1.15.0-dev' s.source = { :git => 'https://github.com/grpc/grpc.git', diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index 997617c530..23edaec656 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -22,7 +22,7 @@ Pod::Spec.new do |s| s.name = 'gRPC-Core' - version = '1.14.0-dev' + version = '1.15.0-dev' s.version = version s.summary = 'Core cross-platform gRPC library, written in C' s.homepage = 'https://grpc.io' diff --git a/gRPC-ProtoRPC.podspec b/gRPC-ProtoRPC.podspec index 6548f36a32..79ec679e3a 100644 --- a/gRPC-ProtoRPC.podspec +++ b/gRPC-ProtoRPC.podspec @@ -21,7 +21,7 @@ Pod::Spec.new do |s| s.name = 'gRPC-ProtoRPC' - version = '1.14.0-dev' + version = '1.15.0-dev' s.version = version s.summary = 'RPC library for Protocol Buffers, based on gRPC' s.homepage = 'https://grpc.io' diff --git a/gRPC-RxLibrary.podspec b/gRPC-RxLibrary.podspec index ebd942cf78..7a461e270f 100644 --- a/gRPC-RxLibrary.podspec +++ b/gRPC-RxLibrary.podspec @@ -21,7 +21,7 @@ Pod::Spec.new do |s| s.name = 'gRPC-RxLibrary' - version = '1.14.0-dev' + version = '1.15.0-dev' s.version = version s.summary = 'Reactive Extensions library for iOS/OSX.' s.homepage = 'https://grpc.io' diff --git a/gRPC.podspec b/gRPC.podspec index d6d59c2141..bf3f56bd54 100644 --- a/gRPC.podspec +++ b/gRPC.podspec @@ -20,7 +20,7 @@ Pod::Spec.new do |s| s.name = 'gRPC' - version = '1.14.0-dev' + version = '1.15.0-dev' s.version = version s.summary = 'gRPC client library for iOS/OSX' s.homepage = 'https://grpc.io' @@ -1718,6 +1718,7 @@ 'test/cpp/qps/client_sync.cc', 'test/cpp/qps/driver.cc', 'test/cpp/qps/parse_json.cc', + 'test/cpp/qps/qps_server_builder.cc', 'test/cpp/qps/qps_worker.cc', 'test/cpp/qps/report.cc', 'test/cpp/qps/server_async.cc', diff --git a/package.xml b/package.xml index a9dc2dc79a..7f71536b1d 100644 --- a/package.xml +++ b/package.xml @@ -13,8 +13,8 @@ <date>2018-01-19</date> <time>16:06:07</time> <version> - <release>1.14.0dev</release> - <api>1.14.0dev</api> + <release>1.15.0dev</release> + <api>1.15.0dev</api> </version> <stability> <release>beta</release> diff --git a/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc b/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc index 023281db97..d217dc0e63 100644 --- a/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc +++ b/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc @@ -451,6 +451,7 @@ void PickFirst::PickFirstSubchannelData::ProcessConnectivityChangeLocked( // latest pending subchannel lists. GPR_ASSERT(subchannel_list() == p->subchannel_list_.get() || subchannel_list() == p->latest_pending_subchannel_list_.get()); + GPR_ASSERT(connectivity_state != GRPC_CHANNEL_SHUTDOWN); // Handle updates for the currently selected subchannel. if (p->selected_ == this) { if (grpc_lb_pick_first_trace.enabled()) { @@ -480,14 +481,12 @@ void PickFirst::PickFirstSubchannelData::ProcessConnectivityChangeLocked( "update"), "selected_not_ready+switch_to_update"); } else { - // TODO(juanlishen): we re-resolve when the selected subchannel goes to - // TRANSIENT_FAILURE because we used to shut down in this case before - // re-resolution is introduced. But we need to investigate whether we - // really want to take any action instead of waiting for the selected - // subchannel reconnecting. - GPR_ASSERT(connectivity_state != GRPC_CHANNEL_SHUTDOWN); if (connectivity_state == GRPC_CHANNEL_TRANSIENT_FAILURE) { - // If the selected channel goes bad, request a re-resolution. + // If the selected subchannel goes bad, request a re-resolution. We also + // set the channel state to IDLE and reset started_picking_. The reason + // is that if the new state is TRANSIENT_FAILURE due to a GOAWAY + // reception we don't want to connect to the re-resolved backends until + // we leave the IDLE state. grpc_connectivity_state_set(&p->state_tracker_, GRPC_CHANNEL_IDLE, GRPC_ERROR_NONE, "selected_changed+reresolve"); @@ -568,9 +567,10 @@ void PickFirst::PickFirstSubchannelData::ProcessConnectivityChangeLocked( // Case 1: Only set state to TRANSIENT_FAILURE if we've tried // all subchannels. if (sd->Index() == 0 && subchannel_list() == p->subchannel_list_.get()) { + p->TryReresolutionLocked(&grpc_lb_pick_first_trace, GRPC_ERROR_NONE); grpc_connectivity_state_set( &p->state_tracker_, GRPC_CHANNEL_TRANSIENT_FAILURE, - GRPC_ERROR_REF(error), "connecting_transient_failure"); + GRPC_ERROR_REF(error), "exhausted_subchannels"); } sd->StartConnectivityWatchLocked(); break; diff --git a/src/core/ext/filters/client_channel/subchannel.cc b/src/core/ext/filters/client_channel/subchannel.cc index 5894d52e6b..d7b64a900f 100644 --- a/src/core/ext/filters/client_channel/subchannel.cc +++ b/src/core/ext/filters/client_channel/subchannel.cc @@ -418,8 +418,6 @@ static void continue_connect_locked(grpc_subchannel* c) { c->next_attempt_deadline = c->backoff->NextAttemptTime(); args.deadline = std::max(c->next_attempt_deadline, min_deadline); args.channel_args = c->args; - grpc_connectivity_state_set(&c->state_tracker, GRPC_CHANNEL_CONNECTING, - GRPC_ERROR_NONE, "state_change"); grpc_connector_connect(c->connector, &args, &c->connecting_result, &c->on_connected); } @@ -475,27 +473,24 @@ static void maybe_start_connecting_locked(grpc_subchannel* c) { /* Don't try to connect if we're already disconnected */ return; } - if (c->connecting) { /* Already connecting: don't restart */ return; } - if (c->connected_subchannel != nullptr) { /* Already connected: don't restart */ return; } - if (!grpc_connectivity_state_has_watchers(&c->state_tracker)) { /* Nobody is interested in connecting: so don't just yet */ return; } - c->connecting = true; GRPC_SUBCHANNEL_WEAK_REF(c, "connecting"); - if (!c->backoff_begun) { c->backoff_begun = true; + grpc_connectivity_state_set(&c->state_tracker, GRPC_CHANNEL_CONNECTING, + GRPC_ERROR_NONE, "connecting"); continue_connect_locked(c); } else { GPR_ASSERT(!c->have_alarm); @@ -510,6 +505,11 @@ static void maybe_start_connecting_locked(grpc_subchannel* c) { } GRPC_CLOSURE_INIT(&c->on_alarm, on_alarm, c, grpc_schedule_on_exec_ctx); grpc_timer_init(&c->alarm, c->next_attempt_deadline, &c->on_alarm); + // During backoff, we prefer the connectivity state of CONNECTING instead of + // TRANSIENT_FAILURE in order to prevent triggering re-resolution + // continuously in pick_first. + grpc_connectivity_state_set(&c->state_tracker, GRPC_CHANNEL_CONNECTING, + GRPC_ERROR_NONE, "backoff"); } } diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.cc b/src/core/ext/transport/chttp2/transport/chttp2_transport.cc index bc6fa0d0eb..9ad271753c 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.cc +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.cc @@ -813,7 +813,11 @@ static void set_write_state(grpc_chttp2_transport* t, write_state_name(st), reason)); t->write_state = st; if (st == GRPC_CHTTP2_WRITE_STATE_IDLE) { - GRPC_CLOSURE_LIST_SCHED(&t->run_after_write); + grpc_chttp2_stream* s; + while (grpc_chttp2_list_pop_waiting_for_write_stream(t, &s)) { + GRPC_CLOSURE_LIST_SCHED(&s->run_after_write); + GRPC_CHTTP2_STREAM_UNREF(s, "chttp2:write_closure_sched"); + } if (t->close_transport_on_writes_finished != nullptr) { grpc_error* err = t->close_transport_on_writes_finished; t->close_transport_on_writes_finished = nullptr; @@ -1208,7 +1212,10 @@ void grpc_chttp2_complete_closure_step(grpc_chttp2_transport* t, !(closure->next_data.scratch & CLOSURE_BARRIER_MAY_COVER_WRITE)) { GRPC_CLOSURE_RUN(closure, closure->error_data.error); } else { - grpc_closure_list_append(&t->run_after_write, closure, + if (grpc_chttp2_list_add_waiting_for_write_stream(t, s)) { + GRPC_CHTTP2_STREAM_REF(s, "chttp2:pending_write_closure"); + } + grpc_closure_list_append(&s->run_after_write, closure, closure->error_data.error); } } @@ -2009,6 +2016,10 @@ static void remove_stream(grpc_chttp2_transport* t, uint32_t id, void grpc_chttp2_cancel_stream(grpc_chttp2_transport* t, grpc_chttp2_stream* s, grpc_error* due_to_error) { + GRPC_CLOSURE_LIST_SCHED(&s->run_after_write); + if (grpc_chttp2_list_remove_waiting_for_write_stream(t, s)) { + GRPC_CHTTP2_STREAM_UNREF(s, "chttp2:pending_write_closure"); + } if (!t->is_client && !s->sent_trailing_metadata && grpc_error_has_clear_grpc_status(due_to_error)) { close_from_api(t, s, due_to_error); diff --git a/src/core/ext/transport/chttp2/transport/internal.h b/src/core/ext/transport/chttp2/transport/internal.h index ca6e715978..4f1a08d98b 100644 --- a/src/core/ext/transport/chttp2/transport/internal.h +++ b/src/core/ext/transport/chttp2/transport/internal.h @@ -54,6 +54,8 @@ typedef enum { /** streams that are waiting to start because there are too many concurrent streams on the connection */ GRPC_CHTTP2_LIST_WAITING_FOR_CONCURRENCY, + /** streams with closures waiting to be run on a write **/ + GRPC_CHTTP2_LIST_WAITING_FOR_WRITE, STREAM_LIST_COUNT /* must be last */ } grpc_chttp2_stream_list_id; @@ -431,9 +433,6 @@ struct grpc_chttp2_transport { */ grpc_error* close_transport_on_writes_finished; - /* a list of closures to run after writes are finished */ - grpc_closure_list run_after_write; - /* buffer pool state */ /** have we scheduled a benign cleanup? */ bool benign_reclaimer_registered; @@ -584,6 +583,7 @@ struct grpc_chttp2_stream { grpc_slice_buffer flow_controlled_buffer; + grpc_closure_list run_after_write; grpc_chttp2_write_cb* on_flow_controlled_cbs; grpc_chttp2_write_cb* on_write_finished_cbs; grpc_chttp2_write_cb* finish_after_write; @@ -686,6 +686,13 @@ bool grpc_chttp2_list_pop_stalled_by_stream(grpc_chttp2_transport* t, bool grpc_chttp2_list_remove_stalled_by_stream(grpc_chttp2_transport* t, grpc_chttp2_stream* s); +bool grpc_chttp2_list_add_waiting_for_write_stream(grpc_chttp2_transport* t, + grpc_chttp2_stream* s); +bool grpc_chttp2_list_pop_waiting_for_write_stream(grpc_chttp2_transport* t, + grpc_chttp2_stream** s); +bool grpc_chttp2_list_remove_waiting_for_write_stream(grpc_chttp2_transport* t, + grpc_chttp2_stream* s); + /********* Flow Control ***************/ // Takes in a flow control action and performs all the needed operations. diff --git a/src/core/ext/transport/chttp2/transport/stream_lists.cc b/src/core/ext/transport/chttp2/transport/stream_lists.cc index 6626170a7e..50bfe36a86 100644 --- a/src/core/ext/transport/chttp2/transport/stream_lists.cc +++ b/src/core/ext/transport/chttp2/transport/stream_lists.cc @@ -35,6 +35,8 @@ static const char* stream_list_id_string(grpc_chttp2_stream_list_id id) { return "stalled_by_stream"; case GRPC_CHTTP2_LIST_WAITING_FOR_CONCURRENCY: return "waiting_for_concurrency"; + case GRPC_CHTTP2_LIST_WAITING_FOR_WRITE: + return "waiting_for_write"; case STREAM_LIST_COUNT: GPR_UNREACHABLE_CODE(return "unknown"); } @@ -214,3 +216,18 @@ bool grpc_chttp2_list_remove_stalled_by_stream(grpc_chttp2_transport* t, grpc_chttp2_stream* s) { return stream_list_maybe_remove(t, s, GRPC_CHTTP2_LIST_STALLED_BY_STREAM); } + +bool grpc_chttp2_list_add_waiting_for_write_stream(grpc_chttp2_transport* t, + grpc_chttp2_stream* s) { + return stream_list_add(t, s, GRPC_CHTTP2_LIST_WAITING_FOR_WRITE); +} + +bool grpc_chttp2_list_pop_waiting_for_write_stream(grpc_chttp2_transport* t, + grpc_chttp2_stream** s) { + return stream_list_pop(t, s, GRPC_CHTTP2_LIST_WAITING_FOR_WRITE); +} + +bool grpc_chttp2_list_remove_waiting_for_write_stream(grpc_chttp2_transport* t, + grpc_chttp2_stream* s) { + return stream_list_maybe_remove(t, s, GRPC_CHTTP2_LIST_WAITING_FOR_WRITE); +} diff --git a/src/core/lib/channel/channel_stack.h b/src/core/lib/channel/channel_stack.h index 7581f937b6..35c3fb01ea 100644 --- a/src/core/lib/channel/channel_stack.h +++ b/src/core/lib/channel/channel_stack.h @@ -82,7 +82,7 @@ typedef struct { typedef struct { grpc_call_stats stats; grpc_status_code final_status; - const char** error_string; + const char* error_string; } grpc_call_final_info; /* Channel filters specify: diff --git a/src/core/lib/gpr/arena.cc b/src/core/lib/gpr/arena.cc index 141de69426..e30b297aea 100644 --- a/src/core/lib/gpr/arena.cc +++ b/src/core/lib/gpr/arena.cc @@ -25,6 +25,7 @@ #include <grpc/support/alloc.h> #include <grpc/support/atm.h> #include <grpc/support/log.h> +#include <grpc/support/sync.h> #include "src/core/lib/gpr/alloc.h" @@ -36,8 +37,6 @@ #ifdef SIMPLE_ARENA_FOR_DEBUGGING -#include <grpc/support/sync.h> - struct gpr_arena { gpr_mu mu; void** ptrs; @@ -78,14 +77,17 @@ void* gpr_arena_alloc(gpr_arena* arena, size_t size) { // would allow us to use the alignment actually needed by the caller. typedef struct zone { - size_t size_begin; - size_t size_end; - gpr_atm next_atm; + size_t size_begin; // All the space we have set aside for allocations up + // until this zone. + size_t size_end; // size_end = size_begin plus all the space we set aside for + // allocations in zone z itself. + zone* next; } zone; struct gpr_arena { gpr_atm size_so_far; zone initial_zone; + gpr_mu arena_growth_mutex; }; static void* zalloc_aligned(size_t size) { @@ -99,15 +101,17 @@ gpr_arena* gpr_arena_create(size_t initial_size) { gpr_arena* a = static_cast<gpr_arena*>(zalloc_aligned( GPR_ROUND_UP_TO_ALIGNMENT_SIZE(sizeof(gpr_arena)) + initial_size)); a->initial_zone.size_end = initial_size; + gpr_mu_init(&a->arena_growth_mutex); return a; } size_t gpr_arena_destroy(gpr_arena* arena) { + gpr_mu_destroy(&arena->arena_growth_mutex); gpr_atm size = gpr_atm_no_barrier_load(&arena->size_so_far); - zone* z = (zone*)gpr_atm_no_barrier_load(&arena->initial_zone.next_atm); + zone* z = arena->initial_zone.next; gpr_free_aligned(arena); while (z) { - zone* next_z = (zone*)gpr_atm_no_barrier_load(&z->next_atm); + zone* next_z = z->next; gpr_free_aligned(z); z = next_z; } @@ -116,37 +120,55 @@ size_t gpr_arena_destroy(gpr_arena* arena) { void* gpr_arena_alloc(gpr_arena* arena, size_t size) { size = GPR_ROUND_UP_TO_ALIGNMENT_SIZE(size); - size_t start = static_cast<size_t>( + size_t previous_size_of_arena_allocations = static_cast<size_t>( gpr_atm_no_barrier_fetch_add(&arena->size_so_far, size)); + size_t updated_size_of_arena_allocations = + previous_size_of_arena_allocations + size; zone* z = &arena->initial_zone; - while (start > z->size_end) { - zone* next_z = (zone*)gpr_atm_acq_load(&z->next_atm); - if (next_z == nullptr) { - size_t next_z_size = - static_cast<size_t>(gpr_atm_no_barrier_load(&arena->size_so_far)); - next_z = static_cast<zone*>(zalloc_aligned( - GPR_ROUND_UP_TO_ALIGNMENT_SIZE(sizeof(zone)) + next_z_size)); - next_z->size_begin = z->size_end; - next_z->size_end = z->size_end + next_z_size; - if (!gpr_atm_rel_cas(&z->next_atm, static_cast<gpr_atm>(NULL), - (gpr_atm)next_z)) { - gpr_free_aligned(next_z); - next_z = (zone*)gpr_atm_acq_load(&z->next_atm); + // Check to see if the allocation isn't able to end in the initial zone. + // This statement is true only in the uncommon case because of our arena + // sizing historesis (that is, most calls should have a large enough initial + // zone and will not need to grow the arena). + if (updated_size_of_arena_allocations > z->size_end) { + // Find a zone to fit this allocation + gpr_mu_lock(&arena->arena_growth_mutex); + while (updated_size_of_arena_allocations > z->size_end) { + if (z->next == nullptr) { + // Note that we do an extra increment of size_so_far to prevent multiple + // simultaneous callers from stepping on each other. However, this extra + // increment means some space in the arena is wasted. + // So whenever we need to allocate x bytes and there are x - n (where + // n > 0) remaining in the current zone, we will waste x bytes (x - n + // in the current zone and n in the new zone). + previous_size_of_arena_allocations = static_cast<size_t>( + gpr_atm_no_barrier_fetch_add(&arena->size_so_far, size)); + updated_size_of_arena_allocations = + previous_size_of_arena_allocations + size; + size_t next_z_size = updated_size_of_arena_allocations; + z->next = static_cast<zone*>(zalloc_aligned( + GPR_ROUND_UP_TO_ALIGNMENT_SIZE(sizeof(zone)) + next_z_size)); + z->next->size_begin = z->size_end; + z->next->size_end = z->size_end + next_z_size; } + z = z->next; } - z = next_z; - } - if (start + size > z->size_end) { - return gpr_arena_alloc(arena, size); + gpr_mu_unlock(&arena->arena_growth_mutex); } - GPR_ASSERT(start >= z->size_begin); - GPR_ASSERT(start + size <= z->size_end); - char* ptr = (z == &arena->initial_zone) - ? reinterpret_cast<char*>(arena) + - GPR_ROUND_UP_TO_ALIGNMENT_SIZE(sizeof(gpr_arena)) - : reinterpret_cast<char*>(z) + - GPR_ROUND_UP_TO_ALIGNMENT_SIZE(sizeof(zone)); - return ptr + start - z->size_begin; + GPR_ASSERT(previous_size_of_arena_allocations >= z->size_begin); + GPR_ASSERT(updated_size_of_arena_allocations <= z->size_end); + // Skip the first part of the zone, which just contains tracking information. + // For the initial zone, this is the gpr_arena struct and for any other zone, + // it's the zone struct. + char* start_of_allocation_space = + (z == &arena->initial_zone) + ? reinterpret_cast<char*>(arena) + + GPR_ROUND_UP_TO_ALIGNMENT_SIZE(sizeof(gpr_arena)) + : reinterpret_cast<char*>(z) + + GPR_ROUND_UP_TO_ALIGNMENT_SIZE(sizeof(zone)); + // previous_size_of_arena_allocations - size_begin is how many bytes have been + // allocated into the current zone + return start_of_allocation_space + previous_size_of_arena_allocations - + z->size_begin; } #endif // SIMPLE_ARENA_FOR_DEBUGGING diff --git a/src/core/lib/iomgr/ev_epollex_linux.cc b/src/core/lib/iomgr/ev_epollex_linux.cc index 7b368410cf..e1f3e43af7 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.cc +++ b/src/core/lib/iomgr/ev_epollex_linux.cc @@ -135,7 +135,7 @@ struct pollable { // underlying epoll set (i.e whenever fd_orphan() is called). // // Implementing (2) above (i.e removing fds from cache on fd_orphan) adds a - // lot of complexity since an fd can be present in multiple pollalbles. So our + // lot of complexity since an fd can be present in multiple pollables. So our // implementation ONLY DOES (1) and NOT (2). // // The cache_fd.salt variable helps here to maintain correctness (it serves as diff --git a/src/core/lib/iomgr/executor.cc b/src/core/lib/iomgr/executor.cc index 1ad13b831d..45d96b80eb 100644 --- a/src/core/lib/iomgr/executor.cc +++ b/src/core/lib/iomgr/executor.cc @@ -40,19 +40,25 @@ gpr_log(GPR_INFO, "EXECUTOR " format, __VA_ARGS__); \ } +#define EXECUTOR_TRACE0(str) \ + if (executor_trace.enabled()) { \ + gpr_log(GPR_INFO, "EXECUTOR " str); \ + } + grpc_core::TraceFlag executor_trace(false, "executor"); GPR_TLS_DECL(g_this_thread_state); -GrpcExecutor::GrpcExecutor(const char* executor_name) : name_(executor_name) { +GrpcExecutor::GrpcExecutor(const char* name) : name_(name) { adding_thread_lock_ = GPR_SPINLOCK_STATIC_INITIALIZER; - gpr_atm_no_barrier_store(&num_threads_, 0); + gpr_atm_rel_store(&num_threads_, 0); max_threads_ = GPR_MAX(1, 2 * gpr_cpu_num_cores()); } void GrpcExecutor::Init() { SetThreading(true); } -size_t GrpcExecutor::RunClosures(grpc_closure_list list) { +size_t GrpcExecutor::RunClosures(const char* executor_name, + grpc_closure_list list) { size_t n = 0; grpc_closure* c = list.head; @@ -60,11 +66,11 @@ size_t GrpcExecutor::RunClosures(grpc_closure_list list) { grpc_closure* next = c->next_data.next; grpc_error* error = c->error_data.error; #ifndef NDEBUG - EXECUTOR_TRACE("run %p [created by %s:%d]", c, c->file_created, - c->line_created); + EXECUTOR_TRACE("(%s) run %p [created by %s:%d]", executor_name, c, + c->file_created, c->line_created); c->scheduled = false; #else - EXECUTOR_TRACE("run %p", c); + EXECUTOR_TRACE("(%s) run %p", executor_name, c); #endif c->cb(c->cb_arg, error); GRPC_ERROR_UNREF(error); @@ -77,17 +83,21 @@ size_t GrpcExecutor::RunClosures(grpc_closure_list list) { } bool GrpcExecutor::IsThreaded() const { - return gpr_atm_no_barrier_load(&num_threads_) > 0; + return gpr_atm_acq_load(&num_threads_) > 0; } void GrpcExecutor::SetThreading(bool threading) { - gpr_atm curr_num_threads = gpr_atm_no_barrier_load(&num_threads_); + gpr_atm curr_num_threads = gpr_atm_acq_load(&num_threads_); + EXECUTOR_TRACE("(%s) SetThreading(%d) begin", name_, threading); if (threading) { - if (curr_num_threads > 0) return; + if (curr_num_threads > 0) { + EXECUTOR_TRACE("(%s) SetThreading(true). curr_num_threads == 0", name_); + return; + } GPR_ASSERT(num_threads_ == 0); - gpr_atm_no_barrier_store(&num_threads_, 1); + gpr_atm_rel_store(&num_threads_, 1); gpr_tls_init(&g_this_thread_state); thd_state_ = static_cast<ThreadState*>( gpr_zalloc(sizeof(ThreadState) * max_threads_)); @@ -96,6 +106,7 @@ void GrpcExecutor::SetThreading(bool threading) { gpr_mu_init(&thd_state_[i].mu); gpr_cv_init(&thd_state_[i].cv); thd_state_[i].id = i; + thd_state_[i].name = name_; thd_state_[i].thd = grpc_core::Thread(); thd_state_[i].elems = GRPC_CLOSURE_LIST_INIT; } @@ -104,7 +115,10 @@ void GrpcExecutor::SetThreading(bool threading) { grpc_core::Thread(name_, &GrpcExecutor::ThreadMain, &thd_state_[0]); thd_state_[0].thd.Start(); } else { // !threading - if (curr_num_threads == 0) return; + if (curr_num_threads == 0) { + EXECUTOR_TRACE("(%s) SetThreading(false). curr_num_threads == 0", name_); + return; + } for (size_t i = 0; i < max_threads_; i++) { gpr_mu_lock(&thd_state_[i].mu); @@ -121,20 +135,22 @@ void GrpcExecutor::SetThreading(bool threading) { curr_num_threads = gpr_atm_no_barrier_load(&num_threads_); for (gpr_atm i = 0; i < curr_num_threads; i++) { thd_state_[i].thd.Join(); - EXECUTOR_TRACE(" Thread %" PRIdPTR " of %" PRIdPTR " joined", i, - curr_num_threads); + EXECUTOR_TRACE("(%s) Thread %" PRIdPTR " of %" PRIdPTR " joined", name_, + i + 1, curr_num_threads); } - gpr_atm_no_barrier_store(&num_threads_, 0); + gpr_atm_rel_store(&num_threads_, 0); for (size_t i = 0; i < max_threads_; i++) { gpr_mu_destroy(&thd_state_[i].mu); gpr_cv_destroy(&thd_state_[i].cv); - RunClosures(thd_state_[i].elems); + RunClosures(thd_state_[i].name, thd_state_[i].elems); } gpr_free(thd_state_); gpr_tls_destroy(&g_this_thread_state); } + + EXECUTOR_TRACE("(%s) SetThreading(%d) done", name_, threading); } void GrpcExecutor::Shutdown() { SetThreading(false); } @@ -147,8 +163,8 @@ void GrpcExecutor::ThreadMain(void* arg) { size_t subtract_depth = 0; for (;;) { - EXECUTOR_TRACE("[%" PRIdPTR "]: step (sub_depth=%" PRIdPTR ")", ts->id, - subtract_depth); + EXECUTOR_TRACE("(%s) [%" PRIdPTR "]: step (sub_depth=%" PRIdPTR ")", + ts->name, ts->id, subtract_depth); gpr_mu_lock(&ts->mu); ts->depth -= subtract_depth; @@ -159,7 +175,7 @@ void GrpcExecutor::ThreadMain(void* arg) { } if (ts->shutdown) { - EXECUTOR_TRACE("[%" PRIdPTR "]: shutdown", ts->id); + EXECUTOR_TRACE("(%s) [%" PRIdPTR "]: shutdown", ts->name, ts->id); gpr_mu_unlock(&ts->mu); break; } @@ -169,10 +185,10 @@ void GrpcExecutor::ThreadMain(void* arg) { ts->elems = GRPC_CLOSURE_LIST_INIT; gpr_mu_unlock(&ts->mu); - EXECUTOR_TRACE("[%" PRIdPTR "]: execute", ts->id); + EXECUTOR_TRACE("(%s) [%" PRIdPTR "]: execute", ts->name, ts->id); grpc_core::ExecCtx::Get()->InvalidateNow(); - subtract_depth = RunClosures(closures); + subtract_depth = RunClosures(ts->name, closures); } } @@ -188,16 +204,16 @@ void GrpcExecutor::Enqueue(grpc_closure* closure, grpc_error* error, do { retry_push = false; size_t cur_thread_count = - static_cast<size_t>(gpr_atm_no_barrier_load(&num_threads_)); + static_cast<size_t>(gpr_atm_acq_load(&num_threads_)); // If the number of threads is zero(i.e either the executor is not threaded // or already shutdown), then queue the closure on the exec context itself if (cur_thread_count == 0) { #ifndef NDEBUG - EXECUTOR_TRACE("schedule %p (created %s:%d) inline", closure, + EXECUTOR_TRACE("(%s) schedule %p (created %s:%d) inline", name_, closure, closure->file_created, closure->line_created); #else - EXECUTOR_TRACE("schedule %p inline", closure); + EXECUTOR_TRACE("(%s) schedule %p inline", name_, closure); #endif grpc_closure_list_append(grpc_core::ExecCtx::Get()->closure_list(), closure, error); @@ -213,18 +229,18 @@ void GrpcExecutor::Enqueue(grpc_closure* closure, grpc_error* error, } ThreadState* orig_ts = ts; - bool try_new_thread = false; + for (;;) { #ifndef NDEBUG EXECUTOR_TRACE( - "try to schedule %p (%s) (created %s:%d) to thread " + "(%s) try to schedule %p (%s) (created %s:%d) to thread " "%" PRIdPTR, - closure, is_short ? "short" : "long", closure->file_created, + name_, closure, is_short ? "short" : "long", closure->file_created, closure->line_created, ts->id); #else - EXECUTOR_TRACE("try to schedule %p (%s) to thread %" PRIdPTR, closure, - is_short ? "short" : "long", ts->id); + EXECUTOR_TRACE("(%s) try to schedule %p (%s) to thread %" PRIdPTR, name_, + closure, is_short ? "short" : "long", ts->id); #endif gpr_mu_lock(&ts->mu); @@ -236,18 +252,22 @@ void GrpcExecutor::Enqueue(grpc_closure* closure, grpc_error* error, size_t idx = ts->id; ts = &thd_state_[(idx + 1) % cur_thread_count]; if (ts == orig_ts) { - // We cycled through all the threads. Retry enqueue again (by creating - // a new thread) + // We cycled through all the threads. Retry enqueue again by creating + // a new thread + // + // TODO (sreek): There is a potential issue here. We are + // unconditionally setting try_new_thread to true here. What if the + // executor is shutdown OR if cur_thread_count is already equal to + // max_threads ? + // (Fortunately, this is not an issue yet (as of july 2018) because + // there is only one instance of long job in gRPC and hence we will + // not hit this code path) retry_push = true; - // TODO (sreek): What if the executor is shutdown OR if - // cur_thread_count is already equal to max_threads ? (currently - as - // of July 2018, we do not run in to this issue because there is only - // one instance of long job in gRPC. This has to be fixed soon) try_new_thread = true; break; } - continue; + continue; // Try the next thread-state } // == Found the thread state (i.e thread) to enqueue this closure! == @@ -277,13 +297,11 @@ void GrpcExecutor::Enqueue(grpc_closure* closure, grpc_error* error, } if (try_new_thread && gpr_spinlock_trylock(&adding_thread_lock_)) { - cur_thread_count = - static_cast<size_t>(gpr_atm_no_barrier_load(&num_threads_)); + cur_thread_count = static_cast<size_t>(gpr_atm_acq_load(&num_threads_)); if (cur_thread_count < max_threads_) { - // Increment num_threads (Safe to do a no_barrier_store instead of a - // cas because we always increment num_threads under the - // 'adding_thread_lock') - gpr_atm_no_barrier_store(&num_threads_, cur_thread_count + 1); + // Increment num_threads (safe to do a store instead of a cas because we + // always increment num_threads under the 'adding_thread_lock') + gpr_atm_rel_store(&num_threads_, cur_thread_count + 1); thd_state_[cur_thread_count].thd = grpc_core::Thread( name_, &GrpcExecutor::ThreadMain, &thd_state_[cur_thread_count]); @@ -298,60 +316,118 @@ void GrpcExecutor::Enqueue(grpc_closure* closure, grpc_error* error, } while (retry_push); } -static GrpcExecutor* global_executor; +static GrpcExecutor* executors[GRPC_NUM_EXECUTORS]; -void enqueue_long(grpc_closure* closure, grpc_error* error) { - global_executor->Enqueue(closure, error, false /* is_short */); +void default_enqueue_short(grpc_closure* closure, grpc_error* error) { + executors[GRPC_DEFAULT_EXECUTOR]->Enqueue(closure, error, + true /* is_short */); } -void enqueue_short(grpc_closure* closure, grpc_error* error) { - global_executor->Enqueue(closure, error, true /* is_short */); +void default_enqueue_long(grpc_closure* closure, grpc_error* error) { + executors[GRPC_DEFAULT_EXECUTOR]->Enqueue(closure, error, + false /* is_short */); } -// Short-Job executor scheduler -static const grpc_closure_scheduler_vtable global_executor_vtable_short = { - enqueue_short, enqueue_short, "executor-short"}; -static grpc_closure_scheduler global_scheduler_short = { - &global_executor_vtable_short}; +void resolver_enqueue_short(grpc_closure* closure, grpc_error* error) { + executors[GRPC_RESOLVER_EXECUTOR]->Enqueue(closure, error, + true /* is_short */); +} -// Long-job executor scheduler -static const grpc_closure_scheduler_vtable global_executor_vtable_long = { - enqueue_long, enqueue_long, "executor-long"}; -static grpc_closure_scheduler global_scheduler_long = { - &global_executor_vtable_long}; +void resolver_enqueue_long(grpc_closure* closure, grpc_error* error) { + executors[GRPC_RESOLVER_EXECUTOR]->Enqueue(closure, error, + false /* is_short */); +} + +static const grpc_closure_scheduler_vtable + vtables_[GRPC_NUM_EXECUTORS][GRPC_NUM_EXECUTOR_JOB_TYPES] = { + {{&default_enqueue_short, &default_enqueue_short, "def-ex-short"}, + {&default_enqueue_long, &default_enqueue_long, "def-ex-long"}}, + {{&resolver_enqueue_short, &resolver_enqueue_short, "res-ex-short"}, + {&resolver_enqueue_long, &resolver_enqueue_long, "res-ex-long"}}}; + +static grpc_closure_scheduler + schedulers_[GRPC_NUM_EXECUTORS][GRPC_NUM_EXECUTOR_JOB_TYPES] = { + {{&vtables_[GRPC_DEFAULT_EXECUTOR][GRPC_EXECUTOR_SHORT]}, + {&vtables_[GRPC_DEFAULT_EXECUTOR][GRPC_EXECUTOR_LONG]}}, + {{&vtables_[GRPC_RESOLVER_EXECUTOR][GRPC_EXECUTOR_SHORT]}, + {&vtables_[GRPC_RESOLVER_EXECUTOR][GRPC_EXECUTOR_LONG]}}}; // grpc_executor_init() and grpc_executor_shutdown() functions are called in the // the grpc_init() and grpc_shutdown() code paths which are protected by a // global mutex. So it is okay to assume that these functions are thread-safe void grpc_executor_init() { - if (global_executor != nullptr) { - // grpc_executor_init() already called once (and grpc_executor_shutdown() - // wasn't called) + EXECUTOR_TRACE0("grpc_executor_init() enter"); + + // Return if grpc_executor_init() is already called earlier + if (executors[GRPC_DEFAULT_EXECUTOR] != nullptr) { + GPR_ASSERT(executors[GRPC_RESOLVER_EXECUTOR] != nullptr); return; } - global_executor = grpc_core::New<GrpcExecutor>("global-executor"); - global_executor->Init(); + executors[GRPC_DEFAULT_EXECUTOR] = + grpc_core::New<GrpcExecutor>("default-executor"); + executors[GRPC_RESOLVER_EXECUTOR] = + grpc_core::New<GrpcExecutor>("resolver-executor"); + + executors[GRPC_DEFAULT_EXECUTOR]->Init(); + executors[GRPC_RESOLVER_EXECUTOR]->Init(); + + EXECUTOR_TRACE0("grpc_executor_init() done"); +} + +grpc_closure_scheduler* grpc_executor_scheduler(GrpcExecutorType executor_type, + GrpcExecutorJobType job_type) { + return &schedulers_[executor_type][job_type]; +} + +grpc_closure_scheduler* grpc_executor_scheduler(GrpcExecutorJobType job_type) { + return grpc_executor_scheduler(GRPC_DEFAULT_EXECUTOR, job_type); } void grpc_executor_shutdown() { - // Shutdown already called - if (global_executor == nullptr) { + EXECUTOR_TRACE0("grpc_executor_shutdown() enter"); + + // Return if grpc_executor_shutdown() is already called earlier + if (executors[GRPC_DEFAULT_EXECUTOR] == nullptr) { + GPR_ASSERT(executors[GRPC_RESOLVER_EXECUTOR] == nullptr); return; } - global_executor->Shutdown(); - grpc_core::Delete<GrpcExecutor>(global_executor); - global_executor = nullptr; + executors[GRPC_DEFAULT_EXECUTOR]->Shutdown(); + executors[GRPC_RESOLVER_EXECUTOR]->Shutdown(); + + // Delete the executor objects. + // + // NOTE: It is important to call Shutdown() on all executors first before + // calling Delete() because it is possible for one executor (that is not + // shutdown yet) to call Enqueue() on a different executor which is already + // shutdown. This is legal and in such cases, the Enqueue() operation + // effectively "fails" and enqueues that closure on the calling thread's + // exec_ctx. + // + // By ensuring that all executors are shutdown first, we are also ensuring + // that no thread is active across all executors. + + grpc_core::Delete<GrpcExecutor>(executors[GRPC_DEFAULT_EXECUTOR]); + grpc_core::Delete<GrpcExecutor>(executors[GRPC_RESOLVER_EXECUTOR]); + executors[GRPC_DEFAULT_EXECUTOR] = nullptr; + executors[GRPC_RESOLVER_EXECUTOR] = nullptr; + + EXECUTOR_TRACE0("grpc_executor_shutdown() done"); } -bool grpc_executor_is_threaded() { return global_executor->IsThreaded(); } +bool grpc_executor_is_threaded(GrpcExecutorType executor_type) { + GPR_ASSERT(executor_type < GRPC_NUM_EXECUTORS); + return executors[executor_type]->IsThreaded(); +} -void grpc_executor_set_threading(bool enable) { - global_executor->SetThreading(enable); +bool grpc_executor_is_threaded() { + return grpc_executor_is_threaded(GRPC_DEFAULT_EXECUTOR); } -grpc_closure_scheduler* grpc_executor_scheduler(GrpcExecutorJobType job_type) { - return job_type == GRPC_EXECUTOR_SHORT ? &global_scheduler_short - : &global_scheduler_long; +void grpc_executor_set_threading(bool enable) { + EXECUTOR_TRACE("grpc_executor_set_threading(%d) called", enable); + for (int i = 0; i < GRPC_NUM_EXECUTORS; i++) { + executors[i]->SetThreading(enable); + } } diff --git a/src/core/lib/iomgr/executor.h b/src/core/lib/iomgr/executor.h index 395fc52863..8829138c5f 100644 --- a/src/core/lib/iomgr/executor.h +++ b/src/core/lib/iomgr/executor.h @@ -27,7 +27,8 @@ typedef struct { gpr_mu mu; - size_t id; // For debugging purposes + size_t id; // For debugging purposes + const char* name; // Thread state name gpr_cv cv; grpc_closure_list elems; size_t depth; // Number of closures in the closure list @@ -36,7 +37,11 @@ typedef struct { grpc_core::Thread thd; } ThreadState; -typedef enum { GRPC_EXECUTOR_SHORT, GRPC_EXECUTOR_LONG } GrpcExecutorJobType; +typedef enum { + GRPC_EXECUTOR_SHORT = 0, + GRPC_EXECUTOR_LONG, + GRPC_NUM_EXECUTOR_JOB_TYPES // Add new values above this +} GrpcExecutorJobType; class GrpcExecutor { public: @@ -58,7 +63,7 @@ class GrpcExecutor { void Enqueue(grpc_closure* closure, grpc_error* error, bool is_short); private: - static size_t RunClosures(grpc_closure_list list); + static size_t RunClosures(const char* executor_name, grpc_closure_list list); static void ThreadMain(void* arg); const char* name_; @@ -70,14 +75,42 @@ class GrpcExecutor { // == Global executor functions == +typedef enum { + GRPC_DEFAULT_EXECUTOR = 0, + GRPC_RESOLVER_EXECUTOR, + + GRPC_NUM_EXECUTORS // Add new values above this +} GrpcExecutorType; + +// TODO(sreek): Currently we have two executors (available globally): The +// default executor and the resolver executor. +// +// Some of the functions below operate on the DEFAULT executor only while some +// operate of ALL the executors. This is a bit confusing and should be cleaned +// up in future (where we make all the following functions take executor_type +// and/or job_type) + +// Initialize ALL the executors void grpc_executor_init(); +// Shutdown ALL the executors +void grpc_executor_shutdown(); + +// Set the threading mode for ALL the executors +void grpc_executor_set_threading(bool enable); + +// Get the DEFAULT executor scheduler for the given job_type grpc_closure_scheduler* grpc_executor_scheduler(GrpcExecutorJobType job_type); -void grpc_executor_shutdown(); +// Get the executor scheduler for a given executor_type and a job_type +grpc_closure_scheduler* grpc_executor_scheduler(GrpcExecutorType executor_type, + GrpcExecutorJobType job_type); -bool grpc_executor_is_threaded(); +// Return if a given executor is running in threaded mode (i.e if +// grpc_executor_set_threading(true) was called previously on that executor) +bool grpc_executor_is_threaded(GrpcExecutorType executor_type); -void grpc_executor_set_threading(bool enable); +// Return if the DEFAULT executor is threaded +bool grpc_executor_is_threaded(); #endif /* GRPC_CORE_LIB_IOMGR_EXECUTOR_H */ diff --git a/src/core/lib/iomgr/resolve_address_posix.cc b/src/core/lib/iomgr/resolve_address_posix.cc index 7a825643e1..c285d7eca6 100644 --- a/src/core/lib/iomgr/resolve_address_posix.cc +++ b/src/core/lib/iomgr/resolve_address_posix.cc @@ -166,8 +166,9 @@ static void posix_resolve_address(const char* name, const char* default_port, grpc_closure* on_done, grpc_resolved_addresses** addrs) { request* r = static_cast<request*>(gpr_malloc(sizeof(request))); - GRPC_CLOSURE_INIT(&r->request_closure, do_request_thread, r, - grpc_executor_scheduler(GRPC_EXECUTOR_SHORT)); + GRPC_CLOSURE_INIT( + &r->request_closure, do_request_thread, r, + grpc_executor_scheduler(GRPC_RESOLVER_EXECUTOR, GRPC_EXECUTOR_SHORT)); r->name = gpr_strdup(name); r->default_port = gpr_strdup(default_port); r->on_done = on_done; diff --git a/src/core/lib/iomgr/resolve_address_windows.cc b/src/core/lib/iomgr/resolve_address_windows.cc index 71c92615ad..3e977dca2d 100644 --- a/src/core/lib/iomgr/resolve_address_windows.cc +++ b/src/core/lib/iomgr/resolve_address_windows.cc @@ -151,8 +151,9 @@ static void windows_resolve_address(const char* name, const char* default_port, grpc_closure* on_done, grpc_resolved_addresses** addresses) { request* r = (request*)gpr_malloc(sizeof(request)); - GRPC_CLOSURE_INIT(&r->request_closure, do_request_thread, r, - grpc_executor_scheduler(GRPC_EXECUTOR_SHORT)); + GRPC_CLOSURE_INIT( + &r->request_closure, do_request_thread, r, + grpc_executor_scheduler(GRPC_RESOLVER_EXECUTOR, GRPC_EXECUTOR_SHORT)); r->name = gpr_strdup(name); r->default_port = gpr_strdup(default_port); r->on_done = on_done; diff --git a/src/core/lib/security/credentials/alts/check_gcp_environment_linux.cc b/src/core/lib/security/credentials/alts/check_gcp_environment_linux.cc index 7c4d7a71cd..8454fd7558 100644 --- a/src/core/lib/security/credentials/alts/check_gcp_environment_linux.cc +++ b/src/core/lib/security/credentials/alts/check_gcp_environment_linux.cc @@ -41,8 +41,9 @@ namespace internal { bool check_bios_data(const char* bios_data_file) { char* bios_data = read_bios_file(bios_data_file); - bool result = (!strcmp(bios_data, GRPC_ALTS_EXPECT_NAME_GOOGLE)) || - (!strcmp(bios_data, GRPC_ALTS_EXPECT_NAME_GCE)); + bool result = + bios_data && ((!strcmp(bios_data, GRPC_ALTS_EXPECT_NAME_GOOGLE)) || + (!strcmp(bios_data, GRPC_ALTS_EXPECT_NAME_GCE))); gpr_free(bios_data); return result; } diff --git a/src/core/lib/security/security_connector/security_connector.cc b/src/core/lib/security/security_connector/security_connector.cc index cc72bb6164..59cf3a0af1 100644 --- a/src/core/lib/security/security_connector/security_connector.cc +++ b/src/core/lib/security/security_connector/security_connector.cc @@ -57,6 +57,10 @@ static const char* installed_roots_path = INSTALL_PREFIX "/share/grpc/roots.pem"; #endif +#ifndef TSI_OPENSSL_ALPN_SUPPORT +#define TSI_OPENSSL_ALPN_SUPPORT 1 +#endif + /* -- Overridden default roots. -- */ static grpc_ssl_roots_override_callback ssl_roots_override_cb = nullptr; @@ -850,7 +854,8 @@ grpc_auth_context* grpc_ssl_peer_to_auth_context(const tsi_peer* peer) { static grpc_error* ssl_check_peer(grpc_security_connector* sc, const char* peer_name, const tsi_peer* peer, grpc_auth_context** auth_context) { - /* Check the ALPN. */ +#if TSI_OPENSSL_ALPN_SUPPORT + /* Check the ALPN if ALPN is supported. */ const tsi_peer_property* p = tsi_peer_get_property_by_name(peer, TSI_SSL_ALPN_SELECTED_PROTOCOL); if (p == nullptr) { @@ -861,7 +866,7 @@ static grpc_error* ssl_check_peer(grpc_security_connector* sc, return GRPC_ERROR_CREATE_FROM_STATIC_STRING( "Cannot check peer: invalid ALPN value."); } - +#endif /* TSI_OPENSSL_ALPN_SUPPORT */ /* Check the peer name if specified. */ if (peer_name != nullptr && !grpc_ssl_host_matches_name(peer, peer_name)) { char* msg; diff --git a/src/core/lib/surface/call.cc b/src/core/lib/surface/call.cc index b86ff048cf..03b7cedcdb 100644 --- a/src/core/lib/surface/call.cc +++ b/src/core/lib/surface/call.cc @@ -561,7 +561,7 @@ static void destroy_call(void* call, grpc_error* error) { } get_final_status(c, set_status_value_directly, &c->final_info.final_status, - nullptr, c->final_info.error_string); + nullptr, &(c->final_info.error_string)); c->final_info.stats.latency = gpr_time_sub(gpr_now(GPR_CLOCK_MONOTONIC), c->start_time); @@ -573,6 +573,7 @@ static void destroy_call(void* call, grpc_error* error) { grpc_call_stack_destroy(CALL_STACK_FROM_CALL(c), &c->final_info, GRPC_CLOSURE_INIT(&c->release_call, release_call, c, grpc_schedule_on_exec_ctx)); + gpr_free(static_cast<void*>(const_cast<char*>(c->final_info.error_string))); } void grpc_call_ref(grpc_call* c) { gpr_ref(&c->ext_ref); } diff --git a/src/core/lib/surface/version.cc b/src/core/lib/surface/version.cc index ac8cec2980..e92fe2c5a1 100644 --- a/src/core/lib/surface/version.cc +++ b/src/core/lib/surface/version.cc @@ -25,4 +25,4 @@ const char* grpc_version_string(void) { return "6.0.0-dev"; } -const char* grpc_g_stands_for(void) { return "gladiolus"; } +const char* grpc_g_stands_for(void) { return "glider"; } diff --git a/src/cpp/common/version_cc.cc b/src/cpp/common/version_cc.cc index a7b093dfd2..b8fca9a6ee 100644 --- a/src/cpp/common/version_cc.cc +++ b/src/cpp/common/version_cc.cc @@ -22,5 +22,5 @@ #include <grpcpp/grpcpp.h> namespace grpc { -grpc::string Version() { return "1.14.0-dev"; } +grpc::string Version() { return "1.15.0-dev"; } } // namespace grpc diff --git a/src/csharp/Grpc.Core/Grpc.Core.csproj b/src/csharp/Grpc.Core/Grpc.Core.csproj index 6d44be7ddd..fc32271063 100755 --- a/src/csharp/Grpc.Core/Grpc.Core.csproj +++ b/src/csharp/Grpc.Core/Grpc.Core.csproj @@ -46,10 +46,38 @@ <PackagePath>runtimes/win/native/grpc_csharp_ext.x86.dll</PackagePath> <Pack>true</Pack> </Content> - <Content Include="Grpc.Core.targets"> + <Content Include="..\nativelibs\csharp_ext_linux_android_armeabi-v7a\libgrpc_csharp_ext.so"> + <PackagePath>runtimes/monoandroid/armeabi-v7a/libgrpc_csharp_ext.so</PackagePath> + <Pack>true</Pack> + </Content> + <Content Include="..\nativelibs\csharp_ext_linux_android_arm64-v8a\libgrpc_csharp_ext.so"> + <PackagePath>runtimes/monoandroid/arm64-v8a/libgrpc_csharp_ext.so</PackagePath> + <Pack>true</Pack> + </Content> + <Content Include="..\nativelibs\csharp_ext_linux_android_x86\libgrpc_csharp_ext.so"> + <PackagePath>runtimes/monoandroid/x86/libgrpc_csharp_ext.so</PackagePath> + <Pack>true</Pack> + </Content> + <Content Include="..\nativelibs\csharp_ext_macos_ios\libgrpc_csharp_ext.a"> + <PackagePath>runtimes/ios/native/libgrpc_csharp_ext.a</PackagePath> + <Pack>true</Pack> + </Content> + <Content Include="..\nativelibs\csharp_ext_macos_ios\libgrpc.a"> + <PackagePath>runtimes/ios/native/libgrpc.a</PackagePath> + <Pack>true</Pack> + </Content> + <Content Include="build\net45\Grpc.Core.targets"> <PackagePath>build/net45/</PackagePath> <Pack>true</Pack> </Content> + <Content Include="build\MonoAndroid\Grpc.Core.targets"> + <PackagePath>build/MonoAndroid/</PackagePath> + <Pack>true</Pack> + </Content> + <Content Include="build\Xamarin.iOS\Grpc.Core.targets"> + <PackagePath>build/Xamarin.iOS/</PackagePath> + <Pack>true</Pack> + </Content> </ItemGroup> <ItemGroup> diff --git a/src/csharp/Grpc.Core/Internal/NativeExtension.cs b/src/csharp/Grpc.Core/Internal/NativeExtension.cs index d5ec998bbd..f526b913af 100644 --- a/src/csharp/Grpc.Core/Internal/NativeExtension.cs +++ b/src/csharp/Grpc.Core/Internal/NativeExtension.cs @@ -106,7 +106,15 @@ namespace Grpc.Core.Internal /// </summary> private static NativeMethods LoadNativeMethods() { - return PlatformApis.IsUnity ? LoadNativeMethodsUnity() : new NativeMethods(LoadUnmanagedLibrary()); + if (PlatformApis.IsUnity) + { + return LoadNativeMethodsUnity(); + } + if (PlatformApis.IsXamarin) + { + return LoadNativeMethodsXamarin(); + } + return new NativeMethods(LoadUnmanagedLibrary()); } /// <summary> @@ -128,6 +136,20 @@ namespace Grpc.Core.Internal } } + /// <summary> + /// Return native method delegates when running on the Xamarin platform. + /// WARNING: Xamarin support is experimental and work-in-progress. Don't expect it to work. + /// </summary> + private static NativeMethods LoadNativeMethodsXamarin() + { + if (PlatformApis.IsXamarinAndroid) + { + return new NativeMethods(new NativeMethods.DllImportsFromSharedLib()); + } + // not tested yet + return new NativeMethods(new NativeMethods.DllImportsFromStaticLib()); + } + private static string GetAssemblyPath() { var assembly = typeof(NativeExtension).GetTypeInfo().Assembly; diff --git a/src/csharp/Grpc.Core/Internal/NativeLogRedirector.cs b/src/csharp/Grpc.Core/Internal/NativeLogRedirector.cs index bf6440123a..30264acb10 100644 --- a/src/csharp/Grpc.Core/Internal/NativeLogRedirector.cs +++ b/src/csharp/Grpc.Core/Internal/NativeLogRedirector.cs @@ -51,6 +51,7 @@ namespace Grpc.Core.Internal } } + [MonoPInvokeCallback(typeof(GprLogDelegate))] private static void HandleWrite(IntPtr fileStringPtr, int line, ulong threadId, IntPtr severityStringPtr, IntPtr msgPtr) { try @@ -86,4 +87,22 @@ namespace Grpc.Core.Internal } } } + + /// <summary> + /// Use this attribute to mark methods that will be called back from P/Invoke calls. + /// iOS (and probably other AOT platforms) needs to have delegates registered. + /// Instead of depending on Xamarin.iOS for this, we can just create our own, + /// the iOS runtime just checks for the type name. + /// See: https://docs.microsoft.com/en-gb/xamarin/ios/internals/limitations#reverse-callbacks + /// </summary> + [AttributeUsage(AttributeTargets.Method)] + internal sealed class MonoPInvokeCallbackAttribute : Attribute + { + public MonoPInvokeCallbackAttribute(Type type) + { + Type = type; + } + + public Type Type { get; private set; } + } } diff --git a/src/csharp/Grpc.Core/Internal/PlatformApis.cs b/src/csharp/Grpc.Core/Internal/PlatformApis.cs index b90fbccb2b..c501aa89fb 100644 --- a/src/csharp/Grpc.Core/Internal/PlatformApis.cs +++ b/src/csharp/Grpc.Core/Internal/PlatformApis.cs @@ -33,12 +33,18 @@ namespace Grpc.Core.Internal internal static class PlatformApis { const string UnityEngineApplicationClassName = "UnityEngine.Application, UnityEngine"; + const string XamarinAndroidObjectClassName = "Java.Lang.Object, Mono.Android"; + const string XamarinIOSObjectClassName = "Foundation.NSObject, Xamarin.iOS"; + static readonly bool isLinux; static readonly bool isMacOSX; static readonly bool isWindows; static readonly bool isMono; static readonly bool isNetCore; static readonly bool isUnity; + static readonly bool isXamarin; + static readonly bool isXamarinIOS; + static readonly bool isXamarinAndroid; static PlatformApis() { @@ -58,6 +64,9 @@ namespace Grpc.Core.Internal #endif isMono = Type.GetType("Mono.Runtime") != null; isUnity = Type.GetType(UnityEngineApplicationClassName) != null; + isXamarinIOS = Type.GetType(XamarinIOSObjectClassName) != null; + isXamarinAndroid = Type.GetType(XamarinAndroidObjectClassName) != null; + isXamarin = isXamarinIOS || isXamarinAndroid; } public static bool IsLinux @@ -89,6 +98,31 @@ namespace Grpc.Core.Internal } /// <summary> + /// true if running on a Xamarin platform (either Xamarin.Android or Xamarin.iOS), + /// false otherwise. + /// </summary> + public static bool IsXamarin + { + get { return isXamarin; } + } + + /// <summary> + /// true if running on Xamarin.iOS, false otherwise. + /// </summary> + public static bool IsXamarinIOS + { + get { return isXamarinIOS; } + } + + /// <summary> + /// true if running on Xamarin.Android, false otherwise. + /// </summary> + public static bool IsXamarinAndroid + { + get { return isXamarinAndroid; } + } + + /// <summary> /// true if running on .NET Core (CoreCLR), false otherwise. /// </summary> public static bool IsNetCore diff --git a/src/csharp/Grpc.Core/Version.csproj.include b/src/csharp/Grpc.Core/Version.csproj.include index 2b45e7ae20..1a746acdd4 100755 --- a/src/csharp/Grpc.Core/Version.csproj.include +++ b/src/csharp/Grpc.Core/Version.csproj.include @@ -1,7 +1,7 @@ <!-- This file is generated --> <Project> <PropertyGroup> - <GrpcCsharpVersion>1.14.0-dev</GrpcCsharpVersion> + <GrpcCsharpVersion>1.15.0-dev</GrpcCsharpVersion> <GoogleProtobufVersion>3.5.1</GoogleProtobufVersion> </PropertyGroup> </Project> diff --git a/src/csharp/Grpc.Core/VersionInfo.cs b/src/csharp/Grpc.Core/VersionInfo.cs index c4531e5275..295e0f2577 100644 --- a/src/csharp/Grpc.Core/VersionInfo.cs +++ b/src/csharp/Grpc.Core/VersionInfo.cs @@ -33,11 +33,11 @@ namespace Grpc.Core /// <summary> /// Current <c>AssemblyFileVersion</c> of gRPC C# assemblies /// </summary> - public const string CurrentAssemblyFileVersion = "1.14.0.0"; + public const string CurrentAssemblyFileVersion = "1.15.0.0"; /// <summary> /// Current version of gRPC C# /// </summary> - public const string CurrentVersion = "1.14.0-dev"; + public const string CurrentVersion = "1.15.0-dev"; } } diff --git a/src/csharp/Grpc.Core/build/MonoAndroid/Grpc.Core.targets b/src/csharp/Grpc.Core/build/MonoAndroid/Grpc.Core.targets new file mode 100644 index 0000000000..d75e5a2f2f --- /dev/null +++ b/src/csharp/Grpc.Core/build/MonoAndroid/Grpc.Core.targets @@ -0,0 +1,28 @@ +<?xml version="1.0" encoding="utf-8"?> +<Project xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> + <PropertyGroup> + <_GrpcCoreNugetNativePath Condition="'$(_GrpcCoreNugetNativePath)' == ''">$(MSBuildThisFileDirectory)..\..\</_GrpcCoreNugetNativePath> + </PropertyGroup> + + <ItemGroup Condition="'$(TargetFrameworkIdentifier)' == 'MonoAndroid'"> + <AndroidNativeLibrary Include="$(_GrpcCoreNugetNativePath)runtimes\monoandroid\arm64-v8a\libgrpc_csharp_ext.so"> + <CopyToOutputDirectory>Always</CopyToOutputDirectory> + <Abi>arm64-v8a</Abi> + </AndroidNativeLibrary> + </ItemGroup> + + <ItemGroup Condition="'$(TargetFrameworkIdentifier)' == 'MonoAndroid'"> + <AndroidNativeLibrary Include="$(_GrpcCoreNugetNativePath)runtimes\monoandroid\armeabi-v7a\libgrpc_csharp_ext.so"> + <CopyToOutputDirectory>Always</CopyToOutputDirectory> + <Abi>armeabi-v7a</Abi> + </AndroidNativeLibrary> + </ItemGroup> + + <ItemGroup Condition="'$(TargetFrameworkIdentifier)' == 'MonoAndroid'"> + <AndroidNativeLibrary Include="$(_GrpcCoreNugetNativePath)runtimes\monoandroid\x86\libgrpc_csharp_ext.so"> + <CopyToOutputDirectory>Always</CopyToOutputDirectory> + <Abi>x86</Abi> + </AndroidNativeLibrary> + </ItemGroup> + +</Project> diff --git a/src/csharp/Grpc.Core/build/Xamarin.iOS/Grpc.Core.targets b/src/csharp/Grpc.Core/build/Xamarin.iOS/Grpc.Core.targets new file mode 100644 index 0000000000..658158f6ea --- /dev/null +++ b/src/csharp/Grpc.Core/build/Xamarin.iOS/Grpc.Core.targets @@ -0,0 +1,15 @@ +<?xml version="1.0" encoding="utf-8"?> +<Project xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> + + <ItemGroup> + <NativeReference Include="$(MSBuildThisFileDirectory)..\..\runtimes\ios\native\libgrpc_csharp_ext.a"> + <Kind>Static</Kind> + <ForceLoad>True</ForceLoad> + </NativeReference> + <NativeReference Include="$(MSBuildThisFileDirectory)..\..\runtimes\ios\native\libgrpc.a"> + <Kind>Static</Kind> + <ForceLoad>True</ForceLoad> + </NativeReference> + </ItemGroup> + +</Project> diff --git a/src/csharp/Grpc.Core/Grpc.Core.targets b/src/csharp/Grpc.Core/build/net45/Grpc.Core.targets index cce53db82b..cce53db82b 100644 --- a/src/csharp/Grpc.Core/Grpc.Core.targets +++ b/src/csharp/Grpc.Core/build/net45/Grpc.Core.targets diff --git a/src/csharp/build_packages_dotnetcli.bat b/src/csharp/build_packages_dotnetcli.bat index 394b859e0b..67af258354 100755 --- a/src/csharp/build_packages_dotnetcli.bat +++ b/src/csharp/build_packages_dotnetcli.bat @@ -13,7 +13,7 @@ @rem limitations under the License. @rem Current package versions -set VERSION=1.14.0-dev +set VERSION=1.15.0-dev @rem Adjust the location of nuget.exe set NUGET=C:\nuget\nuget.exe diff --git a/src/csharp/build_packages_dotnetcli.sh b/src/csharp/build_packages_dotnetcli.sh index 273d745f17..88b4d63538 100755 --- a/src/csharp/build_packages_dotnetcli.sh +++ b/src/csharp/build_packages_dotnetcli.sh @@ -45,8 +45,8 @@ dotnet pack --configuration Release Grpc.Auth --output ../../../artifacts dotnet pack --configuration Release Grpc.HealthCheck --output ../../../artifacts dotnet pack --configuration Release Grpc.Reflection --output ../../../artifacts -nuget pack Grpc.nuspec -Version "1.14.0-dev" -OutputDirectory ../../artifacts -nuget pack Grpc.Core.NativeDebug.nuspec -Version "1.14.0-dev" -OutputDirectory ../../artifacts -nuget pack Grpc.Tools.nuspec -Version "1.14.0-dev" -OutputDirectory ../../artifacts +nuget pack Grpc.nuspec -Version "1.15.0-dev" -OutputDirectory ../../artifacts +nuget pack Grpc.Core.NativeDebug.nuspec -Version "1.15.0-dev" -OutputDirectory ../../artifacts +nuget pack Grpc.Tools.nuspec -Version "1.15.0-dev" -OutputDirectory ../../artifacts (cd ../../artifacts && zip csharp_nugets_dotnetcli.zip *.nupkg) diff --git a/src/csharp/experimental/build_native_ext_for_android.sh b/src/csharp/experimental/build_native_ext_for_android.sh index 8197df7c53..6518e03c53 100755 --- a/src/csharp/experimental/build_native_ext_for_android.sh +++ b/src/csharp/experimental/build_native_ext_for_android.sh @@ -23,17 +23,30 @@ mkdir -p build cd build # set to the location where Android SDK is installed -# e.g. ANDROID_NDK_PATH="$HOME/android-ndk-r16b" +# e.g. ANDROID_SDK_PATH="$HOME/Android/Sdk" -cmake ../.. \ - -DCMAKE_SYSTEM_NAME=Android \ - -DCMAKE_SYSTEM_VERSION=15 \ - -DCMAKE_ANDROID_ARCH_ABI=armeabi-v7a \ +# set to location where Android NDK is installed, usually a subfolder of Android SDK +# to install the Android NKD, use the "sdkmanager" tool +# e.g. ANDROID_NDK_PATH=${ANDROID_SDK_PATH}/ndk-bundle + +# set to location of the cmake executable from the Android SDK +# to install cmake, use the "sdkmanager" tool +# e.g. ANDROID_SDK_CMAKE=${ANDROID_SDK_PATH}/cmake/3.6.4111459/bin/cmake + +# ANDROID_ABI in ('arm64-v8a', 'armeabi-v7a') +# e.g. ANDROID_ABI=armeabi-v7a + +# android-19 corresponds to Kitkat 4.4 +${ANDROID_SDK_CMAKE} ../.. \ + -DCMAKE_TOOLCHAIN_FILE="${ANDROID_NDK_PATH}/build/cmake/android.toolchain.cmake" \ -DCMAKE_ANDROID_NDK="${ANDROID_NDK_PATH}" \ -DCMAKE_ANDROID_STL_TYPE=c++_static \ -DRUN_HAVE_POSIX_REGEX=0 \ -DRUN_HAVE_STD_REGEX=0 \ -DRUN_HAVE_STEADY_CLOCK=0 \ - -DCMAKE_BUILD_TYPE=Release + -DCMAKE_BUILD_TYPE=Release \ + -DANDROID_PLATFORM=android-19 \ + -DANDROID_ABI="${ANDROID_ABI}" \ + -DANDROID_NDK="${ANDROID_NDK_PATH}" make -j4 grpc_csharp_ext diff --git a/src/csharp/experimental/build_native_ext_for_ios.sh b/src/csharp/experimental/build_native_ext_for_ios.sh new file mode 100755 index 0000000000..69c9cdf021 --- /dev/null +++ b/src/csharp/experimental/build_native_ext_for_ios.sh @@ -0,0 +1,62 @@ +#!/bin/sh +# Copyright 2018 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Helper script to crosscompile grpc_csharp_ext native extension for Android. + +set -ex + +cd "$(dirname "$0")/../../.." + +# Usage: build <iphoneos|iphonesimulator> <arm64|x86_64|...> +function build { + SDK="$1" + ARCH="$2" + + PATH_AR="$(xcrun --sdk $SDK --find ar)" + PATH_CC="$(xcrun --sdk $SDK --find clang)" + PATH_CXX="$(xcrun --sdk $SDK --find clang++)" + + # TODO(jtattermusch): add -mios-version-min=6.0 and -Wl,ios_version_min=6.0 + CPPFLAGS="-O2 -Wframe-larger-than=16384 -arch $ARCH -isysroot $(xcrun --sdk $SDK --show-sdk-path) -DPB_NO_PACKED_STRUCTS=1" + LDFLAGS="-arch $ARCH -isysroot $(xcrun --sdk $SDK --show-sdk-path)" + + # TODO(jtattermusch): revisit the build arguments + make -j4 static_csharp \ + VALID_CONFIG_ios_$ARCH="1" \ + CC_ios_$ARCH="$PATH_CC" \ + CXX_ios_$ARCH="$PATH_CXX" \ + LD_ios_$ARCH="$PATH_CC" \ + LDXX_ios_$ARCH="$PATH_CXX" \ + CPPFLAGS_ios_$ARCH="$CPPFLAGS" \ + LDFLAGS_ios_$ARCH="$LDFLAGS" \ + DEFINES_ios_$ARCH="NDEBUG" \ + CONFIG="ios_$ARCH" +} + +# Usage: fatten <grpc_csharp_ext|...> +function fatten { + LIB_NAME="$1" + + mkdir -p libs/ios + lipo -create -output libs/ios/lib$LIB_NAME.a \ + libs/ios_arm64/lib$LIB_NAME.a \ + libs/ios_x86_64/lib$LIB_NAME.a +} + +build iphoneos arm64 +build iphonesimulator x86_64 + +fatten grpc +fatten grpc_csharp_ext diff --git a/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec b/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec index 28dfffd863..6ad9166b32 100644 --- a/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec +++ b/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec @@ -42,7 +42,7 @@ Pod::Spec.new do |s| # exclamation mark ensures that other "regular" pods will be able to find it as it'll be installed # before them. s.name = '!ProtoCompiler-gRPCPlugin' - v = '1.14.0-dev' + v = '1.15.0-dev' s.version = v s.summary = 'The gRPC ProtoC plugin generates Objective-C files from .proto services.' s.description = <<-DESC diff --git a/src/objective-c/GRPCClient/private/version.h b/src/objective-c/GRPCClient/private/version.h index 60571b1f43..52fe0dd050 100644 --- a/src/objective-c/GRPCClient/private/version.h +++ b/src/objective-c/GRPCClient/private/version.h @@ -22,4 +22,4 @@ // instead. This file can be regenerated from the template by running // `tools/buildgen/generate_projects.sh`. -#define GRPC_OBJC_VERSION_STRING @"1.14.0-dev" +#define GRPC_OBJC_VERSION_STRING @"1.15.0-dev" diff --git a/src/objective-c/tests/version.h b/src/objective-c/tests/version.h index fafcac6507..d532eed245 100644 --- a/src/objective-c/tests/version.h +++ b/src/objective-c/tests/version.h @@ -22,5 +22,5 @@ // instead. This file can be regenerated from the template by running // `tools/buildgen/generate_projects.sh`. -#define GRPC_OBJC_VERSION_STRING @"1.14.0-dev" +#define GRPC_OBJC_VERSION_STRING @"1.15.0-dev" #define GRPC_C_VERSION_STRING @"6.0.0-dev" diff --git a/src/php/composer.json b/src/php/composer.json index 94918417d9..b6716b8b4c 100644 --- a/src/php/composer.json +++ b/src/php/composer.json @@ -2,7 +2,7 @@ "name": "grpc/grpc-dev", "description": "gRPC library for PHP - for Developement use only", "license": "Apache-2.0", - "version": "1.14.0", + "version": "1.15.0", "require": { "php": ">=5.5.0", "google/protobuf": "^v3.3.0" diff --git a/src/php/ext/grpc/version.h b/src/php/ext/grpc/version.h index c985863cb5..99dd9d68f0 100644 --- a/src/php/ext/grpc/version.h +++ b/src/php/ext/grpc/version.h @@ -20,6 +20,6 @@ #ifndef VERSION_H #define VERSION_H -#define PHP_GRPC_VERSION "1.14.0dev" +#define PHP_GRPC_VERSION "1.15.0dev" #endif /* VERSION_H */ diff --git a/src/php/lib/Grpc/BaseStub.php b/src/php/lib/Grpc/BaseStub.php index ecb419ac8f..fe81e37761 100644 --- a/src/php/lib/Grpc/BaseStub.php +++ b/src/php/lib/Grpc/BaseStub.php @@ -83,10 +83,11 @@ class BaseStub } private static function updateOpts($opts) { - $package_config = json_decode( - file_get_contents(dirname(__FILE__).'/../../composer.json'), - true - ); + if (!file_exists($composerFile = __DIR__.'/../../composer.json')) { + // for grpc/grpc-php subpackage + $composerFile = __DIR__.'/../composer.json'; + } + $package_config = json_decode(file_get_contents($composerFile), true); if (!empty($opts['grpc.primary_user_agent'])) { $opts['grpc.primary_user_agent'] .= ' '; } else { diff --git a/src/python/grpcio/grpc/_grpcio_metadata.py b/src/python/grpcio/grpc/_grpcio_metadata.py index b336e6aae5..c33911ebc1 100644 --- a/src/python/grpcio/grpc/_grpcio_metadata.py +++ b/src/python/grpcio/grpc/_grpcio_metadata.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio/grpc/_grpcio_metadata.py.template`!!! -__version__ = """1.14.0.dev0""" +__version__ = """1.15.0.dev0""" diff --git a/src/python/grpcio/grpc_version.py b/src/python/grpcio/grpc_version.py index 2eeaabc8b8..9337800a33 100644 --- a/src/python/grpcio/grpc_version.py +++ b/src/python/grpcio/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio/grpc_version.py.template`!!! -VERSION = '1.14.0.dev0' +VERSION = '1.15.0.dev0' diff --git a/src/python/grpcio_health_checking/grpc_health/v1/health.py b/src/python/grpcio_health_checking/grpc_health/v1/health.py index c8498104b1..0583659428 100644 --- a/src/python/grpcio_health_checking/grpc_health/v1/health.py +++ b/src/python/grpcio_health_checking/grpc_health/v1/health.py @@ -17,11 +17,13 @@ import threading import grpc -from grpc_health.v1 import health_pb2 -from grpc_health.v1 import health_pb2_grpc +from grpc_health.v1 import health_pb2 as _health_pb2 +from grpc_health.v1 import health_pb2_grpc as _health_pb2_grpc +SERVICE_NAME = _health_pb2.DESCRIPTOR.services_by_name['Health'].full_name -class HealthServicer(health_pb2_grpc.HealthServicer): + +class HealthServicer(_health_pb2_grpc.HealthServicer): """Servicer handling RPCs for service statuses.""" def __init__(self): @@ -33,9 +35,9 @@ class HealthServicer(health_pb2_grpc.HealthServicer): status = self._server_status.get(request.service) if status is None: context.set_code(grpc.StatusCode.NOT_FOUND) - return health_pb2.HealthCheckResponse() + return _health_pb2.HealthCheckResponse() else: - return health_pb2.HealthCheckResponse(status=status) + return _health_pb2.HealthCheckResponse(status=status) def set(self, service, status): """Sets the status of a service. diff --git a/src/python/grpcio_health_checking/grpc_version.py b/src/python/grpcio_health_checking/grpc_version.py index f36de8d0fa..3b84f7a4c5 100644 --- a/src/python/grpcio_health_checking/grpc_version.py +++ b/src/python/grpcio_health_checking/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_health_checking/grpc_version.py.template`!!! -VERSION = '1.14.0.dev0' +VERSION = '1.15.0.dev0' diff --git a/src/python/grpcio_reflection/grpc_reflection/v1alpha/reflection.py b/src/python/grpcio_reflection/grpc_reflection/v1alpha/reflection.py index 0c564f10e5..6df1a36426 100644 --- a/src/python/grpcio_reflection/grpc_reflection/v1alpha/reflection.py +++ b/src/python/grpcio_reflection/grpc_reflection/v1alpha/reflection.py @@ -17,15 +17,17 @@ import grpc from google.protobuf import descriptor_pb2 from google.protobuf import descriptor_pool -from grpc_reflection.v1alpha import reflection_pb2 -from grpc_reflection.v1alpha import reflection_pb2_grpc +from grpc_reflection.v1alpha import reflection_pb2 as _reflection_pb2 +from grpc_reflection.v1alpha import reflection_pb2_grpc as _reflection_pb2_grpc _POOL = descriptor_pool.Default() +SERVICE_NAME = _reflection_pb2.DESCRIPTOR.services_by_name[ + 'ServerReflection'].full_name def _not_found_error(): - return reflection_pb2.ServerReflectionResponse( - error_response=reflection_pb2.ErrorResponse( + return _reflection_pb2.ServerReflectionResponse( + error_response=_reflection_pb2.ErrorResponse( error_code=grpc.StatusCode.NOT_FOUND.value[0], error_message=grpc.StatusCode.NOT_FOUND.value[1].encode(), )) @@ -35,12 +37,12 @@ def _file_descriptor_response(descriptor): proto = descriptor_pb2.FileDescriptorProto() descriptor.CopyToProto(proto) serialized_proto = proto.SerializeToString() - return reflection_pb2.ServerReflectionResponse( - file_descriptor_response=reflection_pb2.FileDescriptorResponse( + return _reflection_pb2.ServerReflectionResponse( + file_descriptor_response=_reflection_pb2.FileDescriptorResponse( file_descriptor_proto=(serialized_proto,)),) -class ReflectionServicer(reflection_pb2_grpc.ServerReflectionServicer): +class ReflectionServicer(_reflection_pb2_grpc.ServerReflectionServicer): """Servicer handling RPCs for service statuses.""" def __init__(self, service_names, pool=None): @@ -94,17 +96,17 @@ class ReflectionServicer(reflection_pb2_grpc.ServerReflectionServicer): except KeyError: return _not_found_error() else: - return reflection_pb2.ServerReflectionResponse( - all_extension_numbers_response=reflection_pb2. + return _reflection_pb2.ServerReflectionResponse( + all_extension_numbers_response=_reflection_pb2. ExtensionNumberResponse( base_type_name=message_descriptor.full_name, extension_number=extension_numbers)) def _list_services(self): - return reflection_pb2.ServerReflectionResponse( - list_services_response=reflection_pb2.ListServiceResponse( + return _reflection_pb2.ServerReflectionResponse( + list_services_response=_reflection_pb2.ListServiceResponse( service=[ - reflection_pb2.ServiceResponse(name=service_name) + _reflection_pb2.ServiceResponse(name=service_name) for service_name in self._service_names ])) @@ -126,8 +128,8 @@ class ReflectionServicer(reflection_pb2_grpc.ServerReflectionServicer): elif request.HasField('list_services'): yield self._list_services() else: - yield reflection_pb2.ServerReflectionResponse( - error_response=reflection_pb2.ErrorResponse( + yield _reflection_pb2.ServerReflectionResponse( + error_response=_reflection_pb2.ErrorResponse( error_code=grpc.StatusCode.INVALID_ARGUMENT.value[0], error_message=grpc.StatusCode.INVALID_ARGUMENT.value[1] .encode(), @@ -142,5 +144,5 @@ def enable_server_reflection(service_names, server, pool=None): server: grpc.Server to which reflection service will be added. pool: DescriptorPool object to use (descriptor_pool.Default() if None). """ - reflection_pb2_grpc.add_ServerReflectionServicer_to_server( + _reflection_pb2_grpc.add_ServerReflectionServicer_to_server( ReflectionServicer(service_names, pool=pool), server) diff --git a/src/python/grpcio_reflection/grpc_version.py b/src/python/grpcio_reflection/grpc_version.py index 2249b07d9d..7b0e48ea23 100644 --- a/src/python/grpcio_reflection/grpc_version.py +++ b/src/python/grpcio_reflection/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_reflection/grpc_version.py.template`!!! -VERSION = '1.14.0.dev0' +VERSION = '1.15.0.dev0' diff --git a/src/python/grpcio_testing/grpc_version.py b/src/python/grpcio_testing/grpc_version.py index a5b275aff6..df9953fa25 100644 --- a/src/python/grpcio_testing/grpc_version.py +++ b/src/python/grpcio_testing/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_testing/grpc_version.py.template`!!! -VERSION = '1.14.0.dev0' +VERSION = '1.15.0.dev0' diff --git a/src/python/grpcio_tests/grpc_version.py b/src/python/grpcio_tests/grpc_version.py index 816dfb55bc..b2cf129e4f 100644 --- a/src/python/grpcio_tests/grpc_version.py +++ b/src/python/grpcio_tests/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_tests/grpc_version.py.template`!!! -VERSION = '1.14.0.dev0' +VERSION = '1.15.0.dev0' diff --git a/src/python/grpcio_tests/tests/health_check/_health_servicer_test.py b/src/python/grpcio_tests/tests/health_check/_health_servicer_test.py index 3cbbb8de33..350b5eebe5 100644 --- a/src/python/grpcio_tests/tests/health_check/_health_servicer_test.py +++ b/src/python/grpcio_tests/tests/health_check/_health_servicer_test.py @@ -73,6 +73,9 @@ class HealthServicerTest(unittest.TestCase): self.assertEqual(grpc.StatusCode.NOT_FOUND, context.exception.code()) + def test_health_service_name(self): + self.assertEqual(health.SERVICE_NAME, 'grpc.health.v1.Health') + if __name__ == '__main__': unittest.main(verbosity=2) diff --git a/src/python/grpcio_tests/tests/reflection/_reflection_servicer_test.py b/src/python/grpcio_tests/tests/reflection/_reflection_servicer_test.py index 7ffdba6a67..bcd9e14a38 100644 --- a/src/python/grpcio_tests/tests/reflection/_reflection_servicer_test.py +++ b/src/python/grpcio_tests/tests/reflection/_reflection_servicer_test.py @@ -171,6 +171,10 @@ class ReflectionServicerTest(unittest.TestCase): for name in _SERVICE_NAMES))),) self.assertSequenceEqual(expected_responses, responses) + def testReflectionServiceName(self): + self.assertEqual(reflection.SERVICE_NAME, + 'grpc.reflection.v1alpha.ServerReflection') + if __name__ == '__main__': unittest.main(verbosity=2) diff --git a/src/python/grpcio_tests/tests/unit/beta/_utilities_test.py b/src/python/grpcio_tests/tests/unit/beta/_utilities_test.py index aebee4da96..e042262796 100644 --- a/src/python/grpcio_tests/tests/unit/beta/_utilities_test.py +++ b/src/python/grpcio_tests/tests/unit/beta/_utilities_test.py @@ -41,6 +41,7 @@ class _Callback(object): return self._value +@unittest.skip('https://github.com/grpc/grpc/issues/16134') class ChannelConnectivityTest(unittest.TestCase): def test_lonely_channel_connectivity(self): diff --git a/src/ruby/ext/grpc/extconf.rb b/src/ruby/ext/grpc/extconf.rb index 4760f33e38..505357021e 100644 --- a/src/ruby/ext/grpc/extconf.rb +++ b/src/ruby/ext/grpc/extconf.rb @@ -15,31 +15,6 @@ require 'etc' require 'mkmf' -LIBDIR = RbConfig::CONFIG['libdir'] -INCLUDEDIR = RbConfig::CONFIG['includedir'] - -HEADER_DIRS = [ - # Search /opt/local (Mac source install) - '/opt/local/include', - - # Search /usr/local (Source install) - '/usr/local/include', - - # Check the ruby install locations - INCLUDEDIR -] - -LIB_DIRS = [ - # Search /opt/local (Mac source install) - '/opt/local/lib', - - # Search /usr/local (Source install) - '/usr/local/lib', - - # Check the ruby install locations - LIBDIR -] - windows = RUBY_PLATFORM =~ /mingw|mswin/ bsd = RUBY_PLATFORM =~ /bsd/ @@ -110,7 +85,7 @@ if grpc_config == 'opt' o.puts i end o.puts - o.puts 'strip:' + o.puts 'strip: $(DLLIB)' o.puts "\t$(ECHO) Stripping $(DLLIB)" o.puts "\t$(Q) #{strip_tool} $(DLLIB)" end diff --git a/src/ruby/lib/grpc/version.rb b/src/ruby/lib/grpc/version.rb index 902d59bd71..5c21a5b168 100644 --- a/src/ruby/lib/grpc/version.rb +++ b/src/ruby/lib/grpc/version.rb @@ -14,5 +14,5 @@ # GRPC contains the General RPC module. module GRPC - VERSION = '1.14.0.dev' + VERSION = '1.15.0.dev' end diff --git a/src/ruby/tools/version.rb b/src/ruby/tools/version.rb index dad09bb965..1b5e2c362b 100644 --- a/src/ruby/tools/version.rb +++ b/src/ruby/tools/version.rb @@ -14,6 +14,6 @@ module GRPC module Tools - VERSION = '1.14.0.dev' + VERSION = '1.15.0.dev' end end diff --git a/templates/Makefile.template b/templates/Makefile.template index 628056ed4d..50b81e5f9f 100644 --- a/templates/Makefile.template +++ b/templates/Makefile.template @@ -921,6 +921,16 @@ % endfor + static_csharp: static_c \ + % for lib in libs: + % if 'Makefile' in lib.get('build_system', ['Makefile']): + % if lib.build == 'all' and lib.language == 'csharp': + $(LIBDIR)/$(CONFIG)/lib${lib.name}.a\ + % endif + % endif + % endfor + + shared: shared_c shared_cxx shared_c: pc_c pc_c_unsecure cache.mk\ diff --git a/test/core/security/check_gcp_environment_linux_test.cc b/test/core/security/check_gcp_environment_linux_test.cc index 3acd5b6ae4..b01471abd3 100644 --- a/test/core/security/check_gcp_environment_linux_test.cc +++ b/test/core/security/check_gcp_environment_linux_test.cc @@ -69,6 +69,7 @@ static void test_gcp_environment_check_failure() { GPR_ASSERT(!check_bios_data_linux_test("Amazon")); GPR_ASSERT(!check_bios_data_linux_test("Google-Chrome\t\t")); GPR_ASSERT(!check_bios_data_linux_test("Amazon")); + GPR_ASSERT(!check_bios_data_linux_test("\n")); } int main(int argc, char** argv) { diff --git a/test/cpp/end2end/client_lb_end2end_test.cc b/test/cpp/end2end/client_lb_end2end_test.cc index 8896fc6cae..c5a73a2469 100644 --- a/test/cpp/end2end/client_lb_end2end_test.cc +++ b/test/cpp/end2end/client_lb_end2end_test.cc @@ -279,9 +279,14 @@ class ClientLbEnd2endTest : public ::testing::Test { void WaitForServer( const std::unique_ptr<grpc::testing::EchoTestService::Stub>& stub, - size_t server_idx, const grpc_core::DebugLocation& location) { + size_t server_idx, const grpc_core::DebugLocation& location, + bool ignore_failure = false) { do { - CheckRpcSendOk(stub, location); + if (ignore_failure) { + SendRpc(stub); + } else { + CheckRpcSendOk(stub, location); + } } while (servers_[server_idx]->service_.request_count() == 0); ResetCounters(); } @@ -507,6 +512,37 @@ TEST_F(ClientLbEnd2endTest, PickFirstManyUpdates) { EXPECT_EQ("pick_first", channel->GetLoadBalancingPolicyName()); } +TEST_F(ClientLbEnd2endTest, PickFirstReresolutionNoSelected) { + // Prepare the ports for up servers and down servers. + const int kNumServers = 3; + const int kNumAliveServers = 1; + StartServers(kNumAliveServers); + std::vector<int> alive_ports, dead_ports; + for (size_t i = 0; i < kNumServers; ++i) { + if (i < kNumAliveServers) { + alive_ports.emplace_back(servers_[i]->port_); + } else { + dead_ports.emplace_back(grpc_pick_unused_port_or_die()); + } + } + auto channel = BuildChannel("pick_first"); + auto stub = BuildStub(channel); + // The initial resolution only contains dead ports. There won't be any + // selected subchannel. Re-resolution will return the same result. + SetNextResolution(dead_ports); + gpr_log(GPR_INFO, "****** INITIAL RESOLUTION SET *******"); + for (size_t i = 0; i < 10; ++i) CheckRpcSendFailure(stub); + // Set a re-resolution result that contains reachable ports, so that the + // pick_first LB policy can recover soon. + SetNextResolutionUponError(alive_ports); + gpr_log(GPR_INFO, "****** RE-RESOLUTION SET *******"); + WaitForServer(stub, 0, DEBUG_LOCATION, true /* ignore_failure */); + CheckRpcSendOk(stub, DEBUG_LOCATION); + EXPECT_EQ(servers_[0]->service_.request_count(), 1); + // Check LB policy name for the channel. + EXPECT_EQ("pick_first", channel->GetLoadBalancingPolicyName()); +} + TEST_F(ClientLbEnd2endTest, RoundRobin) { // Start servers and send one RPC per server. const int kNumServers = 3; diff --git a/test/cpp/qps/BUILD b/test/cpp/qps/BUILD index e7d093c71a..b958c75fc7 100644 --- a/test/cpp/qps/BUILD +++ b/test/cpp/qps/BUILD @@ -34,11 +34,13 @@ grpc_cc_library( "qps_worker.cc", "server_async.cc", "server_sync.cc", + "qps_server_builder.cc", ], hdrs = [ "client.h", "qps_worker.h", "server.h", + "qps_server_builder.h", ], deps = [ ":histogram", @@ -55,6 +57,10 @@ grpc_cc_library( "//test/core/util:gpr_test_util", "//test/core/util:grpc_test_util", "//test/cpp/util:test_util", + "//test/cpp/util:test_config", + ], + external_deps = [ + "gflags", ], ) diff --git a/test/cpp/qps/benchmark_config.cc b/test/cpp/qps/benchmark_config.cc index a4fd9de820..5fd0f00038 100644 --- a/test/cpp/qps/benchmark_config.cc +++ b/test/cpp/qps/benchmark_config.cc @@ -22,6 +22,8 @@ #include <grpcpp/create_channel.h> #include <grpcpp/security/credentials.h> +#include "test/cpp/util/test_credentials_provider.h" + DEFINE_bool(enable_log_reporter, true, "Enable reporting of benchmark results through GprLog"); @@ -44,6 +46,10 @@ DEFINE_string(rpc_reporter_server_address, "", DEFINE_bool(enable_rpc_reporter, false, "Enable use of RPC reporter"); +DEFINE_string( + rpc_reporter_credential_type, grpc::testing::kInsecureCredentialsType, + "Credential type for communication to the QPS benchmark report server"); + // In some distros, gflags is in the namespace google, and in some others, // in gflags. This hack is enabling us to find both. namespace google {} @@ -65,11 +71,14 @@ static std::shared_ptr<Reporter> InitBenchmarkReporters() { new JsonReporter("JsonReporter", FLAGS_scenario_result_file))); } if (FLAGS_enable_rpc_reporter) { + ChannelArguments channel_args; + std::shared_ptr<ChannelCredentials> channel_creds = + testing::GetCredentialsProvider()->GetChannelCredentials( + FLAGS_rpc_reporter_credential_type, &channel_args); GPR_ASSERT(!FLAGS_rpc_reporter_server_address.empty()); composite_reporter->add(std::unique_ptr<Reporter>(new RpcReporter( - "RpcReporter", - grpc::CreateChannel(FLAGS_rpc_reporter_server_address, - grpc::InsecureChannelCredentials())))); + "RpcReporter", grpc::CreateChannel(FLAGS_rpc_reporter_server_address, + channel_creds)))); } return std::shared_ptr<Reporter>(composite_reporter); diff --git a/test/cpp/qps/client.h b/test/cpp/qps/client.h index 9d58ea8882..9d7469c9b5 100644 --- a/test/cpp/qps/client.h +++ b/test/cpp/qps/client.h @@ -19,6 +19,8 @@ #ifndef TEST_QPS_CLIENT_H #define TEST_QPS_CLIENT_H +#include <stdlib.h> + #include <condition_variable> #include <mutex> #include <unordered_map> @@ -34,6 +36,7 @@ #include "src/proto/grpc/testing/benchmark_service.grpc.pb.h" #include "src/proto/grpc/testing/payloads.pb.h" +#include "src/core/lib/gpr/env.h" #include "src/cpp/util/core_stats.h" #include "test/cpp/qps/histogram.h" #include "test/cpp/qps/interarrival.h" @@ -441,9 +444,24 @@ class ClientImpl : public Client { std::unique_ptr<std::thread> WaitForReady() { return std::unique_ptr<std::thread>(new std::thread([this]() { if (!is_inproc_) { - GPR_ASSERT(channel_->WaitForConnected( - gpr_time_add(gpr_now(GPR_CLOCK_REALTIME), - gpr_time_from_seconds(10, GPR_TIMESPAN)))); + int connect_deadline = 10; + /* Allow optionally overriding connect_deadline in order + * to deal with benchmark environments in which the server + * can take a long time to become ready. */ + char* channel_connect_timeout_str = + gpr_getenv("QPS_WORKER_CHANNEL_CONNECT_TIMEOUT"); + if (channel_connect_timeout_str != nullptr && + strcmp(channel_connect_timeout_str, "") != 0) { + connect_deadline = atoi(channel_connect_timeout_str); + } + gpr_log(GPR_INFO, + "Waiting for up to %d seconds for the channel %p to connect", + connect_deadline, channel_.get()); + gpr_free(channel_connect_timeout_str); + GPR_ASSERT(channel_->WaitForConnected(gpr_time_add( + gpr_now(GPR_CLOCK_REALTIME), + gpr_time_from_seconds(connect_deadline, GPR_TIMESPAN)))); + gpr_log(GPR_INFO, "Channel %p connected!", channel_.get()); } })); } diff --git a/test/cpp/qps/qps_server_builder.cc b/test/cpp/qps/qps_server_builder.cc new file mode 100644 index 0000000000..5fbc682b75 --- /dev/null +++ b/test/cpp/qps/qps_server_builder.cc @@ -0,0 +1,45 @@ +/* + * + * Copyright 2016 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include "qps_server_builder.h" + +using grpc::ServerBuilder; + +namespace grpc { +namespace testing { + +namespace { +std::unique_ptr<ServerBuilder> DefaultCreateQpsServerBuilder() { + return std::unique_ptr<ServerBuilder>(new ServerBuilder()); +} + +std::function<std::unique_ptr<ServerBuilder>()> g_create_qps_server_builder = + DefaultCreateQpsServerBuilder; +} // namespace + +std::unique_ptr<ServerBuilder> CreateQpsServerBuilder() { + return g_create_qps_server_builder(); +} + +void SetCreateQpsServerBuilderFunc( + std::function<std::unique_ptr<ServerBuilder>()> create_qps_server_builder) { + g_create_qps_server_builder = std::move(create_qps_server_builder); +} + +} // namespace testing +} // namespace grpc diff --git a/test/cpp/qps/qps_server_builder.h b/test/cpp/qps/qps_server_builder.h new file mode 100644 index 0000000000..98f9fa72e9 --- /dev/null +++ b/test/cpp/qps/qps_server_builder.h @@ -0,0 +1,46 @@ +/* + * + * Copyright 2016 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#ifndef GRPC_QPS_SERVER_BUILDER_H +#define GRPC_QPS_SERVER_BUILDER_H + +#include <functional> +#include <memory> + +#include <grpcpp/server_builder.h> + +namespace grpc { +namespace testing { + +// CreateQpsServerBuilder creates a new ServerBuilder. +// This uses the "create ServerBuilder" func that was set +// in SetCreateQpsServerBuilderFunc if one has been set, +// otherwise, this defaults to creating a new ServerBuilder +// with only its default constructor. +std::unique_ptr<ServerBuilder> CreateQpsServerBuilder(); + +// SetCreateQpsServerBuilderFunc sets a function to use to create new +// ServerBuilders in "CreateQpsServerBuilder". It can be used to modify options +// that the server is built with. +void SetCreateQpsServerBuilderFunc( + std::function<std::unique_ptr<ServerBuilder>()>); + +} // namespace testing +} // namespace grpc + +#endif // GRPC_QPS_SERVER_BUILDER_H diff --git a/test/cpp/qps/qps_worker.cc b/test/cpp/qps/qps_worker.cc index d3f0380474..7ddf3c1cf3 100644 --- a/test/cpp/qps/qps_worker.cc +++ b/test/cpp/qps/qps_worker.cc @@ -39,6 +39,7 @@ #include "test/core/util/grpc_profiler.h" #include "test/core/util/histogram.h" #include "test/cpp/qps/client.h" +#include "test/cpp/qps/qps_server_builder.h" #include "test/cpp/qps/server.h" #include "test/cpp/util/create_test_channel.h" #include "test/cpp/util/test_credentials_provider.h" @@ -272,18 +273,18 @@ QpsWorker::QpsWorker(int driver_port, int server_port, impl_.reset(new WorkerServiceImpl(server_port, this)); gpr_atm_rel_store(&done_, static_cast<gpr_atm>(0)); - ServerBuilder builder; + std::unique_ptr<ServerBuilder> builder = CreateQpsServerBuilder(); if (driver_port >= 0) { char* server_address = nullptr; gpr_join_host_port(&server_address, "::", driver_port); - builder.AddListeningPort( + builder->AddListeningPort( server_address, GetCredentialsProvider()->GetServerCredentials(credential_type)); gpr_free(server_address); } - builder.RegisterService(impl_.get()); + builder->RegisterService(impl_.get()); - server_ = builder.BuildAndStart(); + server_ = builder->BuildAndStart(); } QpsWorker::~QpsWorker() {} diff --git a/test/cpp/qps/server_async.cc b/test/cpp/qps/server_async.cc index 1dfef6cfc1..5cd975cf74 100644 --- a/test/cpp/qps/server_async.cc +++ b/test/cpp/qps/server_async.cc @@ -38,6 +38,7 @@ #include "src/core/lib/surface/completion_queue.h" #include "src/proto/grpc/testing/benchmark_service.grpc.pb.h" #include "test/core/util/test_config.h" +#include "test/cpp/qps/qps_server_builder.h" #include "test/cpp/qps/server.h" namespace grpc { @@ -74,19 +75,19 @@ class AsyncQpsServerTest final : public grpc::testing::Server { ResponseType*)> process_rpc) : Server(config) { - ServerBuilder builder; + std::unique_ptr<ServerBuilder> builder = CreateQpsServerBuilder(); auto port_num = port(); // Negative port number means inproc server, so no listen port needed if (port_num >= 0) { char* server_address = nullptr; gpr_join_host_port(&server_address, "::", port_num); - builder.AddListeningPort(server_address, - Server::CreateServerCredentials(config)); + builder->AddListeningPort(server_address, + Server::CreateServerCredentials(config)); gpr_free(server_address); } - register_service(&builder, &async_service_); + register_service(builder.get(), &async_service_); int num_threads = config.async_server_threads(); if (num_threads <= 0) { // dynamic sizing @@ -97,15 +98,15 @@ class AsyncQpsServerTest final : public grpc::testing::Server { int tpc = std::max(1, config.threads_per_cq()); // 1 if unspecified int num_cqs = (num_threads + tpc - 1) / tpc; // ceiling operator for (int i = 0; i < num_cqs; i++) { - srv_cqs_.emplace_back(builder.AddCompletionQueue()); + srv_cqs_.emplace_back(builder->AddCompletionQueue()); } for (int i = 0; i < num_threads; i++) { cq_.emplace_back(i % srv_cqs_.size()); } - ApplyConfigToBuilder(config, &builder); + ApplyConfigToBuilder(config, builder.get()); - server_ = builder.BuildAndStart(); + server_ = builder->BuildAndStart(); auto process_rpc_bound = std::bind(process_rpc, config.payload_config(), std::placeholders::_1, diff --git a/test/cpp/qps/server_sync.cc b/test/cpp/qps/server_sync.cc index b8facf9b56..2e63f5ec86 100644 --- a/test/cpp/qps/server_sync.cc +++ b/test/cpp/qps/server_sync.cc @@ -27,6 +27,7 @@ #include "src/core/lib/gpr/host_port.h" #include "src/proto/grpc/testing/benchmark_service.grpc.pb.h" +#include "test/cpp/qps/qps_server_builder.h" #include "test/cpp/qps/server.h" #include "test/cpp/qps/usage_timer.h" @@ -154,23 +155,23 @@ class BenchmarkServiceImpl final : public BenchmarkService::Service { class SynchronousServer final : public grpc::testing::Server { public: explicit SynchronousServer(const ServerConfig& config) : Server(config) { - ServerBuilder builder; + std::unique_ptr<ServerBuilder> builder = CreateQpsServerBuilder(); auto port_num = port(); // Negative port number means inproc server, so no listen port needed if (port_num >= 0) { char* server_address = nullptr; gpr_join_host_port(&server_address, "::", port_num); - builder.AddListeningPort(server_address, - Server::CreateServerCredentials(config)); + builder->AddListeningPort(server_address, + Server::CreateServerCredentials(config)); gpr_free(server_address); } - ApplyConfigToBuilder(config, &builder); + ApplyConfigToBuilder(config, builder.get()); - builder.RegisterService(&service_); + builder->RegisterService(&service_); - impl_ = builder.BuildAndStart(); + impl_ = builder->BuildAndStart(); } std::shared_ptr<Channel> InProcessChannel( diff --git a/test/cpp/server/load_reporter/load_reporter_test.cc b/test/cpp/server/load_reporter/load_reporter_test.cc index 719c3a67d9..0d56cdf431 100644 --- a/test/cpp/server/load_reporter/load_reporter_test.cc +++ b/test/cpp/server/load_reporter/load_reporter_test.cc @@ -172,9 +172,9 @@ class LbFeedbackTest : public LoadReporterTest { // TODO(juanlishen): The error is big because we use sleep(). It should be // much smaller when we use fake clock. ASSERT_THAT(static_cast<double>(lb_feedback.calls_per_second()), - DoubleNear(expected_qps, expected_qps / 50)); + DoubleNear(expected_qps, expected_qps * 0.05)); ASSERT_THAT(static_cast<double>(lb_feedback.errors_per_second()), - DoubleNear(expected_eps, expected_eps / 50)); + DoubleNear(expected_eps, expected_eps * 0.05)); gpr_log(GPR_INFO, "Verified LB feedback matches the samples of index [%lu, %lu).", start, start + count); diff --git a/third_party/toolchains/BUILD b/third_party/toolchains/BUILD index 29fc4fdaf3..02cd87a7b9 100644 --- a/third_party/toolchains/BUILD +++ b/third_party/toolchains/BUILD @@ -42,7 +42,7 @@ platform( } properties: { name: "gceMachineType" # Small machines for majority of tests. - value: "n1-standard-1" + value: "n1-highmem-2" } properties: { name: "gceMachineType_LARGE" # Large machines for a small set of resource-consuming tests such as combiner_tests under TSAN. diff --git a/tools/distrib/python/grpcio_tools/grpc_version.py b/tools/distrib/python/grpcio_tools/grpc_version.py index ae249e7386..ccb69a8ebc 100644 --- a/tools/distrib/python/grpcio_tools/grpc_version.py +++ b/tools/distrib/python/grpcio_tools/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/tools/distrib/python/grpcio_tools/grpc_version.py.template`!!! -VERSION = '1.14.0.dev0' +VERSION = '1.15.0.dev0' diff --git a/tools/dockerfile/grpc_artifact_android_ndk/Dockerfile b/tools/dockerfile/grpc_artifact_android_ndk/Dockerfile index 77b6acfb16..be96f131f9 100644 --- a/tools/dockerfile/grpc_artifact_android_ndk/Dockerfile +++ b/tools/dockerfile/grpc_artifact_android_ndk/Dockerfile @@ -12,9 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Docker file for building gRPC artifacts. +# Docker file for building gRPC artifacts for Android. -# Recent enough cmake (>=3.9) needed by Android SDK FROM debian:sid RUN apt-get update && apt-get install -y debian-keyring && apt-key update @@ -47,20 +46,26 @@ RUN apt-get update && apt-key update && apt-get install -y \ wget \ zip && apt-get clean -# Cmake for cross-compilation -RUN apt-get update && apt-get install -y cmake golang && apt-get clean +# golang needed to build BoringSSL with cmake +RUN apt-get update && apt-get install -y golang && apt-get clean -################## -# Android NDK +# Java required by Android SDK +RUN apt-get update && apt-get -y install openjdk-8-jdk && apt-get clean -# Download and install Android NDK -RUN wget -q https://dl.google.com/android/repository/android-ndk-r16b-linux-x86_64.zip -O android_ndk.zip \ - && unzip -q android_ndk.zip \ - && rm android_ndk.zip \ - && mv ./android-ndk-r16b /opt -ENV ANDROID_NDK_PATH /opt/android-ndk-r16b +# Install Android SDK +ENV ANDROID_SDK_VERSION 4333796 +RUN mkdir -p /opt/android-sdk && cd /opt/android-sdk && \ + wget -q https://dl.google.com/android/repository/sdk-tools-linux-${ANDROID_SDK_VERSION}.zip && \ + unzip -q sdk-tools-linux-${ANDROID_SDK_VERSION}.zip && \ + rm sdk-tools-linux-${ANDROID_SDK_VERSION}.zip +ENV ANDROID_SDK_PATH /opt/android-sdk -RUN apt-get update && apt-get install -y libpthread-stubs0-dev && apt-get clean +# Install Android NDK and cmake using sdkmanager +RUN mkdir -p ~/.android && touch ~/.android/repositories.cfg +RUN yes | ${ANDROID_SDK_PATH}/tools/bin/sdkmanager --licenses # accept all licenses +RUN ${ANDROID_SDK_PATH}/tools/bin/sdkmanager ndk-bundle 'cmake;3.6.4111459' +ENV ANDROID_NDK_PATH ${ANDROID_SDK_PATH}/ndk-bundle +ENV ANDROID_SDK_CMAKE ${ANDROID_SDK_PATH}/cmake/3.6.4111459/bin/cmake RUN mkdir /var/local/jenkins diff --git a/tools/doxygen/Doxyfile.c++ b/tools/doxygen/Doxyfile.c++ index 322ab5eb98..2f06bda016 100644 --- a/tools/doxygen/Doxyfile.c++ +++ b/tools/doxygen/Doxyfile.c++ @@ -40,7 +40,7 @@ PROJECT_NAME = "GRPC C++" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 1.14.0-dev +PROJECT_NUMBER = 1.15.0-dev # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal index ba322a90a5..a46ebe6197 100644 --- a/tools/doxygen/Doxyfile.c++.internal +++ b/tools/doxygen/Doxyfile.c++.internal @@ -40,7 +40,7 @@ PROJECT_NAME = "GRPC C++" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 1.14.0-dev +PROJECT_NUMBER = 1.15.0-dev # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/tools/internal_ci/linux/grpc_build_artifacts_extra_release.cfg b/tools/internal_ci/linux/grpc_build_artifacts_extra_release.cfg new file mode 100644 index 0000000000..619e3ea3a9 --- /dev/null +++ b/tools/internal_ci/linux/grpc_build_artifacts_extra_release.cfg @@ -0,0 +1,26 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/linux/grpc_build_artifacts_extra.sh" +timeout_mins: 240 +action { + define_artifacts { + regex: "**/*sponge_log.xml" + regex: "github/grpc/reports/**" + regex: "github/grpc/artifacts/**" + } +} diff --git a/tools/internal_ci/linux/grpc_publish_packages.sh b/tools/internal_ci/linux/grpc_publish_packages.sh index 89da36987e..ef943db7e1 100644..100755 --- a/tools/internal_ci/linux/grpc_publish_packages.sh +++ b/tools/internal_ci/linux/grpc_publish_packages.sh @@ -17,94 +17,218 @@ set -ex shopt -s nullglob -export GOOGLE_APPLICATION_CREDENTIALS=${KOKORO_GFILE_DIR}/GrpcTesting-d0eeee2db331.json - -GCS_ROOT=gs://packages.grpc.io -MANIFEST_FILE=index.xml -ARCHIVE_UUID=${KOKORO_BUILD_ID:-$(uuidgen)} -GIT_BRANCH_NAME=master #${KOKORO_GITHUB_COMMIT:-master} -GIT_COMMIT=${KOKORO_GIT_COMMIT:-unknown} -ARCHIVE_TIMESTAMP=$(date -Iseconds) -TARGET_DIR=$(mktemp -d grpc_publish_packages.sh.XXXX) -YEAR_MONTH_PREFIX=$(date "+%Y/%m") -YEAR_PREFIX=${YEAR_MONTH_PREFIX%%/*} -UPLOAD_ROOT=$TARGET_DIR/$YEAR_PREFIX -RELATIVE_PATH=$YEAR_MONTH_PREFIX/$ARCHIVE_UUID -BUILD_ROOT=$TARGET_DIR/$RELATIVE_PATH - -LINUX_PACKAGES=$KOKORO_GFILE_DIR/github/grpc/artifacts -WINDOWS_PACKAGES=$KOKORO_GFILE_DIR/github/grpc/artifacts -# TODO(mmx): enable linux_extra -# LINUX_EXTRA_PACKAGES=$KOKORO_GFILE_DIR/github/grpc/artifacts - -PYTHON_PACKAGES=( - "$LINUX_PACKAGES"/grpcio-[0-9]*.whl - "$LINUX_PACKAGES"/grpcio-[0-9]*.tar.gz - "$LINUX_PACKAGES"/grpcio_tools-[0-9]*.whl - "$LINUX_PACKAGES"/grpcio-tools-[0-9]*.tar.gz - "$LINUX_PACKAGES"/grpcio-health-checking-[0-9]*.tar.gz - "$LINUX_PACKAGES"/grpcio-reflection-[0-9]*.tar.gz - "$LINUX_PACKAGES"/grpcio-testing-[0-9]*.tar.gz - #"$LINUX_EXTRA_PACKAGES"/grpcio-[0-9]*.whl - #"$LINUX_EXTRA_PACKAGES"/grpcio_tools-[0-9]*.whl +cd "$(dirname "$0")/../../.." + +GRPC_VERSION=$(grep -e "^ *version: " build.yaml | head -n 1 | sed 's/.*: //') + +INPUT_ARTIFACTS=$KOKORO_GFILE_DIR/github/grpc/artifacts +INDEX_FILENAME=index.xml + +BUILD_ID=${KOKORO_BUILD_ID:-$(uuidgen)} +BUILD_BRANCH_NAME=master +BUILD_GIT_COMMIT=${KOKORO_GIT_COMMIT:-unknown} +BUILD_TIMESTAMP=$(date -Iseconds) +BUILD_RELPATH=$(date "+%Y/%m")/$BUILD_GIT_COMMIT-$BUILD_ID/ + +GCS_ROOT=gs://packages.grpc.io/ +GCS_ARCHIVE_PREFIX=archive/ +GCS_ARCHIVE_ROOT=$GCS_ROOT$GCS_ARCHIVE_PREFIX +GCS_INDEX=$GCS_ROOT$INDEX_FILENAME + +LOCAL_STAGING_TEMPDIR=$(mktemp -d) +LOCAL_BUILD_ROOT=$LOCAL_STAGING_TEMPDIR/$BUILD_RELPATH +LOCAL_BUILD_INDEX=$LOCAL_BUILD_ROOT$INDEX_FILENAME + +mkdir -p "$LOCAL_BUILD_ROOT" + +find "$INPUT_ARTIFACTS" -type f + +# protoc Plugins +PROTOC_PLUGINS_ZIPPED_PACKAGES=$(mktemp -d) +for zip_dir in protoc_windows_{x86,x64} +do + zip -jr "$PROTOC_PLUGINS_ZIPPED_PACKAGES/grpc-$zip_dir-$GRPC_VERSION.zip" "$INPUT_ARTIFACTS/$zip_dir/"* +done +for tar_dir in protoc_{linux,macos}_{x86,x64} +do + chmod +x "$INPUT_ARTIFACTS/$tar_dir"/* + tar -cvzf "$PROTOC_PLUGINS_ZIPPED_PACKAGES/grpc-$tar_dir-$GRPC_VERSION.tar.gz" -C "$INPUT_ARTIFACTS/$tar_dir" . +done + +PROTOC_PACKAGES=( + "$PROTOC_PLUGINS_ZIPPED_PACKAGES"/grpc-protoc_windows_{x86,x64}-"$GRPC_VERSION.zip" + "$PROTOC_PLUGINS_ZIPPED_PACKAGES"/grpc-protoc_{linux,macos}_{x86,x64}-"$GRPC_VERSION.tar.gz" ) -PHP_PACKAGES=( - "$LINUX_PACKAGES"/grpc-[0-9]*.tgz +# C# +UNZIPPED_CSHARP_PACKAGES=$(mktemp -d) +unzip "$INPUT_ARTIFACTS/csharp_nugets_windows_dotnetcli.zip" -d "$UNZIPPED_CSHARP_PACKAGES" +CSHARP_PACKAGES=( + "$UNZIPPED_CSHARP_PACKAGES"/* ) -RUBY_PACKAGES=( - "$LINUX_PACKAGES"/grpc-[0-9]*.gem - "$LINUX_PACKAGES"/grpc-tools-[0-9]*.gem +# Python +PYTHON_GRPCIO_PACKAGES=( + "$INPUT_ARTIFACTS"/grpcio-[0-9]*.tar.gz + "$INPUT_ARTIFACTS"/grpcio-[0-9]*.whl + "$INPUT_ARTIFACTS"/python_linux_extra_arm*/grpcio-[0-9]*.whl +) +PYTHON_GRPCIO_TOOLS_PACKAGES=( + "$INPUT_ARTIFACTS"/grpcio-tools-[0-9]*.tar.gz + "$INPUT_ARTIFACTS"/grpcio_tools-[0-9]*.whl + "$INPUT_ARTIFACTS"/python_linux_extra_arm*/grpcio_tools-[0-9]*.whl +) +PYTHON_GRPCIO_HEALTH_CHECKING_PACKAGES=( + "$INPUT_ARTIFACTS"/grpcio-health-checking-[0-9]*.tar.gz +) +PYTHON_GRPCIO_REFLECTION_PACKAGES=( + "$INPUT_ARTIFACTS"/grpcio-reflection-[0-9]*.tar.gz +) +PYTHON_GRPCIO_TESTING_PACKAGES=( + "$INPUT_ARTIFACTS"/grpcio-testing-[0-9]*.tar.gz ) -CSHARP_PACKAGES=( - "$WINDOWS_PACKAGES"/csharp_nugets_windows_dotnetcli.zip +# PHP +PHP_PACKAGES=( + "$INPUT_ARTIFACTS"/grpc-[0-9]*.tgz +) + +# Ruby +RUBY_PACKAGES=( + "$INPUT_ARTIFACTS"/grpc-[0-9]*.gem + "$INPUT_ARTIFACTS"/grpc-tools-[0-9]*.gem ) function add_to_manifest() { - local xml_type=$1 - local xml_name - xml_name=$(basename "$2") - local xml_sha256 - xml_sha256=$(openssl sha256 -r "$2" | cut -d " " -f 1) - cp "$2" "$BUILD_ROOT" - echo "<artifact type='$xml_type' name='$xml_name' sha256='$xml_sha256' />" + local artifact_type=$1 + local artifact_file=$2 + local artifact_prefix=$3 + local artifact_name + artifact_name=$(basename "$artifact_file") + local artifact_size + artifact_size=$(stat -c%s "$artifact_file") + local artifact_sha256 + artifact_sha256=$(openssl sha256 -r "$artifact_file" | cut -d " " -f 1) + local artifact_target=$LOCAL_BUILD_ROOT/$artifact_type/$artifact_prefix + mkdir -p "$artifact_target" + cp "$artifact_file" "$artifact_target" + cat <<EOF + <artifact name='$artifact_name' + type='$artifact_type' + path='$artifact_type/$artifact_prefix$artifact_name' + size='$artifact_size' + sha256='$artifact_sha256' /> +EOF } -mkdir -p "$BUILD_ROOT" - { cat <<EOF <?xml version="1.0"?> -<?xml-stylesheet href="/web-assets/build.xsl" type="text/xsl"?> +<?xml-stylesheet href="/web-assets/build-201807.xsl" type="text/xsl"?> +<build id='$BUILD_ID' timestamp='$BUILD_TIMESTAMP' version="201807"> + <metadata> + <project>gRPC</project> + <repository>https://github.com/grpc/grpc</repository> + <branch>$BUILD_BRANCH_NAME</branch> + <commit>$BUILD_GIT_COMMIT</commit> + </metadata> + <artifacts> EOF - echo "<build id='$ARCHIVE_UUID' timestamp='$ARCHIVE_TIMESTAMP'>" - echo "<metadata>" - echo "<branch>$GIT_BRANCH_NAME</branch>" - echo "<commit>$GIT_COMMIT</commit>" - echo "</metadata><artifacts>" - for pkg in "${PYTHON_PACKAGES[@]}"; do add_to_manifest python "$pkg"; done + for pkg in "${PROTOC_PACKAGES[@]}"; do add_to_manifest protoc "$pkg"; done for pkg in "${CSHARP_PACKAGES[@]}"; do add_to_manifest csharp "$pkg"; done for pkg in "${PHP_PACKAGES[@]}"; do add_to_manifest php "$pkg"; done + for pkg in "${PYTHON_GRPCIO_PACKAGES[@]}"; do add_to_manifest python "$pkg" grpcio/; done + for pkg in "${PYTHON_GRPCIO_TOOLS_PACKAGES[@]}"; do add_to_manifest python "$pkg" grpcio-tools/; done + for pkg in "${PYTHON_GRPCIO_HEALTH_CHECKING_PACKAGES[@]}"; do add_to_manifest python "$pkg" grpcio-health-checking/; done + for pkg in "${PYTHON_GRPCIO_REFLECTION_PACKAGES[@]}"; do add_to_manifest python "$pkg" grpcio-reflection/; done + for pkg in "${PYTHON_GRPCIO_TESTING_PACKAGES[@]}"; do add_to_manifest python "$pkg" grpcio-testing/; done for pkg in "${RUBY_PACKAGES[@]}"; do add_to_manifest ruby "$pkg"; done - echo "</artifacts></build>" -}> "$BUILD_ROOT/$MANIFEST_FILE" + cat <<EOF + </artifacts> +</build> +EOF +}> "$LOCAL_BUILD_INDEX" + +LOCAL_BUILD_INDEX_SIZE=$(stat -c%s "$LOCAL_BUILD_INDEX") +LOCAL_BUILD_INDEX_SHA256=$(openssl sha256 -r "$LOCAL_BUILD_INDEX" | cut -d " " -f 1) -BUILD_XML_SHA=$(openssl sha256 -r "$BUILD_ROOT/$MANIFEST_FILE" | cut -d " " -f 1) +OLD_INDEX=$(mktemp) +NEW_INDEX=$(mktemp) -PREV_HOME=$(mktemp old-XXXXX-$MANIFEST_FILE) -NEW_HOME=$(mktemp new-XXXXX-$MANIFEST_FILE) -gsutil cp "$GCS_ROOT/$MANIFEST_FILE" "$PREV_HOME" +# Download the current /index.xml into $OLD_INDEX +gsutil cp "$GCS_INDEX" "$OLD_INDEX" { - head --lines=4 "$PREV_HOME" - echo "<build id='$ARCHIVE_UUID' timestamp='$ARCHIVE_TIMESTAMP' branch='$GIT_BRANCH_NAME' commit='$GIT_COMMIT' manifest='archive/$RELATIVE_PATH/$MANIFEST_FILE' manifest-sha256='$BUILD_XML_SHA' />" - tail --lines=+5 "$PREV_HOME" -}> "$NEW_HOME" + # we want to add an entry as the first child under <builds> tag + # we can get by without a real XML parser by rewriting the header, + # injecting our new tag, and then dumping the rest of the file as is. + cat <<EOF +<?xml version="1.0"?> +<?xml-stylesheet href="/web-assets/home.xsl" type="text/xsl"?> +<packages> + <builds> + <build id='$BUILD_ID' + timestamp='$BUILD_TIMESTAMP' + branch='$BUILD_BRANCH_NAME' + commit='$BUILD_GIT_COMMIT' + path='$GCS_ARCHIVE_PREFIX$BUILD_RELPATH$INDEX_FILENAME' + size='$LOCAL_BUILD_INDEX_SIZE' + sha256='$LOCAL_BUILD_INDEX_SHA256' /> +EOF + tail --lines=+5 "$OLD_INDEX" +}> "$NEW_INDEX" -gsutil -m cp -r "$UPLOAD_ROOT" "$GCS_ROOT/archive" -gsutil -h "Content-Type:application/xml" cp "$NEW_HOME" "$GCS_ROOT/$MANIFEST_FILE" +function generate_directory_index() +{ + local target_dir=$1 + local current_directory_name + current_directory_name=$(basename "$target_dir") + cat <<EOF +<!DOCTYPE html> +<html xmlns="http://www.w3.org/1999/xhtml" lang="" xml:lang=""> + <head> + <meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /> + <title>Index of $current_directory_name - packages.grpc.io</title> + <link rel="stylesheet" type="text/css" href="/web-assets/dirindex.css" /> + </head> + <body> + <h1>Index of <a href="#"><code>$current_directory_name</code></a></h1> + <ul> + <li><a href="#">.</a></li> + <li><a href="..">..</a></li> +EOF + +( + cd "$target_dir" + find * -maxdepth 0 -type d -print | sort | while read -r line + do + echo " <li><a href='$line/'>$line/</a></li>" + done + find * -maxdepth 0 -type f -print | sort | while read -r line + do + echo " <li><a href='$line'>$line</a></li>" + done +) + +cat <<EOF + </ul> + </body> +</html> +EOF +} + +# Upload the current build artifacts +gsutil -m cp -r "$LOCAL_STAGING_TEMPDIR/${BUILD_RELPATH%%/*}" "$GCS_ARCHIVE_ROOT" +# Upload directory indicies for subdirectories +( + cd "$LOCAL_BUILD_ROOT" + find * -type d | while read -r directory + do + generate_directory_index "$directory" | gsutil -h 'Content-Type:text/html' cp - "$GCS_ARCHIVE_ROOT$BUILD_RELPATH$directory/$INDEX_FILENAME" + done +) +# Upload the new /index.xml +gsutil -h "Content-Type:application/xml" cp "$NEW_INDEX" "$GCS_INDEX" diff --git a/tools/package_hosting/404.html b/tools/package_hosting/404.html new file mode 100644 index 0000000000..44d986c4b0 --- /dev/null +++ b/tools/package_hosting/404.html @@ -0,0 +1 @@ +404 Not Found diff --git a/tools/package_hosting/build-201807.xsl b/tools/package_hosting/build-201807.xsl new file mode 100644 index 0000000000..69a190446f --- /dev/null +++ b/tools/package_hosting/build-201807.xsl @@ -0,0 +1,114 @@ +<?xml version="1.0"?> +<xsl:stylesheet version="1.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform"> + +<xsl:template match="//build"> + <html> + <head> + <title>Artifacts for gRPC Build <xsl:value-of select="@id"/> </title> + <link rel="stylesheet" type="text/css" href="/web-assets/style.css" /> + <link rel="apple-touch-icon" href="/web-assets/favicons/apple-touch-icon.png" sizes="180x180" /> + <link rel="icon" type="image/png" href="/web-assets/favicons/android-chrome-192x192.png" sizes="192x192" /> + <link rel="icon" type="image/png" href="/web-assets/favicons/favicon-32x32.png" sizes="32x32" /> + <link rel="icon" type="image/png" href="/web-assets/favicons/favicon-16x16.png" sizes="16x16" /> + <link rel="manifest" href="/web-assets/favicons/manifest.json" /> + <link rel="mask-icon" href="/web-assets/favicons/safari-pinned-tab.svg" color="#2DA6B0" /> + <meta name="msapplication-TileColor" content="#ffffff" /> + <meta name="msapplication-TileImage" content="/web-assets/favicons/mstile-150x150.png" /> + + <meta name="og:title" content="gRPC Package Build"/> + <meta name="og:image" content="https://grpc.io/img/grpc_square_reverse_4x.png"/> + <meta name="og:description" content="gRPC Package Build"/> + </head> + <body bgcolor="#ffffff"> + <div id="topbar"> + <span class="title">Artifacts for gRPC Build <xsl:value-of select="@id"/></span> + </div> + <div id="main"> + <div id="metadata"> + <span class="fieldname">Build: </span> <a href='#'><xsl:value-of select="@id"/></a> + [<a href="https://source.cloud.google.com/results/invocations/{@id}">invocation</a>]<br /> + <span class="fieldname">Timestamp: </span> + <xsl:value-of select="@timestamp"/> <br /> + <span class="fieldname">Branch: </span> + <a href="https://github.com/grpc/grpc/tree/{./metadata/branch[text()]}"> + <xsl:value-of select="./metadata/branch[text()]" /> + </a><br /> + <span class="fieldname">Commit: </span> + <a href="https://github.com/grpc/grpc/tree/{./metadata/commit[text()]}"> + <xsl:value-of select="./metadata/commit[text()]" /><br /></a> + </div> + <xsl:apply-templates select="artifacts" /> + <br /> + <br /> + + <p class="description"><a href="https://grpc.io">gRPC</a> is a <a href="https://www.cncf.io" class="external">Cloud Native Computing Foundation</a> project. <a href="https://policies.google.com/privacy" class="external">Privacy Policy</a>.</p> + <p class="description"> + Copyright © <xsl:value-of select="substring(@timestamp, 1, 4)" /> <a href="https://github.com/grpc/grpc/blob/{./metadata/commit[text()]}/AUTHORS">The gRPC Authors</a></p> + <br /> + <br /> + </div> + </body> + </html> +</xsl:template> + +<xsl:template match="artifacts"> +<h2> gRPC <code>protoc</code> Plugins </h2> +<table> +<xsl:apply-templates select="artifact[@type='protoc']"> + <xsl:sort select="artifact/@name" /> + </xsl:apply-templates> +</table> + +<h2> C# </h2> +<table> +<xsl:apply-templates select="artifact[@type='csharp']"> + <xsl:sort select="artifact/@name" /> + </xsl:apply-templates> +</table> + +<h2> PHP </h2> +<table> +<xsl:apply-templates select="artifact[@type='php']"> + <xsl:sort select="artifact/@name" /> + </xsl:apply-templates> +</table> + +<h2> Python </h2> +<script type="text/javascript"> +// <![CDATA[ +var pythonRepoLink = document.createElement("a"); +pythonRepoLink.href = './python'; +var pythonRepo = pythonRepoLink.href; +document.write("<p><code>" + +"$ pip install --pre --upgrade --force-reinstall --extra-index-url \\<br />" + +" <a href='" + pythonRepo + "'>" + pythonRepo + "</a> \\<br />" + +" grpcio grpcio-{tools,health-checking,reflection,testing}</code></p>"); +// ]]> +</script> +<table> + <xsl:apply-templates select="artifact[@type='python']"> + <xsl:sort select="artifact/@name" /> + </xsl:apply-templates> +</table> + +<h2> Ruby </h2> +<table> +<xsl:apply-templates select="artifact[@type='ruby']"> + <xsl:sort select="artifact/@name" /> + </xsl:apply-templates> +</table> + +</xsl:template> + + +<xsl:template match="artifact"> +<tr> +<td class="name"> <a href="{@path}"><xsl:value-of select="@name" /></a> </td> +<td class="hash"> <xsl:value-of select="@sha256"/> </td> +</tr> +</xsl:template> + +<xsl:template match="metadata"> +</xsl:template> + +</xsl:stylesheet> diff --git a/tools/package_hosting/dirindex.css b/tools/package_hosting/dirindex.css new file mode 100644 index 0000000000..6c4b04ea32 --- /dev/null +++ b/tools/package_hosting/dirindex.css @@ -0,0 +1,16 @@ +ul { + list-style-type: none; +} +a{ + text-decoration: none; +} +a:hover { + text-decoration: underline; +} +ul li a { + font-family: 'SF Mono', 'Menlo', 'Monaco', 'Consolas', 'Courier New', Courier, monospace +} +h1 { + font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Oxygen, Ubuntu, Cantarell, 'Open Sans', 'Helvetica Neue', sans-serif +} + diff --git a/tools/package_hosting/home.xsl b/tools/package_hosting/home.xsl new file mode 100644 index 0000000000..3f79303b1b --- /dev/null +++ b/tools/package_hosting/home.xsl @@ -0,0 +1,86 @@ +<?xml version="1.0"?> +<xsl:stylesheet version="2.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform"> + +<xsl:template match="//packages"> + <html> + <head> + <title>gRPC Packages</title> + <link rel="stylesheet" type="text/css" href="/web-assets/style.css" /> + <link rel="apple-touch-icon" href="/web-assets/favicons/apple-touch-icon.png" sizes="180x180" /> + <link rel="icon" type="image/png" href="/web-assets/favicons/android-chrome-192x192.png" sizes="192x192" /> + <link rel="icon" type="image/png" href="/web-assets/favicons/favicon-32x32.png" sizes="32x32" /> + <link rel="icon" type="image/png" href="/web-assets/favicons/favicon-16x16.png" sizes="16x16" /> + <link rel="manifest" href="/web-assets/favicons/manifest.json" /> + <link rel="mask-icon" href="/web-assets/favicons/safari-pinned-tab.svg" color="#2DA6B0" /> + <meta name="msapplication-TileColor" content="#ffffff" /> + <meta name="msapplication-TileImage" content="/web-assets/favicons/mstile-150x150.png" /> + <meta name="og:title" content="gRPC Packages"/> + <meta name="og:image" content="https://grpc.io/img/grpc_square_reverse_4x.png"/> + <meta name="og:description" content="gRPC Packages"/> + </head> + <body bgcolor="#ffffff"> + <div id="topbar"> + <span class="title">gRPC Packages</span> + </div> + <div id="main"> + <xsl:apply-templates select="releases" /> + <xsl:apply-templates select="builds" /> + <br /> + <br /> + <p class="description"><a href="https://grpc.io">gRPC</a> is a <a href="https://www.cncf.io" class="external">Cloud Native Computing Foundation</a> project. <a href="https://policies.google.com/privacy" class="external">Privacy Policy</a>.</p> + <p class="description">Copyright © 2018 <a href="https://github.com/grpc/grpc/blob/master/AUTHORS">The gRPC Authors</a></p> + </div> + </body> + </html> +</xsl:template> + +<xsl:template match="releases"> + <h2>Official gRPC Releases</h2> + <p>Commits corresponding to <a href="https://github.com/grpc/grpc/releases">official gRPC release points and release candidates</a> are tagged on GitHub.</p> + <p>To maximize usability, gRPC supports the standard way of adding dependencies in your language of choice (if there is one). + In most languages, the gRPC runtime comes in form of a package available in your language's package manager.</p> + <p>For instructions on how to use the language-specific gRPC runtime in your project, please refer to the following:</p> + <ul> + <li><a href="https://github.com/grpc/grpc/blob/master/src/cpp">C++</a>: follow the instructions under the <a href="https://github.com/grpc/grpc/tree/master/src/cpp"><code>src/cpp</code> directory</a></li> + <li><a href="https://github.com/grpc/grpc/blob/master/src/csharp">C#</a>: NuGet package <code>Grpc</code></li> + <li><a href="https://github.com/grpc/grpc-dart">Dart</a>: pub package <code>grpc</code></li> + <li><a href="https://github.com/grpc/grpc-go">Go</a>: <code>go get google.golang.org/grpc</code></li> + <li><a href="https://github.com/grpc/grpc-java">Java</a>: Use JARs from <a href="https://mvnrepository.com/artifact/io.grpc">gRPC Maven Central Repository</a></li> + <li><a href="https://github.com/grpc/grpc-node">Node</a>: <code>npm install grpc</code></li> + <li><a href="https://github.com/grpc/grpc/blob/master/src/objective-c">Objective-C</a>: Add <code>gRPC-ProtoRPC</code> dependency to podspec</li> + <li><a href="https://github.com/grpc/grpc/blob/master/src/php">PHP</a>: <code>pecl install grpc</code></li> + <li><a href="https://github.com/grpc/grpc/blob/master/src/python/grpcio">Python</a>: <code>pip install grpcio</code></li> + <li><a href="https://github.com/grpc/grpc/blob/master/src/ruby">Ruby</a>: <code>gem install grpc</code></li> + <li><a href="https://github.com/grpc/grpc-web">WebJS</a>: follow the <a href="https://github.com/grpc/grpc-web">instructions in <code>grpc-web</code> repository</a></li> + </ul> +</xsl:template> + +<xsl:template match="builds"> + <h2> Daily Builds of <a href="https://github.com/grpc/grpc/tree/master"><code>master</code></a> Branch</h2> + <p>gRPC packages are built on a daily basis at the <code>HEAD</code> of <a href="https://github.com/grpc/grpc/tree/master">the <code>master</code> branch</a> and are archived here.</p> + <p> + <a href="#">The current document</a> (view source) is an XML feed pointing to the packages as they get built and uploaded. + You can subscribe to this feed and fetch, deploy, and test the precompiled packages with your continuous integration infrastructure. + </p> + <p>For stable release packages, please consult the above section and the common package manager for your language.</p> + <table style="border:solid black 1px"> + <tr style="background-color:lightgray"> + <td>Timestamp</td> + <td>Commit</td> + <td>Build ID</td> + </tr> + <xsl:apply-templates select="build[@branch='master']"> + <xsl:sort select="@timestamp" data-type="text" order="descending" /> + </xsl:apply-templates> + </table> +</xsl:template> + +<xsl:template match="build"> + <tr> + <td class="name"><xsl:value-of select="@timestamp" /></td> + <td class="name"> <a href="https://github.com/grpc/grpc/tree/{@commit}"><xsl:value-of select="@commit" /></a></td> + <td class="name"> <a href="{@path}"><xsl:value-of select="@id" /></a></td> + </tr> +</xsl:template> + +</xsl:stylesheet> diff --git a/tools/package_hosting/style.css b/tools/package_hosting/style.css new file mode 100644 index 0000000000..dbd26d50cf --- /dev/null +++ b/tools/package_hosting/style.css @@ -0,0 +1,76 @@ +html, body +{ + margin: 0; + font-family: sans-serif; +} + +a, a:visited, a:link, a:active { + color: #2da6b0; + text-decoration: none; +} + +a:hover { + color: #2da6b0; + text-decoration: underline; +} + +#topbar { + background-color: #2da6b0; + height: 60px; + margin:auto; +} + +#topbar .title { + position: relative; + top: 24px; + left: 24px; + color: white; + font-family: sans-serif; + font-weight: bold; +} + +#main { + max-width:1100px; + margin:auto; +} + +#main h2 { + text-align: left; +} + +#main table { + width:100%; + border-collapse: collapse; + font-size: small; + font-family: 'SF Mono', 'Menlo', 'Monaco', 'Courier New', Courier, monospace; +} +#main table tr td { + border: solid black 1px; + padding: 5px; +} + +#main table tr td.hash { + text-align: right; + border-left: none; + font-size: x-small; +} + +#main table tr td.name { + text-align: left; + border-right: none; +} + +p.description +{ + font-size: smaller; +} + +#metadata { + margin-top: 15px; + padding: 15px; + font-family: 'SF Mono', 'Menlo', 'Monaco', 'Courier New', Courier, monospace; +} + +#metadata span.fieldname { + font-family: sans-serif; +}
\ No newline at end of file diff --git a/tools/package_hosting/upload_web_assets.sh b/tools/package_hosting/upload_web_assets.sh new file mode 100755 index 0000000000..dcf258e38e --- /dev/null +++ b/tools/package_hosting/upload_web_assets.sh @@ -0,0 +1,30 @@ +#!/bin/bash +# Copyright 2018 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +cd "$(dirname "$0")" + +GCS_WEB_ASSETS=gs://packages.grpc.io/web-assets/ + +WEB_ASSETS=( + 404.html + build-201807.xsl + dirindex.css + home.xsl + style.css +) + +gsutil -m cp "${WEB_ASSETS[@]}" "$GCS_WEB_ASSETS" diff --git a/tools/run_tests/artifacts/artifact_targets.py b/tools/run_tests/artifacts/artifact_targets.py index edde3eae28..01323b5326 100644 --- a/tools/run_tests/artifacts/artifact_targets.py +++ b/tools/run_tests/artifacts/artifact_targets.py @@ -212,11 +212,15 @@ class RubyArtifact: class CSharpExtArtifact: """Builds C# native extension library""" - def __init__(self, platform, arch): + def __init__(self, platform, arch, arch_abi=None): self.name = 'csharp_ext_%s_%s' % (platform, arch) self.platform = platform self.arch = arch + self.arch_abi = arch_abi self.labels = ['artifact', 'csharp', platform, arch] + if arch_abi: + self.name += '_%s' % arch_abi + self.labels.append(arch_abi) def pre_build_jobspecs(self): return [] @@ -227,7 +231,14 @@ class CSharpExtArtifact: self.name, 'tools/dockerfile/grpc_artifact_android_ndk', 'tools/run_tests/artifacts/build_artifact_csharp_android.sh', - environ={}) + environ={ + 'ANDROID_ABI': self.arch_abi + }) + elif self.arch == 'ios': + return create_jobspec( + self.name, + ['tools/run_tests/artifacts/build_artifact_csharp_ios.sh'], + use_workspace=True) elif self.platform == 'windows': cmake_arch_option = 'Win32' if self.arch == 'x86' else self.arch return create_jobspec( @@ -348,7 +359,10 @@ def targets(): for Cls in (CSharpExtArtifact, ProtocArtifact) for platform in ('linux', 'macos', 'windows') for arch in ('x86', 'x64') ] + [ - CSharpExtArtifact('linux', 'android'), + CSharpExtArtifact('linux', 'android', arch_abi='arm64-v8a'), + CSharpExtArtifact('linux', 'android', arch_abi='armeabi-v7a'), + CSharpExtArtifact('linux', 'android', arch_abi='x86'), + CSharpExtArtifact('macos', 'ios'), PythonArtifact('linux', 'x86', 'cp27-cp27m'), PythonArtifact('linux', 'x86', 'cp27-cp27mu'), PythonArtifact('linux', 'x86', 'cp34-cp34m'), diff --git a/tools/run_tests/artifacts/build_artifact_csharp_android.sh b/tools/run_tests/artifacts/build_artifact_csharp_android.sh index ba598e76a4..067eb30edc 100755 --- a/tools/run_tests/artifacts/build_artifact_csharp_android.sh +++ b/tools/run_tests/artifacts/build_artifact_csharp_android.sh @@ -17,6 +17,7 @@ set -ex cd "$(dirname "$0")/../../.." +# ANDROID_ABI is set by the job definition in artifact_targets.py src/csharp/experimental/build_native_ext_for_android.sh mkdir -p "${ARTIFACTS_OUT}" diff --git a/tools/run_tests/artifacts/build_artifact_csharp_ios.sh b/tools/run_tests/artifacts/build_artifact_csharp_ios.sh new file mode 100755 index 0000000000..c902a45db7 --- /dev/null +++ b/tools/run_tests/artifacts/build_artifact_csharp_ios.sh @@ -0,0 +1,23 @@ +#!/bin/bash +# Copyright 2018 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +cd "$(dirname "$0")/../../.." + +src/csharp/experimental/build_native_ext_for_ios.sh + +mkdir -p "${ARTIFACTS_OUT}" +cp libs/ios/libgrpc_csharp_ext.a libs/ios/libgrpc.a "${ARTIFACTS_OUT}" diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json index bf3ddb42c9..072402b2cf 100644 --- a/tools/run_tests/generated/sources_and_headers.json +++ b/tools/run_tests/generated/sources_and_headers.json @@ -7761,6 +7761,7 @@ "test/cpp/qps/histogram.h", "test/cpp/qps/interarrival.h", "test/cpp/qps/parse_json.h", + "test/cpp/qps/qps_server_builder.h", "test/cpp/qps/qps_worker.h", "test/cpp/qps/report.h", "test/cpp/qps/server.h", @@ -7782,6 +7783,8 @@ "test/cpp/qps/interarrival.h", "test/cpp/qps/parse_json.cc", "test/cpp/qps/parse_json.h", + "test/cpp/qps/qps_server_builder.cc", + "test/cpp/qps/qps_server_builder.h", "test/cpp/qps/qps_worker.cc", "test/cpp/qps/qps_worker.h", "test/cpp/qps/report.cc", diff --git a/tools/run_tests/performance/README.md b/tools/run_tests/performance/README.md index 2fc1a27c9b..791270ab38 100644 --- a/tools/run_tests/performance/README.md +++ b/tools/run_tests/performance/README.md @@ -104,3 +104,31 @@ Example memory profile of grpc-go server, with `go tools pprof`: ``` $ go tool pprof --text --alloc_space http://localhost:<pprof_port>/debug/heap ``` + +### Configuration environment variables: + +* QPS_WORKER_CHANNEL_CONNECT_TIMEOUT + + Consuming process: qps_worker + + Type: integer (number of seconds) + + This can be used to configure the amount of time that benchmark + clients wait for channels to the benchmark server to become ready. + This is useful in certain benchmark environments in which the + server can take a long time to become ready. Note: if setting + this to a high value, then the scenario config under test should + probably also have a large "warmup_seconds". + +* QPS_WORKERS + + Consuming process: qps_json_driver + + Type: comma separated list of host:port + + Set this to a comma separated list of QPS worker processes/machines. + Each scenario in a scenario config has specifies a certain number + of servers, `num_servers`, and the driver will start + "benchmark servers"'s on the first `num_server` `host:port` pairs in + the comma separated list. The rest will be told to run as clients + against the benchmark server. |