diff options
77 files changed, 479 insertions, 467 deletions
@@ -86,7 +86,7 @@ CC_asan-trace-cmp = clang CXX_asan-trace-cmp = clang++ LD_asan-trace-cmp = clang++ LDXX_asan-trace-cmp = clang++ -CPPFLAGS_asan-trace-cmp = -O0 -fsanitize-coverage=edge -fsanitize-coverage=trace-cmp -fsanitize=address -fno-omit-frame-pointer -Wno-unused-command-line-argument -DGPR_NO_DIRECT_SYSCALLS +CPPFLAGS_asan-trace-cmp = -O0 -fsanitize-coverage=edge,trace-pc-guard -fsanitize-coverage=trace-cmp -fsanitize=address -fno-omit-frame-pointer -Wno-unused-command-line-argument -DGPR_NO_DIRECT_SYSCALLS LDFLAGS_asan-trace-cmp = -fsanitize=address VALID_CONFIG_dbg = 1 @@ -103,7 +103,7 @@ CC_asan = clang CXX_asan = clang++ LD_asan = clang++ LDXX_asan = clang++ -CPPFLAGS_asan = -O0 -fsanitize-coverage=edge -fsanitize=address -fno-omit-frame-pointer -Wno-unused-command-line-argument -DGPR_NO_DIRECT_SYSCALLS +CPPFLAGS_asan = -O0 -fsanitize-coverage=edge,trace-pc-guard -fsanitize=address -fno-omit-frame-pointer -Wno-unused-command-line-argument -DGPR_NO_DIRECT_SYSCALLS LDFLAGS_asan = -fsanitize=address VALID_CONFIG_msan = 1 @@ -112,7 +112,7 @@ CC_msan = clang CXX_msan = clang++ LD_msan = clang++ LDXX_msan = clang++ -CPPFLAGS_msan = -O0 -fsanitize-coverage=edge -fsanitize=memory -fsanitize-memory-track-origins -fsanitize-memory-use-after-dtor -fno-omit-frame-pointer -DGTEST_HAS_TR1_TUPLE=0 -DGTEST_USE_OWN_TR1_TUPLE=1 -Wno-unused-command-line-argument -fPIE -pie -DGPR_NO_DIRECT_SYSCALLS +CPPFLAGS_msan = -O0 -fsanitize-coverage=edge,trace-pc-guard -fsanitize=memory -fsanitize-memory-track-origins -fsanitize-memory-use-after-dtor -fno-omit-frame-pointer -DGTEST_HAS_TR1_TUPLE=0 -DGTEST_USE_OWN_TR1_TUPLE=1 -Wno-unused-command-line-argument -fPIE -pie -DGPR_NO_DIRECT_SYSCALLS LDFLAGS_msan = -fsanitize=memory -DGTEST_HAS_TR1_TUPLE=0 -DGTEST_USE_OWN_TR1_TUPLE=1 -fPIE -pie $(if $(JENKINS_BUILD),-Wl$(comma)-Ttext-segment=0x7e0000000000,) DEFINES_msan = NDEBUG @@ -139,7 +139,7 @@ CC_asan-noleaks = clang CXX_asan-noleaks = clang++ LD_asan-noleaks = clang++ LDXX_asan-noleaks = clang++ -CPPFLAGS_asan-noleaks = -O0 -fsanitize-coverage=edge -fsanitize=address -fno-omit-frame-pointer -Wno-unused-command-line-argument -DGPR_NO_DIRECT_SYSCALLS +CPPFLAGS_asan-noleaks = -O0 -fsanitize-coverage=edge,trace-pc-guard -fsanitize=address -fno-omit-frame-pointer -Wno-unused-command-line-argument -DGPR_NO_DIRECT_SYSCALLS LDFLAGS_asan-noleaks = -fsanitize=address VALID_CONFIG_noexcept = 1 @@ -157,7 +157,7 @@ CC_ubsan = clang CXX_ubsan = clang++ LD_ubsan = clang++ LDXX_ubsan = clang++ -CPPFLAGS_ubsan = -O0 -fsanitize-coverage=edge -fsanitize=undefined -fno-omit-frame-pointer -Wno-unused-command-line-argument -Wvarargs +CPPFLAGS_ubsan = -O0 -fsanitize-coverage=edge,trace-pc-guard -fsanitize=undefined -fno-omit-frame-pointer -Wno-unused-command-line-argument -Wvarargs LDFLAGS_ubsan = -fsanitize=undefined,unsigned-integer-overflow DEFINES_ubsan = NDEBUG GRPC_UBSAN diff --git a/build.yaml b/build.yaml index 6b3955f6a5..16c90fbd4c 100644 --- a/build.yaml +++ b/build.yaml @@ -5470,7 +5470,7 @@ vspackages: configs: asan: CC: clang - CPPFLAGS: -O0 -fsanitize-coverage=edge -fsanitize=address -fno-omit-frame-pointer + CPPFLAGS: -O0 -fsanitize-coverage=edge,trace-pc-guard -fsanitize=address -fno-omit-frame-pointer -Wno-unused-command-line-argument -DGPR_NO_DIRECT_SYSCALLS CXX: clang++ LD: clang++ @@ -5482,7 +5482,7 @@ configs: LSAN_OPTIONS: suppressions=tools/lsan_suppressions.txt:report_objects=1 asan-noleaks: CC: clang - CPPFLAGS: -O0 -fsanitize-coverage=edge -fsanitize=address -fno-omit-frame-pointer + CPPFLAGS: -O0 -fsanitize-coverage=edge,trace-pc-guard -fsanitize=address -fno-omit-frame-pointer -Wno-unused-command-line-argument -DGPR_NO_DIRECT_SYSCALLS CXX: clang++ LD: clang++ @@ -5493,8 +5493,9 @@ configs: ASAN_OPTIONS: detect_leaks=0:color=always asan-trace-cmp: CC: clang - CPPFLAGS: -O0 -fsanitize-coverage=edge -fsanitize-coverage=trace-cmp -fsanitize=address - -fno-omit-frame-pointer -Wno-unused-command-line-argument -DGPR_NO_DIRECT_SYSCALLS + CPPFLAGS: -O0 -fsanitize-coverage=edge,trace-pc-guard -fsanitize-coverage=trace-cmp + -fsanitize=address -fno-omit-frame-pointer -Wno-unused-command-line-argument + -DGPR_NO_DIRECT_SYSCALLS CXX: clang++ LD: clang++ LDFLAGS: -fsanitize=address @@ -5539,7 +5540,7 @@ configs: valgrind: --tool=memcheck --leak-check=full msan: CC: clang - CPPFLAGS: -O0 -fsanitize-coverage=edge -fsanitize=memory -fsanitize-memory-track-origins + CPPFLAGS: -O0 -fsanitize-coverage=edge,trace-pc-guard -fsanitize=memory -fsanitize-memory-track-origins -fsanitize-memory-use-after-dtor -fno-omit-frame-pointer -DGTEST_HAS_TR1_TUPLE=0 -DGTEST_USE_OWN_TR1_TUPLE=1 -Wno-unused-command-line-argument -fPIE -pie -DGPR_NO_DIRECT_SYSCALLS CXX: clang++ @@ -5579,7 +5580,7 @@ configs: TSAN_OPTIONS: suppressions=tools/tsan_suppressions.txt:halt_on_error=1:second_deadlock_stack=1 ubsan: CC: clang - CPPFLAGS: -O0 -fsanitize-coverage=edge -fsanitize=undefined -fno-omit-frame-pointer + CPPFLAGS: -O0 -fsanitize-coverage=edge,trace-pc-guard -fsanitize=undefined -fno-omit-frame-pointer -Wno-unused-command-line-argument -Wvarargs CXX: clang++ DEFINES: NDEBUG GRPC_UBSAN diff --git a/src/core/ext/filters/client_channel/client_channel.cc b/src/core/ext/filters/client_channel/client_channel.cc index fa0c280f80..bc6f733e15 100644 --- a/src/core/ext/filters/client_channel/client_channel.cc +++ b/src/core/ext/filters/client_channel/client_channel.cc @@ -1421,7 +1421,7 @@ static void do_retry(grpc_call_element* elem, } if (grpc_client_channel_trace.enabled()) { gpr_log(GPR_INFO, - "chand=%p calld=%p: retrying failed call in %" PRIuPTR " ms", chand, + "chand=%p calld=%p: retrying failed call in %" PRId64 " ms", chand, calld, next_attempt_time - grpc_core::ExecCtx::Get()->Now()); } // Schedule retry after computed delay. diff --git a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc index 0d57fa2d02..263b51ae89 100644 --- a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc +++ b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc @@ -776,7 +776,7 @@ void GrpcLb::BalancerCallState::OnBalancerMessageReceivedLocked( if (grpc_lb_glb_trace.enabled()) { gpr_log(GPR_INFO, "[grpclb %p] Received initial LB response message; " - "client load reporting interval = %" PRIdPTR " milliseconds", + "client load reporting interval = %" PRId64 " milliseconds", grpclb_policy, lb_calld->client_stats_report_interval_); } } else if (grpc_lb_glb_trace.enabled()) { @@ -1417,7 +1417,7 @@ void GrpcLb::StartBalancerCallRetryTimerLocked() { gpr_log(GPR_INFO, "[grpclb %p] Connection to LB server lost...", this); grpc_millis timeout = next_try - ExecCtx::Get()->Now(); if (timeout > 0) { - gpr_log(GPR_INFO, "[grpclb %p] ... retry_timer_active in %" PRIuPTR "ms.", + gpr_log(GPR_INFO, "[grpclb %p] ... retry_timer_active in %" PRId64 "ms.", this, timeout); } else { gpr_log(GPR_INFO, "[grpclb %p] ... retry_timer_active immediately.", diff --git a/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc b/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc index 76df976698..ff2140e628 100644 --- a/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc +++ b/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc @@ -490,7 +490,7 @@ void PickFirst::PickFirstSubchannelData::ProcessConnectivityChangeLocked( if (grpc_lb_pick_first_trace.enabled()) { gpr_log(GPR_INFO, "Servicing pending pick with selected subchannel %p", - p->selected_); + p->selected_->subchannel()); } GRPC_CLOSURE_SCHED(pick->on_complete, GRPC_ERROR_NONE); } diff --git a/src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc b/src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc index c3c62b60bf..3c40ae14b8 100644 --- a/src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc +++ b/src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc @@ -346,7 +346,7 @@ void AresDnsResolver::OnResolvedLocked(void* arg, grpc_error* error) { RefCountedPtr<Resolver> self = r->Ref(DEBUG_LOCATION, "retry-timer"); self.release(); if (timeout > 0) { - gpr_log(GPR_DEBUG, "retrying in %" PRIdPTR " milliseconds", timeout); + gpr_log(GPR_DEBUG, "retrying in %" PRId64 " milliseconds", timeout); } else { gpr_log(GPR_DEBUG, "retrying immediately"); } @@ -381,8 +381,8 @@ void AresDnsResolver::MaybeStartResolvingLocked() { const grpc_millis last_resolution_ago = grpc_core::ExecCtx::Get()->Now() - last_resolution_timestamp_; gpr_log(GPR_DEBUG, - "In cooldown from last resolution (from %" PRIdPTR - " ms ago). Will resolve again in %" PRIdPTR " ms", + "In cooldown from last resolution (from %" PRId64 + " ms ago). Will resolve again in %" PRId64 " ms", last_resolution_ago, ms_until_next_resolution); have_next_resolution_timer_ = true; // TODO(roth): We currently deal with this ref manually. Once the diff --git a/src/core/ext/filters/client_channel/resolver/dns/native/dns_resolver.cc b/src/core/ext/filters/client_channel/resolver/dns/native/dns_resolver.cc index 920a492fdb..fae4c33a17 100644 --- a/src/core/ext/filters/client_channel/resolver/dns/native/dns_resolver.cc +++ b/src/core/ext/filters/client_channel/resolver/dns/native/dns_resolver.cc @@ -218,7 +218,7 @@ void NativeDnsResolver::OnResolvedLocked(void* arg, grpc_error* error) { r->Ref(DEBUG_LOCATION, "next_resolution_timer"); self.release(); if (timeout > 0) { - gpr_log(GPR_DEBUG, "retrying in %" PRIdPTR " milliseconds", timeout); + gpr_log(GPR_DEBUG, "retrying in %" PRId64 " milliseconds", timeout); } else { gpr_log(GPR_DEBUG, "retrying immediately"); } @@ -254,8 +254,8 @@ void NativeDnsResolver::MaybeStartResolvingLocked() { const grpc_millis last_resolution_ago = grpc_core::ExecCtx::Get()->Now() - last_resolution_timestamp_; gpr_log(GPR_DEBUG, - "In cooldown from last resolution (from %" PRIdPTR - " ms ago). Will resolve again in %" PRIdPTR " ms", + "In cooldown from last resolution (from %" PRId64 + " ms ago). Will resolve again in %" PRId64 " ms", last_resolution_ago, ms_until_next_resolution); have_next_resolution_timer_ = true; // TODO(roth): We currently deal with this ref manually. Once the diff --git a/src/core/ext/filters/client_channel/subchannel.cc b/src/core/ext/filters/client_channel/subchannel.cc index ad6b6dd192..f010002ab9 100644 --- a/src/core/ext/filters/client_channel/subchannel.cc +++ b/src/core/ext/filters/client_channel/subchannel.cc @@ -467,7 +467,7 @@ static void maybe_start_connecting_locked(grpc_subchannel* c) { if (time_til_next <= 0) { gpr_log(GPR_INFO, "Subchannel %p: Retry immediately", c); } else { - gpr_log(GPR_INFO, "Subchannel %p: Retry in %" PRIdPTR " milliseconds", c, + gpr_log(GPR_INFO, "Subchannel %p: Retry in %" PRId64 " milliseconds", c, time_til_next); } GRPC_CLOSURE_INIT(&c->on_alarm, on_alarm, c, grpc_schedule_on_exec_ctx); diff --git a/src/core/lib/debug/trace.h b/src/core/lib/debug/trace.h index 28157c6383..fe6301a3fc 100644 --- a/src/core/lib/debug/trace.h +++ b/src/core/lib/debug/trace.h @@ -57,14 +57,14 @@ class TraceFlag { const char* name() const { return name_; } -// This following define may be commented out to ensure that the compiler -// deletes any "if (tracer.enabled()) {...}" codeblocks. This is useful to -// test the performance impact tracers have on the system. -// -// #define COMPILE_OUT_ALL_TRACERS_IN_OPT_BUILD -#ifdef COMPILE_OUT_ALL_TRACERS_IN_OPT_BUILD - bool enabled() { return false; } -#else +// Use the symbol GRPC_USE_TRACERS to determine if tracers will be enabled in +// opt builds (tracers are always on in dbg builds). The default in OSS is for +// tracers to be on since we support binary distributions of gRPC for the +// wrapped language (wr don't want to force recompilation to get tracing). +// Internally, however, for performance reasons, we compile them out by +// default, since internal build systems make recompiling trivial. +#define GRPC_USE_TRACERS // tracers on by default in OSS +#if defined(GRPC_USE_TRACERS) || !defined(NDEBUG) bool enabled() { #ifdef GRPC_THREADSAFE_TRACER return gpr_atm_no_barrier_load(&value_) != 0; @@ -72,7 +72,9 @@ class TraceFlag { return value_; #endif // GRPC_THREADSAFE_TRACER } -#endif // COMPILE_OUT_ALL_TRACERS_IN_OPT_BUILD +#else + bool enabled() { return false; } +#endif /* defined(GRPC_USE_TRACERS) || !defined(NDEBUG) */ private: friend void grpc_core::testing::grpc_tracer_enable_flag(TraceFlag* flag); diff --git a/src/core/lib/iomgr/ev_epollex_linux.cc b/src/core/lib/iomgr/ev_epollex_linux.cc index 4c6cff7fe2..12f23ea1d6 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.cc +++ b/src/core/lib/iomgr/ev_epollex_linux.cc @@ -1066,7 +1066,7 @@ static grpc_error* pollset_work(grpc_pollset* pollset, #endif if (grpc_polling_trace.enabled()) { gpr_log(GPR_INFO, - "PS:%p work hdl=%p worker=%p now=%" PRIdPTR " deadline=%" PRIdPTR + "PS:%p work hdl=%p worker=%p now=%" PRId64 " deadline=%" PRId64 " kwp=%d pollable=%p", pollset, worker_hdl, WORKER_PTR, grpc_core::ExecCtx::Get()->Now(), deadline, pollset->kicked_without_poller, pollset->active_pollable); diff --git a/src/core/lib/iomgr/ev_posix.cc b/src/core/lib/iomgr/ev_posix.cc index 6bd1dc8e50..902dca934a 100644 --- a/src/core/lib/iomgr/ev_posix.cc +++ b/src/core/lib/iomgr/ev_posix.cc @@ -251,10 +251,10 @@ static void pollset_destroy(grpc_pollset* pollset) { static grpc_error* pollset_work(grpc_pollset* pollset, grpc_pollset_worker** worker, grpc_millis deadline) { - GRPC_POLLING_API_TRACE("pollset_work(%p, %" PRIdPTR ") begin", pollset, + GRPC_POLLING_API_TRACE("pollset_work(%p, %" PRId64 ") begin", pollset, deadline); grpc_error* err = g_event_engine->pollset_work(pollset, worker, deadline); - GRPC_POLLING_API_TRACE("pollset_work(%p, %" PRIdPTR ") end", pollset, + GRPC_POLLING_API_TRACE("pollset_work(%p, %" PRId64 ") end", pollset, deadline); return err; } diff --git a/src/core/lib/iomgr/exec_ctx.cc b/src/core/lib/iomgr/exec_ctx.cc index 2f544b20ab..5d5c355ff9 100644 --- a/src/core/lib/iomgr/exec_ctx.cc +++ b/src/core/lib/iomgr/exec_ctx.cc @@ -53,24 +53,24 @@ static void exec_ctx_sched(grpc_closure* closure, grpc_error* error) { static gpr_timespec g_start_time; -static gpr_atm timespec_to_atm_round_down(gpr_timespec ts) { +static grpc_millis timespec_to_millis_round_down(gpr_timespec ts) { ts = gpr_time_sub(ts, g_start_time); double x = GPR_MS_PER_SEC * static_cast<double>(ts.tv_sec) + static_cast<double>(ts.tv_nsec) / GPR_NS_PER_MS; if (x < 0) return 0; - if (x > GPR_ATM_MAX) return GPR_ATM_MAX; - return static_cast<gpr_atm>(x); + if (x > GRPC_MILLIS_INF_FUTURE) return GRPC_MILLIS_INF_FUTURE; + return static_cast<grpc_millis>(x); } -static gpr_atm timespec_to_atm_round_up(gpr_timespec ts) { +static grpc_millis timespec_to_millis_round_up(gpr_timespec ts) { ts = gpr_time_sub(ts, g_start_time); double x = GPR_MS_PER_SEC * static_cast<double>(ts.tv_sec) + static_cast<double>(ts.tv_nsec) / GPR_NS_PER_MS + static_cast<double>(GPR_NS_PER_SEC - 1) / static_cast<double>(GPR_NS_PER_SEC); if (x < 0) return 0; - if (x > GPR_ATM_MAX) return GPR_ATM_MAX; - return static_cast<gpr_atm>(x); + if (x > GRPC_MILLIS_INF_FUTURE) return GRPC_MILLIS_INF_FUTURE; + return static_cast<grpc_millis>(x); } gpr_timespec grpc_millis_to_timespec(grpc_millis millis, @@ -92,12 +92,12 @@ gpr_timespec grpc_millis_to_timespec(grpc_millis millis, } grpc_millis grpc_timespec_to_millis_round_down(gpr_timespec ts) { - return timespec_to_atm_round_down( + return timespec_to_millis_round_down( gpr_convert_clock_type(ts, g_start_time.clock_type)); } grpc_millis grpc_timespec_to_millis_round_up(gpr_timespec ts) { - return timespec_to_atm_round_up( + return timespec_to_millis_round_up( gpr_convert_clock_type(ts, g_start_time.clock_type)); } @@ -138,7 +138,7 @@ bool ExecCtx::Flush() { grpc_millis ExecCtx::Now() { if (!now_is_valid_) { - now_ = timespec_to_atm_round_down(gpr_now(GPR_CLOCK_MONOTONIC)); + now_ = timespec_to_millis_round_down(gpr_now(GPR_CLOCK_MONOTONIC)); now_is_valid_ = true; } return now_; diff --git a/src/core/lib/iomgr/exec_ctx.h b/src/core/lib/iomgr/exec_ctx.h index 8823dc4b51..cf1118a003 100644 --- a/src/core/lib/iomgr/exec_ctx.h +++ b/src/core/lib/iomgr/exec_ctx.h @@ -29,10 +29,10 @@ #include "src/core/lib/gprpp/fork.h" #include "src/core/lib/iomgr/closure.h" -typedef gpr_atm grpc_millis; +typedef int64_t grpc_millis; -#define GRPC_MILLIS_INF_FUTURE GPR_ATM_MAX -#define GRPC_MILLIS_INF_PAST GPR_ATM_MIN +#define GRPC_MILLIS_INF_FUTURE INT64_MAX +#define GRPC_MILLIS_INF_PAST INT64_MIN /** A workqueue represents a list of work to be executed asynchronously. Forward declared here to avoid a circular dependency with workqueue.h. */ diff --git a/src/core/lib/iomgr/pollset_custom.cc b/src/core/lib/iomgr/pollset_custom.cc index 04bd104055..70e8a4596f 100644 --- a/src/core/lib/iomgr/pollset_custom.cc +++ b/src/core/lib/iomgr/pollset_custom.cc @@ -69,7 +69,7 @@ static grpc_error* pollset_work(grpc_pollset* pollset, GRPC_CUSTOM_IOMGR_ASSERT_SAME_THREAD(); gpr_mu_unlock(&pollset->mu); grpc_millis now = grpc_core::ExecCtx::Get()->Now(); - size_t timeout = 0; + grpc_millis timeout = 0; if (deadline > now) { timeout = deadline - now; } @@ -77,7 +77,7 @@ static grpc_error* pollset_work(grpc_pollset* pollset, // control back to the application grpc_core::ExecCtx* curr = grpc_core::ExecCtx::Get(); grpc_core::ExecCtx::Set(nullptr); - poller_vtable->poll(timeout); + poller_vtable->poll(static_cast<size_t>(timeout)); grpc_core::ExecCtx::Set(curr); grpc_core::ExecCtx::Get()->InvalidateNow(); if (grpc_core::ExecCtx::Get()->HasWork()) { diff --git a/src/core/lib/iomgr/timer.h b/src/core/lib/iomgr/timer.h index 5ff10d3aee..7f534476df 100644 --- a/src/core/lib/iomgr/timer.h +++ b/src/core/lib/iomgr/timer.h @@ -28,7 +28,7 @@ #include "src/core/lib/iomgr/iomgr.h" typedef struct grpc_timer { - gpr_atm deadline; + grpc_millis deadline; uint32_t heap_index; /* INVALID_HEAP_INDEX if not in heap */ bool pending; struct grpc_timer* next; diff --git a/src/core/lib/iomgr/timer_generic.cc b/src/core/lib/iomgr/timer_generic.cc index de2256f7cb..4294162af7 100644 --- a/src/core/lib/iomgr/timer_generic.cc +++ b/src/core/lib/iomgr/timer_generic.cc @@ -34,6 +34,7 @@ #include "src/core/lib/gpr/spinlock.h" #include "src/core/lib/gpr/tls.h" #include "src/core/lib/gpr/useful.h" +#include "src/core/lib/iomgr/exec_ctx.h" #include "src/core/lib/iomgr/time_averaged_stats.h" #include "src/core/lib/iomgr/timer_heap.h" @@ -59,9 +60,9 @@ typedef struct { gpr_mu mu; grpc_time_averaged_stats stats; /* All and only timers with deadlines <= this will be in the heap. */ - gpr_atm queue_deadline_cap; + grpc_millis queue_deadline_cap; /* The deadline of the next timer due in this shard */ - gpr_atm min_deadline; + grpc_millis min_deadline; /* Index of this timer_shard in the g_shard_queue */ uint32_t shard_queue_index; /* This holds all timers with deadlines < queue_deadline_cap. Timers in this @@ -209,15 +210,23 @@ static void validate_non_pending_timer(grpc_timer* t) { #endif +#if GPR_ARCH_64 +/* NOTE: TODO(sreek) - Currently the thread local storage support in grpc is + for intptr_t which means on 32-bit machines it is not wide enough to hold + grpc_millis which is 64-bit. Adding thread local support for 64 bit values + is a lot of work for very little gain. So we are currently restricting this + optimization to only 64 bit machines */ + /* Thread local variable that stores the deadline of the next timer the thread * has last-seen. This is an optimization to prevent the thread from checking * shared_mutables.min_timer (which requires acquiring shared_mutables.mu lock, * an expensive operation) */ GPR_TLS_DECL(g_last_seen_min_timer); +#endif struct shared_mutables { /* The deadline of the next timer due across all timer shards */ - gpr_atm min_timer; + grpc_millis min_timer; /* Allow only one run_some_expired_timers at once */ gpr_spinlock checker_mu; bool initialized; @@ -227,18 +236,18 @@ struct shared_mutables { static struct shared_mutables g_shared_mutables; -static gpr_atm saturating_add(gpr_atm a, gpr_atm b) { - if (a > GPR_ATM_MAX - b) { - return GPR_ATM_MAX; +static grpc_millis saturating_add(grpc_millis a, grpc_millis b) { + if (a > GRPC_MILLIS_INF_FUTURE - b) { + return GRPC_MILLIS_INF_FUTURE; } return a + b; } -static grpc_timer_check_result run_some_expired_timers(gpr_atm now, - gpr_atm* next, +static grpc_timer_check_result run_some_expired_timers(grpc_millis now, + grpc_millis* next, grpc_error* error); -static gpr_atm compute_min_deadline(timer_shard* shard) { +static grpc_millis compute_min_deadline(timer_shard* shard) { return grpc_timer_heap_is_empty(&shard->heap) ? saturating_add(shard->queue_deadline_cap, 1) : grpc_timer_heap_top(&shard->heap)->deadline; @@ -257,8 +266,11 @@ static void timer_list_init() { g_shared_mutables.checker_mu = GPR_SPINLOCK_INITIALIZER; gpr_mu_init(&g_shared_mutables.mu); g_shared_mutables.min_timer = grpc_core::ExecCtx::Get()->Now(); + +#if GPR_ARCH_64 gpr_tls_init(&g_last_seen_min_timer); gpr_tls_set(&g_last_seen_min_timer, 0); +#endif for (i = 0; i < g_num_shards; i++) { timer_shard* shard = &g_shards[i]; @@ -287,7 +299,11 @@ static void timer_list_shutdown() { grpc_timer_heap_destroy(&shard->heap); } gpr_mu_destroy(&g_shared_mutables.mu); + +#if GPR_ARCH_64 gpr_tls_destroy(&g_last_seen_min_timer); +#endif + gpr_free(g_shards); gpr_free(g_shard_queue); g_shared_mutables.initialized = false; @@ -346,7 +362,7 @@ static void timer_init(grpc_timer* timer, grpc_millis deadline, #endif if (grpc_timer_trace.enabled()) { - gpr_log(GPR_INFO, "TIMER %p: SET %" PRIdPTR " now %" PRIdPTR " call %p[%p]", + gpr_log(GPR_INFO, "TIMER %p: SET %" PRId64 " now %" PRId64 " call %p[%p]", timer, deadline, grpc_core::ExecCtx::Get()->Now(), closure, closure->cb); } @@ -383,7 +399,7 @@ static void timer_init(grpc_timer* timer, grpc_millis deadline, } if (grpc_timer_trace.enabled()) { gpr_log(GPR_INFO, - " .. add to shard %d with queue_deadline_cap=%" PRIdPTR + " .. add to shard %d with queue_deadline_cap=%" PRId64 " => is_first_timer=%s", static_cast<int>(shard - g_shards), shard->queue_deadline_cap, is_first_timer ? "true" : "false"); @@ -404,15 +420,27 @@ static void timer_init(grpc_timer* timer, grpc_millis deadline, if (is_first_timer) { gpr_mu_lock(&g_shared_mutables.mu); if (grpc_timer_trace.enabled()) { - gpr_log(GPR_INFO, " .. old shard min_deadline=%" PRIdPTR, + gpr_log(GPR_INFO, " .. old shard min_deadline=%" PRId64, shard->min_deadline); } if (deadline < shard->min_deadline) { - gpr_atm old_min_deadline = g_shard_queue[0]->min_deadline; + grpc_millis old_min_deadline = g_shard_queue[0]->min_deadline; shard->min_deadline = deadline; note_deadline_change(shard); if (shard->shard_queue_index == 0 && deadline < old_min_deadline) { - gpr_atm_no_barrier_store(&g_shared_mutables.min_timer, deadline); +#if GPR_ARCH_64 + // TODO: sreek - Using c-style cast here. static_cast<> gives an error + // (on mac platforms complaining that gpr_atm* is (long *) while + // (&g_shared_mutables.min_timer) is a (long long *). The cast should be + // safe since we know that both are pointer types and 64-bit wide. + gpr_atm_no_barrier_store((gpr_atm*)(&g_shared_mutables.min_timer), + deadline); +#else + // On 32-bit systems, gpr_atm_no_barrier_store does not work on 64-bit + // types (like grpc_millis). So all reads and writes to + // g_shared_mutables.min_timer varialbe under g_shared_mutables.mu + g_shared_mutables.min_timer = deadline; +#endif grpc_kick_poller(); } } @@ -421,8 +449,10 @@ static void timer_init(grpc_timer* timer, grpc_millis deadline, } static void timer_consume_kick(void) { - /* force re-evaluation of last seeen min */ +#if GPR_ARCH_64 + /* Force re-evaluation of last seen min */ gpr_tls_set(&g_last_seen_min_timer, 0); +#endif } static void timer_cancel(grpc_timer* timer) { @@ -459,7 +489,7 @@ static void timer_cancel(grpc_timer* timer) { 'queue_deadline_cap') into into shard->heap. Returns 'true' if shard->heap has atleast ONE element REQUIRES: shard->mu locked */ -static int refill_heap(timer_shard* shard, gpr_atm now) { +static int refill_heap(timer_shard* shard, grpc_millis now) { /* Compute the new queue window width and bound by the limits: */ double computed_deadline_delta = grpc_time_averaged_stats_update_average(&shard->stats) * @@ -472,10 +502,10 @@ static int refill_heap(timer_shard* shard, gpr_atm now) { /* Compute the new cap and put all timers under it into the queue: */ shard->queue_deadline_cap = saturating_add(GPR_MAX(now, shard->queue_deadline_cap), - static_cast<gpr_atm>(deadline_delta * 1000.0)); + static_cast<grpc_millis>(deadline_delta * 1000.0)); if (grpc_timer_check_trace.enabled()) { - gpr_log(GPR_INFO, " .. shard[%d]->queue_deadline_cap --> %" PRIdPTR, + gpr_log(GPR_INFO, " .. shard[%d]->queue_deadline_cap --> %" PRId64, static_cast<int>(shard - g_shards), shard->queue_deadline_cap); } for (timer = shard->list.next; timer != &shard->list; timer = next) { @@ -483,7 +513,7 @@ static int refill_heap(timer_shard* shard, gpr_atm now) { if (timer->deadline < shard->queue_deadline_cap) { if (grpc_timer_check_trace.enabled()) { - gpr_log(GPR_INFO, " .. add timer with deadline %" PRIdPTR " to heap", + gpr_log(GPR_INFO, " .. add timer with deadline %" PRId64 " to heap", timer->deadline); } list_remove(timer); @@ -496,7 +526,7 @@ static int refill_heap(timer_shard* shard, gpr_atm now) { /* This pops the next non-cancelled timer with deadline <= now from the queue, or returns NULL if there isn't one. REQUIRES: shard->mu locked */ -static grpc_timer* pop_one(timer_shard* shard, gpr_atm now) { +static grpc_timer* pop_one(timer_shard* shard, grpc_millis now) { grpc_timer* timer; for (;;) { if (grpc_timer_check_trace.enabled()) { @@ -511,12 +541,12 @@ static grpc_timer* pop_one(timer_shard* shard, gpr_atm now) { timer = grpc_timer_heap_top(&shard->heap); if (grpc_timer_check_trace.enabled()) { gpr_log(GPR_INFO, - " .. check top timer deadline=%" PRIdPTR " now=%" PRIdPTR, + " .. check top timer deadline=%" PRId64 " now=%" PRId64, timer->deadline, now); } if (timer->deadline > now) return nullptr; if (grpc_timer_trace.enabled()) { - gpr_log(GPR_INFO, "TIMER %p: FIRE %" PRIdPTR "ms late via %s scheduler", + gpr_log(GPR_INFO, "TIMER %p: FIRE %" PRId64 "ms late via %s scheduler", timer, now - timer->deadline, timer->closure->scheduler->vtable->name); } @@ -527,8 +557,8 @@ static grpc_timer* pop_one(timer_shard* shard, gpr_atm now) { } /* REQUIRES: shard->mu unlocked */ -static size_t pop_timers(timer_shard* shard, gpr_atm now, - gpr_atm* new_min_deadline, grpc_error* error) { +static size_t pop_timers(timer_shard* shard, grpc_millis now, + grpc_millis* new_min_deadline, grpc_error* error) { size_t n = 0; grpc_timer* timer; gpr_mu_lock(&shard->mu); @@ -546,13 +576,27 @@ static size_t pop_timers(timer_shard* shard, gpr_atm now, return n; } -static grpc_timer_check_result run_some_expired_timers(gpr_atm now, - gpr_atm* next, +static grpc_timer_check_result run_some_expired_timers(grpc_millis now, + grpc_millis* next, grpc_error* error) { grpc_timer_check_result result = GRPC_TIMERS_NOT_CHECKED; - gpr_atm min_timer = gpr_atm_no_barrier_load(&g_shared_mutables.min_timer); +#if GPR_ARCH_64 + // TODO: sreek - Using c-style cast here. static_cast<> gives an error (on + // mac platforms complaining that gpr_atm* is (long *) while + // (&g_shared_mutables.min_timer) is a (long long *). The cast should be + // safe since we know that both are pointer types and 64-bit wide + grpc_millis min_timer = static_cast<grpc_millis>( + gpr_atm_no_barrier_load((gpr_atm*)(&g_shared_mutables.min_timer))); gpr_tls_set(&g_last_seen_min_timer, min_timer); +#else + // On 32-bit systems, gpr_atm_no_barrier_load does not work on 64-bit types + // (like grpc_millis). So all reads and writes to g_shared_mutables.min_timer + // are done under g_shared_mutables.mu + gpr_mu_lock(&g_shared_mutables.mu); + grpc_millis min_timer = g_shared_mutables.min_timer; + gpr_mu_unlock(&g_shared_mutables.mu); +#endif if (now < min_timer) { if (next != nullptr) *next = GPR_MIN(*next, min_timer); return GRPC_TIMERS_CHECKED_AND_EMPTY; @@ -563,14 +607,15 @@ static grpc_timer_check_result run_some_expired_timers(gpr_atm now, result = GRPC_TIMERS_CHECKED_AND_EMPTY; if (grpc_timer_check_trace.enabled()) { - gpr_log(GPR_INFO, " .. shard[%d]->min_deadline = %" PRIdPTR, + gpr_log(GPR_INFO, " .. shard[%d]->min_deadline = %" PRId64, static_cast<int>(g_shard_queue[0] - g_shards), g_shard_queue[0]->min_deadline); } while (g_shard_queue[0]->min_deadline < now || - (now != GPR_ATM_MAX && g_shard_queue[0]->min_deadline == now)) { - gpr_atm new_min_deadline; + (now != GRPC_MILLIS_INF_FUTURE && + g_shard_queue[0]->min_deadline == now)) { + grpc_millis new_min_deadline; /* For efficiency, we pop as many available timers as we can from the shard. This may violate perfect timer deadline ordering, but that @@ -582,8 +627,8 @@ static grpc_timer_check_result run_some_expired_timers(gpr_atm now, if (grpc_timer_check_trace.enabled()) { gpr_log(GPR_INFO, " .. result --> %d" - ", shard[%d]->min_deadline %" PRIdPTR " --> %" PRIdPTR - ", now=%" PRIdPTR, + ", shard[%d]->min_deadline %" PRId64 " --> %" PRId64 + ", now=%" PRId64, result, static_cast<int>(g_shard_queue[0] - g_shards), g_shard_queue[0]->min_deadline, new_min_deadline, now); } @@ -601,8 +646,19 @@ static grpc_timer_check_result run_some_expired_timers(gpr_atm now, *next = GPR_MIN(*next, g_shard_queue[0]->min_deadline); } - gpr_atm_no_barrier_store(&g_shared_mutables.min_timer, +#if GPR_ARCH_64 + // TODO: sreek - Using c-style cast here. static_cast<> gives an error (on + // mac platforms complaining that gpr_atm* is (long *) while + // (&g_shared_mutables.min_timer) is a (long long *). The cast should be + // safe since we know that both are pointer types and 64-bit wide + gpr_atm_no_barrier_store((gpr_atm*)(&g_shared_mutables.min_timer), g_shard_queue[0]->min_deadline); +#else + // On 32-bit systems, gpr_atm_no_barrier_store does not work on 64-bit + // types (like grpc_millis). So all reads and writes to + // g_shared_mutables.min_timer are done under g_shared_mutables.mu + g_shared_mutables.min_timer = g_shard_queue[0]->min_deadline; +#endif gpr_mu_unlock(&g_shared_mutables.mu); gpr_spinlock_unlock(&g_shared_mutables.checker_mu); } @@ -616,17 +672,28 @@ static grpc_timer_check_result timer_check(grpc_millis* next) { // prelude grpc_millis now = grpc_core::ExecCtx::Get()->Now(); +#if GPR_ARCH_64 /* fetch from a thread-local first: this avoids contention on a globally mutable cacheline in the common case */ grpc_millis min_timer = gpr_tls_get(&g_last_seen_min_timer); +#else + // On 32-bit systems, we currently do not have thread local support for 64-bit + // types. In this case, directly read from g_shared_mutables.min_timer. + // Also, note that on 32-bit systems, gpr_atm_no_barrier_store does not work + // on 64-bit types (like grpc_millis). So all reads and writes to + // g_shared_mutables.min_timer are done under g_shared_mutables.mu + gpr_mu_lock(&g_shared_mutables.mu); + grpc_millis min_timer = g_shared_mutables.min_timer; + gpr_mu_unlock(&g_shared_mutables.mu); +#endif + if (now < min_timer) { if (next != nullptr) { *next = GPR_MIN(*next, min_timer); } if (grpc_timer_check_trace.enabled()) { - gpr_log(GPR_INFO, - "TIMER CHECK SKIP: now=%" PRIdPTR " min_timer=%" PRIdPTR, now, - min_timer); + gpr_log(GPR_INFO, "TIMER CHECK SKIP: now=%" PRId64 " min_timer=%" PRId64, + now, min_timer); } return GRPC_TIMERS_CHECKED_AND_EMPTY; } @@ -642,13 +709,18 @@ static grpc_timer_check_result timer_check(grpc_millis* next) { if (next == nullptr) { next_str = gpr_strdup("NULL"); } else { - gpr_asprintf(&next_str, "%" PRIdPTR, *next); + gpr_asprintf(&next_str, "%" PRId64, *next); } +#if GPR_ARCH_64 gpr_log(GPR_INFO, - "TIMER CHECK BEGIN: now=%" PRIdPTR " next=%s tls_min=%" PRIdPTR + "TIMER CHECK BEGIN: now=%" PRId64 " next=%s tls_min=%" PRId64 " glob_min=%" PRIdPTR, - now, next_str, gpr_tls_get(&g_last_seen_min_timer), - gpr_atm_no_barrier_load(&g_shared_mutables.min_timer)); + now, next_str, min_timer, + gpr_atm_no_barrier_load((gpr_atm*)(&g_shared_mutables.min_timer))); +#else + gpr_log(GPR_INFO, "TIMER CHECK BEGIN: now=%" PRId64 " next=%s min=%" PRId64, + now, next_str, min_timer); +#endif gpr_free(next_str); } // actual code @@ -660,7 +732,7 @@ static grpc_timer_check_result timer_check(grpc_millis* next) { if (next == nullptr) { next_str = gpr_strdup("NULL"); } else { - gpr_asprintf(&next_str, "%" PRIdPTR, *next); + gpr_asprintf(&next_str, "%" PRId64, *next); } gpr_log(GPR_INFO, "TIMER CHECK END: r=%d; next=%s", r, next_str); gpr_free(next_str); diff --git a/src/core/lib/iomgr/timer_manager.cc b/src/core/lib/iomgr/timer_manager.cc index 35e7914568..9fdae17909 100644 --- a/src/core/lib/iomgr/timer_manager.cc +++ b/src/core/lib/iomgr/timer_manager.cc @@ -172,7 +172,7 @@ static bool wait_until(grpc_millis next) { if (grpc_timer_check_trace.enabled()) { grpc_millis wait_time = next - grpc_core::ExecCtx::Get()->Now(); - gpr_log(GPR_INFO, "sleep for a %" PRIdPTR " milliseconds", wait_time); + gpr_log(GPR_INFO, "sleep for a %" PRId64 " milliseconds", wait_time); } } else { // g_timed_waiter == true && next >= g_timed_waiter_deadline next = GRPC_MILLIS_INF_FUTURE; diff --git a/src/core/lib/surface/call.cc b/src/core/lib/surface/call.cc index 86e0afa6ee..8946a7bedf 100644 --- a/src/core/lib/surface/call.cc +++ b/src/core/lib/surface/call.cc @@ -67,9 +67,6 @@ #define MAX_SEND_EXTRA_METADATA_COUNT 3 -// Used to create arena for the first call. -#define ESTIMATED_MDELEM_COUNT 16 - /* Status data for a request can come from several sources; this enumerates them all, and acts as a priority sorting for which status to return to the application - earlier entries override @@ -326,11 +323,6 @@ static parent_call* get_parent_call(grpc_call* call) { return (parent_call*)gpr_atm_acq_load(&call->parent_call_atm); } -size_t grpc_call_get_initial_size_estimate() { - return sizeof(grpc_call) + sizeof(batch_control) * MAX_CONCURRENT_BATCHES + - sizeof(grpc_linked_mdelem) * ESTIMATED_MDELEM_COUNT; -} - grpc_error* grpc_call_create(const grpc_call_create_args* args, grpc_call** out_call) { GPR_TIMER_SCOPE("grpc_call_create", 0); diff --git a/src/core/lib/surface/call.h b/src/core/lib/surface/call.h index e000f13e7d..793cce4efa 100644 --- a/src/core/lib/surface/call.h +++ b/src/core/lib/surface/call.h @@ -98,11 +98,6 @@ void* grpc_call_context_get(grpc_call* call, grpc_context_index elem); uint8_t grpc_call_is_client(grpc_call* call); -/* Get the estimated memory size for a call BESIDES the call stack. Combined - * with the size of the call stack, it helps estimate the arena size for the - * initial call. */ -size_t grpc_call_get_initial_size_estimate(); - /* Return an appropriate compression algorithm for the requested compression \a * level in the context of \a call. */ grpc_compression_algorithm grpc_call_compression_for_level( diff --git a/src/core/lib/surface/channel.cc b/src/core/lib/surface/channel.cc index a466b325be..0062d0d4d5 100644 --- a/src/core/lib/surface/channel.cc +++ b/src/core/lib/surface/channel.cc @@ -108,8 +108,7 @@ grpc_channel* grpc_channel_create_with_builder( gpr_atm_no_barrier_store( &channel->call_size_estimate, - (gpr_atm)CHANNEL_STACK_FROM_CHANNEL(channel)->call_stack_size + - grpc_call_get_initial_size_estimate()); + (gpr_atm)CHANNEL_STACK_FROM_CHANNEL(channel)->call_stack_size); grpc_compression_options_init(&channel->compression_options); for (size_t i = 0; i < args->num_args; i++) { diff --git a/src/core/lib/transport/transport_op_string.cc b/src/core/lib/transport/transport_op_string.cc index 99af7c1931..25ab492f3a 100644 --- a/src/core/lib/transport/transport_op_string.cc +++ b/src/core/lib/transport/transport_op_string.cc @@ -52,7 +52,7 @@ static void put_metadata_list(gpr_strvec* b, grpc_metadata_batch md) { } if (md.deadline != GRPC_MILLIS_INF_FUTURE) { char* tmp; - gpr_asprintf(&tmp, " deadline=%" PRIdPTR, md.deadline); + gpr_asprintf(&tmp, " deadline=%" PRId64, md.deadline); gpr_strvec_add(b, tmp); } } diff --git a/src/objective-c/GRPCClient/GRPCCall.m b/src/objective-c/GRPCClient/GRPCCall.m index 75dda9df85..5b48d06158 100644 --- a/src/objective-c/GRPCClient/GRPCCall.m +++ b/src/objective-c/GRPCClient/GRPCCall.m @@ -268,8 +268,10 @@ static NSString *const kBearerPrefix = @"Bearer "; // method. // TODO(jcanizales): Rename to readResponseIfNotPaused. - (void)startNextRead { - if (self.state == GRXWriterStatePaused) { - return; + @synchronized(self) { + if (self.state == GRXWriterStatePaused) { + return; + } } dispatch_async(_callQueue, ^{ diff --git a/templates/tools/dockerfile/bazel.include b/templates/tools/dockerfile/bazel.include new file mode 100644 index 0000000000..2c0f3825fc --- /dev/null +++ b/templates/tools/dockerfile/bazel.include @@ -0,0 +1,5 @@ +#======================== +# Bazel installation +RUN echo "deb [arch=amd64] http://storage.googleapis.com/bazel-apt stable jdk1.8" > /etc/apt/sources.list.d/bazel.list +RUN curl https://bazel.build/bazel-release.pub.gpg | apt-key add - +RUN apt-get -y update && apt-get -y install bazel=0.13.1 && apt-get clean diff --git a/templates/tools/dockerfile/ccache_setup.include b/templates/tools/dockerfile/ccache_setup.include deleted file mode 100644 index 2a2de7fdab..0000000000 --- a/templates/tools/dockerfile/ccache_setup.include +++ /dev/null @@ -1,7 +0,0 @@ -# Prepare ccache -RUN ln -s /usr/bin/ccache /usr/local/bin/gcc -RUN ln -s /usr/bin/ccache /usr/local/bin/g++ -RUN ln -s /usr/bin/ccache /usr/local/bin/cc -RUN ln -s /usr/bin/ccache /usr/local/bin/c++ -RUN ln -s /usr/bin/ccache /usr/local/bin/clang -RUN ln -s /usr/bin/ccache /usr/local/bin/clang++ diff --git a/templates/tools/dockerfile/node_deps.include b/templates/tools/dockerfile/node_deps.include index bee3087b3f..581b6efd1a 100644 --- a/templates/tools/dockerfile/node_deps.include +++ b/templates/tools/dockerfile/node_deps.include @@ -5,9 +5,10 @@ RUN touch .profile RUN curl -o- https://raw.githubusercontent.com/creationix/nvm/v0.25.4/install.sh | bash # Install all versions of node that we want to test -RUN /bin/bash -l -c "nvm install 4 && npm config set cache /tmp/npm-cache && npm install -g npm" -RUN /bin/bash -l -c "nvm install 5 && npm config set cache /tmp/npm-cache && npm install -g npm" -RUN /bin/bash -l -c "nvm install 6 && npm config set cache /tmp/npm-cache && npm install -g npm" -RUN /bin/bash -l -c "nvm install 8 && npm config set cache /tmp/npm-cache && npm install -g npm" -RUN /bin/bash -l -c "nvm install 9 && npm config set cache /tmp/npm-cache && npm install -g npm" -RUN /bin/bash -l -c "nvm alias default 9"
\ No newline at end of file +RUN /bin/bash -l -c "nvm install 4 && npm config set cache /tmp/npm-cache" +RUN /bin/bash -l -c "nvm install 5 && npm config set cache /tmp/npm-cache" +RUN /bin/bash -l -c "nvm install 6 && npm config set cache /tmp/npm-cache" +RUN /bin/bash -l -c "nvm install 8 && npm config set cache /tmp/npm-cache" +RUN /bin/bash -l -c "nvm install 9 && npm config set cache /tmp/npm-cache" +RUN /bin/bash -l -c "nvm install 10 && npm config set cache /tmp/npm-cache" +RUN /bin/bash -l -c "nvm alias default 10"
\ No newline at end of file diff --git a/templates/tools/dockerfile/run_tests_addons.include b/templates/tools/dockerfile/run_tests_addons.include index 3f0a1899c5..74b01e386c 100644 --- a/templates/tools/dockerfile/run_tests_addons.include +++ b/templates/tools/dockerfile/run_tests_addons.include @@ -1,2 +1,2 @@ -<%include file="ccache_setup.include"/> -<%include file="run_tests_addons_nocache.include"/>
\ No newline at end of file + +RUN mkdir /var/local/jenkins diff --git a/templates/tools/dockerfile/run_tests_addons_nocache.include b/templates/tools/dockerfile/run_tests_addons_nocache.include deleted file mode 100644 index 74b01e386c..0000000000 --- a/templates/tools/dockerfile/run_tests_addons_nocache.include +++ /dev/null @@ -1,2 +0,0 @@ - -RUN mkdir /var/local/jenkins diff --git a/templates/tools/dockerfile/test/bazel/Dockerfile.template b/templates/tools/dockerfile/test/bazel/Dockerfile.template new file mode 100644 index 0000000000..8ef2f02e71 --- /dev/null +++ b/templates/tools/dockerfile/test/bazel/Dockerfile.template @@ -0,0 +1,36 @@ +%YAML 1.2 +--- | + # Copyright 2015 gRPC authors. + # + # Licensed under the Apache License, Version 2.0 (the "License"); + # you may not use this file except in compliance with the License. + # You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + + FROM gcr.io/oss-fuzz-base/base-builder + + # Install basic packages and Bazel dependencies. + RUN apt-get update && apt-get install -y software-properties-common python-software-properties + RUN add-apt-repository ppa:webupd8team/java + RUN apt-get update && apt-get -y install ${'\\'} + autoconf ${'\\'} + build-essential ${'\\'} + curl ${'\\'} + libtool ${'\\'} + make ${'\\'} + openjdk-8-jdk ${'\\'} + vim + + <%include file="../../bazel.include"/> + + RUN mkdir -p /var/local/jenkins + + # Define the default command. + CMD ["bash"] diff --git a/templates/tools/dockerfile/test/cxx_jessie_x64/Dockerfile.template b/templates/tools/dockerfile/test/cxx_jessie_x64/Dockerfile.template index 9d559dfafa..01e22cce4c 100644 --- a/templates/tools/dockerfile/test/cxx_jessie_x64/Dockerfile.template +++ b/templates/tools/dockerfile/test/cxx_jessie_x64/Dockerfile.template @@ -21,7 +21,6 @@ <%include file="../../python_deps.include"/> <%include file="../../cxx_deps.include"/> <%include file="../../cmake_jessie_backports.include"/> - <%include file="../../clang_update.include"/> <%include file="../../run_tests_addons.include"/> <%include file="../../libuv_install.include"/> diff --git a/templates/tools/dockerfile/test/cxx_jessie_x86/Dockerfile.template b/templates/tools/dockerfile/test/cxx_jessie_x86/Dockerfile.template index 8011448b69..b9cd1e9828 100644 --- a/templates/tools/dockerfile/test/cxx_jessie_x86/Dockerfile.template +++ b/templates/tools/dockerfile/test/cxx_jessie_x86/Dockerfile.template @@ -21,6 +21,7 @@ <%include file="../../python_deps.include"/> <%include file="../../cxx_deps.include"/> <%include file="../../run_tests_addons.include"/> + <%include file="../../cmake_jessie_backports.include"/> # Install gcc-4.8 and other relevant items RUN apt-get update && apt-get -y install gcc-4.8 gcc-4.8-multilib g++-4.8 g++-4.8-multilib && apt-get clean diff --git a/templates/tools/dockerfile/test/cxx_sanitizers_jessie_x64/Dockerfile.template b/templates/tools/dockerfile/test/cxx_sanitizers_jessie_x64/Dockerfile.template new file mode 100644 index 0000000000..6070330e91 --- /dev/null +++ b/templates/tools/dockerfile/test/cxx_sanitizers_jessie_x64/Dockerfile.template @@ -0,0 +1,31 @@ +%YAML 1.2 +--- | + # Copyright 2018 gRPC authors. + # + # Licensed under the Apache License, Version 2.0 (the "License"); + # you may not use this file except in compliance with the License. + # You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + + # This is the base Docker image we use for running tests on RBE + FROM gcr.io/cloud-marketplace/google/rbe-debian8@sha256:1ede2a929b44d629ec5abe86eee6d7ffea1d5a4d247489a8867d46cfde3e38bd + + <%include file="../../apt_get_basic.include"/> + <%include file="../../gcp_api_libraries.include"/> + <%include file="../../python_deps.include"/> + #================= + # C++ dependencies (purposely excluding Clang because it's part of the base image) + RUN apt-get update && apt-get -y install libgflags-dev libgtest-dev libc++-dev && apt-get clean + + # Link llvm-symbolizer to where our test scripts expect to find it + RUN ln -s /usr/local/bin/llvm-symbolizer /usr/bin/llvm-symbolizer + + # Define the default command. + CMD ["bash"] diff --git a/templates/tools/dockerfile/test/cxx_ubuntu1404_x64/Dockerfile.template b/templates/tools/dockerfile/test/cxx_ubuntu1404_x64/Dockerfile.template index 56d8eff06e..3668c97bcd 100644 --- a/templates/tools/dockerfile/test/cxx_ubuntu1404_x64/Dockerfile.template +++ b/templates/tools/dockerfile/test/cxx_ubuntu1404_x64/Dockerfile.template @@ -20,6 +20,6 @@ <%include file="../../gcp_api_libraries.include"/> <%include file="../../python_deps.include"/> <%include file="../../cxx_deps.include"/> - <%include file="../../run_tests_addons_nocache.include"/> + <%include file="../../run_tests_addons.include"/> # Define the default command. CMD ["bash"] diff --git a/templates/tools/dockerfile/test/sanity/Dockerfile.template b/templates/tools/dockerfile/test/sanity/Dockerfile.template index 69bb7c4671..eac7f2ab01 100644 --- a/templates/tools/dockerfile/test/sanity/Dockerfile.template +++ b/templates/tools/dockerfile/test/sanity/Dockerfile.template @@ -33,26 +33,6 @@ shellcheck RUN pip install simplejson mako - #====================================== - # More sanity test dependencies (bazel) - RUN echo "deb http://http.debian.net/debian jessie-backports main" >> /etc/apt/sources.list - RUN apt-get update - RUN apt-get install -y -t jessie-backports openjdk-8-jdk - - #======================== - # Bazel installation - RUN echo "deb [arch=amd64] http://storage.googleapis.com/bazel-apt stable jdk1.8" > /etc/apt/sources.list.d/bazel.list - RUN curl https://bazel.build/bazel-release.pub.gpg | apt-key add - - RUN apt-get -y update - RUN apt-get -y install bazel - - # Pin Bazel to 0.9.0 - # Installing Bazel via apt-get first is required before installing 0.9.0 to - # allow gRPC to build without errors. See https://github.com/grpc/grpc/issues/10553 - RUN curl -fSsL -O https://github.com/bazelbuild/bazel/releases/download/0.9.0/bazel-0.9.0-installer-linux-x86_64.sh - RUN chmod +x ./bazel-0.9.0-installer-linux-x86_64.sh - RUN ./bazel-0.9.0-installer-linux-x86_64.sh - <%include file="../../clang5.include"/> <%include file="../../run_tests_addons.include"/> diff --git a/test/core/client_channel/resolvers/dns_resolver_cooldown_test.cc b/test/core/client_channel/resolvers/dns_resolver_cooldown_test.cc index 37e1d714ca..521fc3107e 100644 --- a/test/core/client_channel/resolvers/dns_resolver_cooldown_test.cc +++ b/test/core/client_channel/resolvers/dns_resolver_cooldown_test.cc @@ -128,7 +128,7 @@ static void poll_pollset_until_request_done(iomgr_args* args) { break; } grpc_millis time_left = deadline - grpc_core::ExecCtx::Get()->Now(); - gpr_log(GPR_DEBUG, "done=%d, time_left=%" PRIdPTR, done, time_left); + gpr_log(GPR_DEBUG, "done=%d, time_left=%" PRId64, done, time_left); GPR_ASSERT(time_left >= 0); grpc_pollset_worker* worker = nullptr; gpr_mu_lock(args->mu); diff --git a/test/core/end2end/fuzzers/server_fuzzer.cc b/test/core/end2end/fuzzers/server_fuzzer.cc index 5eb83ede7a..248c34cbc1 100644 --- a/test/core/end2end/fuzzers/server_fuzzer.cc +++ b/test/core/end2end/fuzzers/server_fuzzer.cc @@ -103,15 +103,33 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { grpc_metadata_array_destroy(&request_metadata1); grpc_server_shutdown_and_notify(server, cq, tag(0xdead)); grpc_server_cancel_all_calls(server); + grpc_millis deadline = grpc_core::ExecCtx::Get()->Now() + 5000; for (int i = 0; i <= requested_calls; i++) { - ev = grpc_completion_queue_next(cq, gpr_inf_past(GPR_CLOCK_REALTIME), - nullptr); + // A single grpc_completion_queue_next might not be sufficient for getting + // the tag from shutdown, because we might potentially get blocked by + // an operation happening on the timer thread. + // For example, the deadline timer might expire, leading to the timer + // thread trying to cancel the RPC and thereby acquiring a few references + // to the call. This will prevent the shutdown to complete till the timer + // thread releases those references. + // As a solution, we are going to keep performing a cq_next for a + // liberal period of 5 seconds for the timer thread to complete its work. + do { + ev = grpc_completion_queue_next(cq, gpr_inf_past(GPR_CLOCK_REALTIME), + nullptr); + grpc_core::ExecCtx::Get()->InvalidateNow(); + } while (ev.type != GRPC_OP_COMPLETE && + grpc_core::ExecCtx::Get()->Now() < deadline); GPR_ASSERT(ev.type == GRPC_OP_COMPLETE); } grpc_completion_queue_shutdown(cq); for (int i = 0; i <= requested_calls; i++) { - ev = grpc_completion_queue_next(cq, gpr_inf_past(GPR_CLOCK_REALTIME), - nullptr); + do { + ev = grpc_completion_queue_next(cq, gpr_inf_past(GPR_CLOCK_REALTIME), + nullptr); + grpc_core::ExecCtx::Get()->InvalidateNow(); + } while (ev.type != GRPC_QUEUE_SHUTDOWN && + grpc_core::ExecCtx::Get()->Now() < deadline); GPR_ASSERT(ev.type == GRPC_QUEUE_SHUTDOWN); } grpc_server_destroy(server); diff --git a/test/core/iomgr/resolve_address_posix_test.cc b/test/core/iomgr/resolve_address_posix_test.cc index 79b2b50e70..e495e4c877 100644 --- a/test/core/iomgr/resolve_address_posix_test.cc +++ b/test/core/iomgr/resolve_address_posix_test.cc @@ -91,7 +91,7 @@ static void actually_poll(void* argsp) { break; } grpc_millis time_left = deadline - grpc_core::ExecCtx::Get()->Now(); - gpr_log(GPR_DEBUG, "done=%d, time_left=%" PRIdPTR, done, time_left); + gpr_log(GPR_DEBUG, "done=%d, time_left=%" PRId64, done, time_left); GPR_ASSERT(time_left >= 0); grpc_pollset_worker* worker = nullptr; gpr_mu_lock(args->mu); diff --git a/test/core/iomgr/resolve_address_test.cc b/test/core/iomgr/resolve_address_test.cc index a0dc484f3e..2fb831a6a4 100644 --- a/test/core/iomgr/resolve_address_test.cc +++ b/test/core/iomgr/resolve_address_test.cc @@ -82,7 +82,7 @@ static void poll_pollset_until_request_done(args_struct* args) { break; } grpc_millis time_left = deadline - grpc_core::ExecCtx::Get()->Now(); - gpr_log(GPR_DEBUG, "done=%d, time_left=%" PRIdPTR, done, time_left); + gpr_log(GPR_DEBUG, "done=%d, time_left=%" PRId64, done, time_left); GPR_ASSERT(time_left >= 0); grpc_pollset_worker* worker = nullptr; gpr_mu_lock(args->mu); diff --git a/test/core/iomgr/timer_heap_test.cc b/test/core/iomgr/timer_heap_test.cc index ebe5e32f3a..ebe5e6610d 100644 --- a/test/core/iomgr/timer_heap_test.cc +++ b/test/core/iomgr/timer_heap_test.cc @@ -204,7 +204,7 @@ static void test2(void) { } if (num_inserted) { - gpr_atm* min_deadline = nullptr; + grpc_millis* min_deadline = nullptr; for (size_t i = 0; i < elems_size; i++) { if (elems[i].inserted) { if (min_deadline == nullptr) { diff --git a/test/core/transport/timeout_encoding_test.cc b/test/core/transport/timeout_encoding_test.cc index 2367accc2e..b7044b5b41 100644 --- a/test/core/transport/timeout_encoding_test.cc +++ b/test/core/transport/timeout_encoding_test.cc @@ -71,7 +71,7 @@ static void assert_decodes_as(const char* buffer, grpc_millis expected) { GPR_ASSERT(1 == grpc_http2_decode_timeout( grpc_slice_from_static_string(buffer), &got)); if (got != expected) { - gpr_log(GPR_ERROR, "got:'%" PRIdPTR "' != expected:'%" PRIdPTR "'", got, + gpr_log(GPR_ERROR, "got:'%" PRId64 "' != expected:'%" PRId64 "'", got, expected); abort(); } diff --git a/test/cpp/end2end/client_lb_end2end_test.cc b/test/cpp/end2end/client_lb_end2end_test.cc index a39e443cf0..eeec8e90bd 100644 --- a/test/cpp/end2end/client_lb_end2end_test.cc +++ b/test/cpp/end2end/client_lb_end2end_test.cc @@ -198,7 +198,8 @@ class ClientLbEnd2endTest : public ::testing::Test { bool SendRpc( const std::unique_ptr<grpc::testing::EchoTestService::Stub>& stub, - EchoResponse* response = nullptr, int timeout_ms = 1000) { + EchoResponse* response = nullptr, int timeout_ms = 1000, + Status* result = nullptr) { const bool local_response = (response == nullptr); if (local_response) response = new EchoResponse; EchoRequest request; @@ -206,6 +207,7 @@ class ClientLbEnd2endTest : public ::testing::Test { ClientContext context; context.set_deadline(grpc_timeout_milliseconds_to_deadline(timeout_ms)); Status status = stub->Echo(&context, request, response); + if (result != nullptr) *result = status; if (local_response) delete response; return status.ok(); } @@ -214,12 +216,15 @@ class ClientLbEnd2endTest : public ::testing::Test { const std::unique_ptr<grpc::testing::EchoTestService::Stub>& stub, const grpc_core::DebugLocation& location) { EchoResponse response; - const bool success = SendRpc(stub, &response); - if (!success) abort(); - ASSERT_TRUE(success) << "From " << location.file() << ":" - << location.line(); + Status status; + const bool success = SendRpc(stub, &response, 2000, &status); + ASSERT_TRUE(success) << "From " << location.file() << ":" << location.line() + << "\n" + << "Error: " << status.error_message() << " " + << status.error_details(); ASSERT_EQ(response.message(), kRequestMessage_) << "From " << location.file() << ":" << location.line(); + if (!success) abort(); } void CheckRpcSendFailure( @@ -362,7 +367,7 @@ TEST_F(ClientLbEnd2endTest, PickFirstBackOffInitialReconnect) { grpc_timeout_milliseconds_to_deadline(kInitialBackOffMs * 2))); const gpr_timespec t1 = gpr_now(GPR_CLOCK_MONOTONIC); const grpc_millis waited_ms = gpr_time_to_millis(gpr_time_sub(t1, t0)); - gpr_log(GPR_DEBUG, "Waited %ld milliseconds", waited_ms); + gpr_log(GPR_DEBUG, "Waited %" PRId64 " milliseconds", waited_ms); // We should have waited at least kInitialBackOffMs. We substract one to // account for test and precision accuracy drift. EXPECT_GE(waited_ms, kInitialBackOffMs - 1); @@ -391,7 +396,7 @@ TEST_F(ClientLbEnd2endTest, PickFirstBackOffMinReconnect) { grpc_timeout_milliseconds_to_deadline(kMinReconnectBackOffMs * 2)); const gpr_timespec t1 = gpr_now(GPR_CLOCK_MONOTONIC); const grpc_millis waited_ms = gpr_time_to_millis(gpr_time_sub(t1, t0)); - gpr_log(GPR_DEBUG, "Waited %ld ms", waited_ms); + gpr_log(GPR_DEBUG, "Waited %" PRId64 " ms", waited_ms); // We should have waited at least kMinReconnectBackOffMs. We substract one to // account for test and precision accuracy drift. EXPECT_GE(waited_ms, kMinReconnectBackOffMs - 1); diff --git a/tools/distrib/python/make_grpcio_tools.py b/tools/distrib/python/make_grpcio_tools.py index 4847233217..a6fdae28d7 100755 --- a/tools/distrib/python/make_grpcio_tools.py +++ b/tools/distrib/python/make_grpcio_tools.py @@ -98,6 +98,7 @@ def protobuf_submodule_commit_hash(): def bazel_query(query): + print('Running "bazel query %s"' % query) output = subprocess.check_output([BAZEL_DEPS, query]) return output.splitlines() @@ -156,6 +157,7 @@ def main(): shutil.copyfile(source_file, target_file) try: + print('Invoking "bazel query" to gather the protobuf dependencies.') protoc_lib_deps_content = get_deps() except Exception as error: # We allow this script to succeed even if we couldn't get the dependencies, @@ -167,6 +169,8 @@ def main(): # If we successfully got the dependencies, truncate and rewrite the deps file. with open(GRPC_PYTHON_PROTOC_LIB_DEPS, 'w') as deps_file: deps_file.write(protoc_lib_deps_content) + print('File "%s" updated.' % GRPC_PYTHON_PROTOC_LIB_DEPS) + print('Done.') if __name__ == '__main__': diff --git a/tools/dockerfile/interoptest/grpc_interop_csharp/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_csharp/Dockerfile index e806ba5923..511e2932d6 100644 --- a/tools/dockerfile/interoptest/grpc_interop_csharp/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_csharp/Dockerfile @@ -100,14 +100,6 @@ RUN mkdir warmup \ && cd .. \ && rm -rf warmup -# Prepare ccache -RUN ln -s /usr/bin/ccache /usr/local/bin/gcc -RUN ln -s /usr/bin/ccache /usr/local/bin/g++ -RUN ln -s /usr/bin/ccache /usr/local/bin/cc -RUN ln -s /usr/bin/ccache /usr/local/bin/c++ -RUN ln -s /usr/bin/ccache /usr/local/bin/clang -RUN ln -s /usr/bin/ccache /usr/local/bin/clang++ - RUN mkdir /var/local/jenkins diff --git a/tools/dockerfile/interoptest/grpc_interop_csharpcoreclr/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_csharpcoreclr/Dockerfile index e806ba5923..511e2932d6 100644 --- a/tools/dockerfile/interoptest/grpc_interop_csharpcoreclr/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_csharpcoreclr/Dockerfile @@ -100,14 +100,6 @@ RUN mkdir warmup \ && cd .. \ && rm -rf warmup -# Prepare ccache -RUN ln -s /usr/bin/ccache /usr/local/bin/gcc -RUN ln -s /usr/bin/ccache /usr/local/bin/g++ -RUN ln -s /usr/bin/ccache /usr/local/bin/cc -RUN ln -s /usr/bin/ccache /usr/local/bin/c++ -RUN ln -s /usr/bin/ccache /usr/local/bin/clang -RUN ln -s /usr/bin/ccache /usr/local/bin/clang++ - RUN mkdir /var/local/jenkins diff --git a/tools/dockerfile/interoptest/grpc_interop_cxx/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_cxx/Dockerfile index d3eb456e94..9ede89b78d 100644 --- a/tools/dockerfile/interoptest/grpc_interop_cxx/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_cxx/Dockerfile @@ -68,14 +68,6 @@ RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.5.2.post1 six==1.10.0 t # C++ dependencies RUN apt-get update && apt-get -y install libgflags-dev libgtest-dev libc++-dev clang && apt-get clean -# Prepare ccache -RUN ln -s /usr/bin/ccache /usr/local/bin/gcc -RUN ln -s /usr/bin/ccache /usr/local/bin/g++ -RUN ln -s /usr/bin/ccache /usr/local/bin/cc -RUN ln -s /usr/bin/ccache /usr/local/bin/c++ -RUN ln -s /usr/bin/ccache /usr/local/bin/clang -RUN ln -s /usr/bin/ccache /usr/local/bin/clang++ - RUN mkdir /var/local/jenkins diff --git a/tools/dockerfile/interoptest/grpc_interop_node/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_node/Dockerfile index 539a869a7b..015641684b 100644 --- a/tools/dockerfile/interoptest/grpc_interop_node/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_node/Dockerfile @@ -71,20 +71,13 @@ RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.5.2.post1 six==1.10.0 t RUN touch .profile RUN curl -o- https://raw.githubusercontent.com/creationix/nvm/v0.25.4/install.sh | bash # Install all versions of node that we want to test -RUN /bin/bash -l -c "nvm install 4 && npm config set cache /tmp/npm-cache && npm install -g npm" -RUN /bin/bash -l -c "nvm install 5 && npm config set cache /tmp/npm-cache && npm install -g npm" -RUN /bin/bash -l -c "nvm install 6 && npm config set cache /tmp/npm-cache && npm install -g npm" -RUN /bin/bash -l -c "nvm install 8 && npm config set cache /tmp/npm-cache && npm install -g npm" -RUN /bin/bash -l -c "nvm install 9 && npm config set cache /tmp/npm-cache && npm install -g npm" -RUN /bin/bash -l -c "nvm alias default 9" -# Prepare ccache -RUN ln -s /usr/bin/ccache /usr/local/bin/gcc -RUN ln -s /usr/bin/ccache /usr/local/bin/g++ -RUN ln -s /usr/bin/ccache /usr/local/bin/cc -RUN ln -s /usr/bin/ccache /usr/local/bin/c++ -RUN ln -s /usr/bin/ccache /usr/local/bin/clang -RUN ln -s /usr/bin/ccache /usr/local/bin/clang++ - +RUN /bin/bash -l -c "nvm install 4 && npm config set cache /tmp/npm-cache" +RUN /bin/bash -l -c "nvm install 5 && npm config set cache /tmp/npm-cache" +RUN /bin/bash -l -c "nvm install 6 && npm config set cache /tmp/npm-cache" +RUN /bin/bash -l -c "nvm install 8 && npm config set cache /tmp/npm-cache" +RUN /bin/bash -l -c "nvm install 9 && npm config set cache /tmp/npm-cache" +RUN /bin/bash -l -c "nvm install 10 && npm config set cache /tmp/npm-cache" +RUN /bin/bash -l -c "nvm alias default 10" RUN mkdir /var/local/jenkins diff --git a/tools/dockerfile/interoptest/grpc_interop_nodepurejs/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_nodepurejs/Dockerfile index 7f3b20aea2..0006be790e 100644 --- a/tools/dockerfile/interoptest/grpc_interop_nodepurejs/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_nodepurejs/Dockerfile @@ -56,20 +56,13 @@ RUN apt-get update && apt-get install -y time && apt-get clean RUN touch .profile RUN curl -o- https://raw.githubusercontent.com/creationix/nvm/v0.25.4/install.sh | bash # Install all versions of node that we want to test -RUN /bin/bash -l -c "nvm install 4 && npm config set cache /tmp/npm-cache && npm install -g npm" -RUN /bin/bash -l -c "nvm install 5 && npm config set cache /tmp/npm-cache && npm install -g npm" -RUN /bin/bash -l -c "nvm install 6 && npm config set cache /tmp/npm-cache && npm install -g npm" -RUN /bin/bash -l -c "nvm install 8 && npm config set cache /tmp/npm-cache && npm install -g npm" -RUN /bin/bash -l -c "nvm install 9 && npm config set cache /tmp/npm-cache && npm install -g npm" -RUN /bin/bash -l -c "nvm alias default 9" -# Prepare ccache -RUN ln -s /usr/bin/ccache /usr/local/bin/gcc -RUN ln -s /usr/bin/ccache /usr/local/bin/g++ -RUN ln -s /usr/bin/ccache /usr/local/bin/cc -RUN ln -s /usr/bin/ccache /usr/local/bin/c++ -RUN ln -s /usr/bin/ccache /usr/local/bin/clang -RUN ln -s /usr/bin/ccache /usr/local/bin/clang++ - +RUN /bin/bash -l -c "nvm install 4 && npm config set cache /tmp/npm-cache" +RUN /bin/bash -l -c "nvm install 5 && npm config set cache /tmp/npm-cache" +RUN /bin/bash -l -c "nvm install 6 && npm config set cache /tmp/npm-cache" +RUN /bin/bash -l -c "nvm install 8 && npm config set cache /tmp/npm-cache" +RUN /bin/bash -l -c "nvm install 9 && npm config set cache /tmp/npm-cache" +RUN /bin/bash -l -c "nvm install 10 && npm config set cache /tmp/npm-cache" +RUN /bin/bash -l -c "nvm alias default 10" RUN mkdir /var/local/jenkins diff --git a/tools/dockerfile/interoptest/grpc_interop_php/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_php/Dockerfile index d6e229f84a..e987d4c88e 100644 --- a/tools/dockerfile/interoptest/grpc_interop_php/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_php/Dockerfile @@ -57,14 +57,6 @@ RUN apt-get update && apt-get install -y time && apt-get clean RUN apt-get update && apt-get install -y \ git php5 php5-dev phpunit unzip -# Prepare ccache -RUN ln -s /usr/bin/ccache /usr/local/bin/gcc -RUN ln -s /usr/bin/ccache /usr/local/bin/g++ -RUN ln -s /usr/bin/ccache /usr/local/bin/cc -RUN ln -s /usr/bin/ccache /usr/local/bin/c++ -RUN ln -s /usr/bin/ccache /usr/local/bin/clang -RUN ln -s /usr/bin/ccache /usr/local/bin/clang++ - RUN mkdir /var/local/jenkins diff --git a/tools/dockerfile/interoptest/grpc_interop_php7/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_php7/Dockerfile index 7ab80e1ef2..71a2381b77 100644 --- a/tools/dockerfile/interoptest/grpc_interop_php7/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_php7/Dockerfile @@ -60,14 +60,6 @@ RUN cd /var/local/git/php-src \ && make \ && make install -# Prepare ccache -RUN ln -s /usr/bin/ccache /usr/local/bin/gcc -RUN ln -s /usr/bin/ccache /usr/local/bin/g++ -RUN ln -s /usr/bin/ccache /usr/local/bin/cc -RUN ln -s /usr/bin/ccache /usr/local/bin/c++ -RUN ln -s /usr/bin/ccache /usr/local/bin/clang -RUN ln -s /usr/bin/ccache /usr/local/bin/clang++ - RUN mkdir /var/local/jenkins diff --git a/tools/dockerfile/interoptest/grpc_interop_python/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_python/Dockerfile index 94290f3620..dadd856740 100644 --- a/tools/dockerfile/interoptest/grpc_interop_python/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_python/Dockerfile @@ -68,14 +68,6 @@ RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.5.2.post1 six==1.10.0 t RUN curl https://bootstrap.pypa.io/get-pip.py | python3.4 RUN python3.4 -m pip install virtualenv -# Prepare ccache -RUN ln -s /usr/bin/ccache /usr/local/bin/gcc -RUN ln -s /usr/bin/ccache /usr/local/bin/g++ -RUN ln -s /usr/bin/ccache /usr/local/bin/cc -RUN ln -s /usr/bin/ccache /usr/local/bin/c++ -RUN ln -s /usr/bin/ccache /usr/local/bin/clang -RUN ln -s /usr/bin/ccache /usr/local/bin/clang++ - RUN mkdir /var/local/jenkins diff --git a/tools/dockerfile/interoptest/grpc_interop_ruby/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_ruby/Dockerfile index 75e3314e2b..97c146bb53 100644 --- a/tools/dockerfile/interoptest/grpc_interop_ruby/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_ruby/Dockerfile @@ -79,14 +79,6 @@ RUN /bin/bash -l -c "echo 'export PATH=/usr/local/rvm/bin:$PATH' >> ~/.bashrc" RUN /bin/bash -l -c "echo 'rvm --default use ruby-2.1' >> ~/.bashrc" RUN /bin/bash -l -c "gem install bundler --no-ri --no-rdoc" -# Prepare ccache -RUN ln -s /usr/bin/ccache /usr/local/bin/gcc -RUN ln -s /usr/bin/ccache /usr/local/bin/g++ -RUN ln -s /usr/bin/ccache /usr/local/bin/cc -RUN ln -s /usr/bin/ccache /usr/local/bin/c++ -RUN ln -s /usr/bin/ccache /usr/local/bin/clang -RUN ln -s /usr/bin/ccache /usr/local/bin/clang++ - RUN mkdir /var/local/jenkins diff --git a/tools/dockerfile/test/bazel/Dockerfile b/tools/dockerfile/test/bazel/Dockerfile index e5e81136cb..2cb2c1260e 100644 --- a/tools/dockerfile/test/bazel/Dockerfile +++ b/tools/dockerfile/test/bazel/Dockerfile @@ -30,15 +30,8 @@ RUN apt-get update && apt-get -y install \ # Bazel installation RUN echo "deb [arch=amd64] http://storage.googleapis.com/bazel-apt stable jdk1.8" > /etc/apt/sources.list.d/bazel.list RUN curl https://bazel.build/bazel-release.pub.gpg | apt-key add - -RUN apt-get -y update -RUN apt-get -y install bazel +RUN apt-get -y update && apt-get -y install bazel=0.13.1 && apt-get clean -# Pin Bazel to 0.9.0 -# Installing Bazel via apt-get first is required before installing 0.9.0 to -# allow gRPC to build without errors. See https://github.com/grpc/grpc/issues/10553 -RUN curl -fSsL -O https://github.com/bazelbuild/bazel/releases/download/0.9.0/bazel-0.9.0-installer-linux-x86_64.sh -RUN chmod +x ./bazel-0.9.0-installer-linux-x86_64.sh -RUN ./bazel-0.9.0-installer-linux-x86_64.sh RUN mkdir -p /var/local/jenkins diff --git a/tools/dockerfile/test/csharp_jessie_x64/Dockerfile b/tools/dockerfile/test/csharp_jessie_x64/Dockerfile index 7a8e26d2a7..56bfb89925 100644 --- a/tools/dockerfile/test/csharp_jessie_x64/Dockerfile +++ b/tools/dockerfile/test/csharp_jessie_x64/Dockerfile @@ -104,14 +104,6 @@ RUN mkdir warmup \ && cd .. \ && rm -rf warmup -# Prepare ccache -RUN ln -s /usr/bin/ccache /usr/local/bin/gcc -RUN ln -s /usr/bin/ccache /usr/local/bin/g++ -RUN ln -s /usr/bin/ccache /usr/local/bin/cc -RUN ln -s /usr/bin/ccache /usr/local/bin/c++ -RUN ln -s /usr/bin/ccache /usr/local/bin/clang -RUN ln -s /usr/bin/ccache /usr/local/bin/clang++ - RUN mkdir /var/local/jenkins diff --git a/tools/dockerfile/test/cxx_alpine_x64/Dockerfile b/tools/dockerfile/test/cxx_alpine_x64/Dockerfile index 3449af1e20..82333d305b 100644 --- a/tools/dockerfile/test/cxx_alpine_x64/Dockerfile +++ b/tools/dockerfile/test/cxx_alpine_x64/Dockerfile @@ -50,14 +50,6 @@ RUN git clone https://github.com/gflags/gflags.git && cd gflags && git checkout RUN cd gflags && cmake . && make && make install RUN ln -s /usr/local/include/gflags /usr/include/gflags -# Prepare ccache -RUN ln -s /usr/bin/ccache /usr/local/bin/gcc -RUN ln -s /usr/bin/ccache /usr/local/bin/g++ -RUN ln -s /usr/bin/ccache /usr/local/bin/cc -RUN ln -s /usr/bin/ccache /usr/local/bin/c++ -RUN ln -s /usr/bin/ccache /usr/local/bin/clang -RUN ln -s /usr/bin/ccache /usr/local/bin/clang++ - RUN mkdir /var/local/jenkins diff --git a/tools/dockerfile/test/cxx_jessie_x64/Dockerfile b/tools/dockerfile/test/cxx_jessie_x64/Dockerfile index 08efcde5ed..c4f959fc1d 100644 --- a/tools/dockerfile/test/cxx_jessie_x64/Dockerfile +++ b/tools/dockerfile/test/cxx_jessie_x64/Dockerfile @@ -79,45 +79,6 @@ RUN apt-get update && apt-get -y install libgflags-dev libgtest-dev libc++-dev c RUN echo "deb http://ftp.debian.org/debian jessie-backports main" | tee /etc/apt/sources.list.d/jessie-backports.list RUN apt-get update && apt-get install -t jessie-backports -y cmake && apt-get clean -#================= -# Update clang to a version with improved tsan and fuzzing capabilities - -RUN git clone -n -b release_38 http://llvm.org/git/llvm.git && \ - cd llvm && git checkout ad57503 && cd .. -RUN git clone -n -b release_38 http://llvm.org/git/clang.git && \ - cd clang && git checkout ad2c56e && cd .. -RUN git clone -n -b release_38 http://llvm.org/git/compiler-rt.git && \ - cd compiler-rt && git checkout 3176922 && cd .. -RUN git clone -n -b release_38 \ - http://llvm.org/git/clang-tools-extra.git && cd clang-tools-extra && \ - git checkout c288525 && cd .. -RUN git clone -n -b release_38 http://llvm.org/git/libcxx.git && \ - cd libcxx && git checkout fda3549 && cd .. -RUN git clone -n -b release_38 http://llvm.org/git/libcxxabi.git && \ - cd libcxxabi && git checkout 8d4e51d && cd .. - -RUN mv clang llvm/tools -RUN mv compiler-rt llvm/projects -RUN mv clang-tools-extra llvm/tools/clang/tools -RUN mv libcxx llvm/projects -RUN mv libcxxabi llvm/projects - -RUN mkdir llvm-build -RUN cd llvm-build && cmake \ - -DCMAKE_BUILD_TYPE:STRING=Release \ - -DCMAKE_INSTALL_PREFIX:STRING=/usr \ - -DLLVM_TARGETS_TO_BUILD:STRING=X86 \ - ../llvm -RUN make -C llvm-build -j 12 && make -C llvm-build install && rm -rf llvm-build - -# Prepare ccache -RUN ln -s /usr/bin/ccache /usr/local/bin/gcc -RUN ln -s /usr/bin/ccache /usr/local/bin/g++ -RUN ln -s /usr/bin/ccache /usr/local/bin/cc -RUN ln -s /usr/bin/ccache /usr/local/bin/c++ -RUN ln -s /usr/bin/ccache /usr/local/bin/clang -RUN ln -s /usr/bin/ccache /usr/local/bin/clang++ - RUN mkdir /var/local/jenkins diff --git a/tools/dockerfile/test/cxx_jessie_x86/Dockerfile b/tools/dockerfile/test/cxx_jessie_x86/Dockerfile index bb9c7511e3..d07ea9a9b0 100644 --- a/tools/dockerfile/test/cxx_jessie_x86/Dockerfile +++ b/tools/dockerfile/test/cxx_jessie_x86/Dockerfile @@ -72,17 +72,16 @@ RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.5.2.post1 six==1.10.0 t # C++ dependencies RUN apt-get update && apt-get -y install libgflags-dev libgtest-dev libc++-dev clang && apt-get clean -# Prepare ccache -RUN ln -s /usr/bin/ccache /usr/local/bin/gcc -RUN ln -s /usr/bin/ccache /usr/local/bin/g++ -RUN ln -s /usr/bin/ccache /usr/local/bin/cc -RUN ln -s /usr/bin/ccache /usr/local/bin/c++ -RUN ln -s /usr/bin/ccache /usr/local/bin/clang -RUN ln -s /usr/bin/ccache /usr/local/bin/clang++ - RUN mkdir /var/local/jenkins +#================= +# Use cmake 3.6 from jessie-backports +# should only be used for images based on debian jessie. + +RUN echo "deb http://ftp.debian.org/debian jessie-backports main" | tee /etc/apt/sources.list.d/jessie-backports.list +RUN apt-get update && apt-get install -t jessie-backports -y cmake && apt-get clean + # Install gcc-4.8 and other relevant items RUN apt-get update && apt-get -y install gcc-4.8 gcc-4.8-multilib g++-4.8 g++-4.8-multilib && apt-get clean diff --git a/tools/dockerfile/test/cxx_sanitizers_jessie_x64/Dockerfile b/tools/dockerfile/test/cxx_sanitizers_jessie_x64/Dockerfile new file mode 100644 index 0000000000..f60f67665d --- /dev/null +++ b/tools/dockerfile/test/cxx_sanitizers_jessie_x64/Dockerfile @@ -0,0 +1,80 @@ +# Copyright 2018 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This is the base Docker image we use for running tests on RBE +FROM gcr.io/cloud-marketplace/google/rbe-debian8@sha256:1ede2a929b44d629ec5abe86eee6d7ffea1d5a4d247489a8867d46cfde3e38bd + +# Install Git and basic packages. +RUN apt-get update && apt-get install -y \ + autoconf \ + autotools-dev \ + build-essential \ + bzip2 \ + ccache \ + curl \ + dnsutils \ + gcc \ + gcc-multilib \ + git \ + golang \ + gyp \ + lcov \ + libc6 \ + libc6-dbg \ + libc6-dev \ + libgtest-dev \ + libtool \ + make \ + perl \ + strace \ + python-dev \ + python-setuptools \ + python-yaml \ + telnet \ + unzip \ + wget \ + zip && apt-get clean + +#================ +# Build profiling +RUN apt-get update && apt-get install -y time && apt-get clean + +# Google Cloud platform API libraries +RUN apt-get update && apt-get install -y python-pip && apt-get clean +RUN pip install --upgrade google-api-python-client + +#==================== +# Python dependencies + +# Install dependencies + +RUN apt-get update && apt-get install -y \ + python-all-dev \ + python3-all-dev \ + python-pip + +# Install Python packages from PyPI +RUN pip install --upgrade pip==10.0.1 +RUN pip install virtualenv +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.5.2.post1 six==1.10.0 twisted==17.5.0 + +#================= +# C++ dependencies (purposely excluding Clang because it's part of the base image) +RUN apt-get update && apt-get -y install libgflags-dev libgtest-dev libc++-dev && apt-get clean + +# Link llvm-symbolizer to where our test scripts expect to find it +RUN ln -s /usr/local/bin/llvm-symbolizer /usr/bin/llvm-symbolizer + +# Define the default command. +CMD ["bash"] diff --git a/tools/dockerfile/test/cxx_ubuntu1604_x64/Dockerfile b/tools/dockerfile/test/cxx_ubuntu1604_x64/Dockerfile index 65ff58e728..439baadc2c 100644 --- a/tools/dockerfile/test/cxx_ubuntu1604_x64/Dockerfile +++ b/tools/dockerfile/test/cxx_ubuntu1604_x64/Dockerfile @@ -72,14 +72,6 @@ RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.5.2.post1 six==1.10.0 t # C++ dependencies RUN apt-get update && apt-get -y install libgflags-dev libgtest-dev libc++-dev clang && apt-get clean -# Prepare ccache -RUN ln -s /usr/bin/ccache /usr/local/bin/gcc -RUN ln -s /usr/bin/ccache /usr/local/bin/g++ -RUN ln -s /usr/bin/ccache /usr/local/bin/cc -RUN ln -s /usr/bin/ccache /usr/local/bin/c++ -RUN ln -s /usr/bin/ccache /usr/local/bin/clang -RUN ln -s /usr/bin/ccache /usr/local/bin/clang++ - RUN mkdir /var/local/jenkins diff --git a/tools/dockerfile/test/cxx_ubuntu1710_x64/Dockerfile b/tools/dockerfile/test/cxx_ubuntu1710_x64/Dockerfile index e926443134..b80249d2d8 100644 --- a/tools/dockerfile/test/cxx_ubuntu1710_x64/Dockerfile +++ b/tools/dockerfile/test/cxx_ubuntu1710_x64/Dockerfile @@ -72,14 +72,6 @@ RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.5.2.post1 six==1.10.0 t # C++ dependencies RUN apt-get update && apt-get -y install libgflags-dev libgtest-dev libc++-dev clang && apt-get clean -# Prepare ccache -RUN ln -s /usr/bin/ccache /usr/local/bin/gcc -RUN ln -s /usr/bin/ccache /usr/local/bin/g++ -RUN ln -s /usr/bin/ccache /usr/local/bin/cc -RUN ln -s /usr/bin/ccache /usr/local/bin/c++ -RUN ln -s /usr/bin/ccache /usr/local/bin/clang -RUN ln -s /usr/bin/ccache /usr/local/bin/clang++ - RUN mkdir /var/local/jenkins diff --git a/tools/dockerfile/test/fuzzer/Dockerfile b/tools/dockerfile/test/fuzzer/Dockerfile index b53feedfae..986cfcbada 100644 --- a/tools/dockerfile/test/fuzzer/Dockerfile +++ b/tools/dockerfile/test/fuzzer/Dockerfile @@ -110,14 +110,6 @@ RUN cd llvm-build && cmake \ ../llvm RUN make -C llvm-build -j 12 && make -C llvm-build install && rm -rf llvm-build -# Prepare ccache -RUN ln -s /usr/bin/ccache /usr/local/bin/gcc -RUN ln -s /usr/bin/ccache /usr/local/bin/g++ -RUN ln -s /usr/bin/ccache /usr/local/bin/cc -RUN ln -s /usr/bin/ccache /usr/local/bin/c++ -RUN ln -s /usr/bin/ccache /usr/local/bin/clang -RUN ln -s /usr/bin/ccache /usr/local/bin/clang++ - RUN mkdir /var/local/jenkins diff --git a/tools/dockerfile/test/multilang_jessie_x64/Dockerfile b/tools/dockerfile/test/multilang_jessie_x64/Dockerfile index 40e13c385e..a82e7050fc 100644 --- a/tools/dockerfile/test/multilang_jessie_x64/Dockerfile +++ b/tools/dockerfile/test/multilang_jessie_x64/Dockerfile @@ -100,12 +100,13 @@ RUN apt-get update && apt-get -y install libgflags-dev libgtest-dev libc++-dev c RUN touch .profile RUN curl -o- https://raw.githubusercontent.com/creationix/nvm/v0.25.4/install.sh | bash # Install all versions of node that we want to test -RUN /bin/bash -l -c "nvm install 4 && npm config set cache /tmp/npm-cache && npm install -g npm" -RUN /bin/bash -l -c "nvm install 5 && npm config set cache /tmp/npm-cache && npm install -g npm" -RUN /bin/bash -l -c "nvm install 6 && npm config set cache /tmp/npm-cache && npm install -g npm" -RUN /bin/bash -l -c "nvm install 8 && npm config set cache /tmp/npm-cache && npm install -g npm" -RUN /bin/bash -l -c "nvm install 9 && npm config set cache /tmp/npm-cache && npm install -g npm" -RUN /bin/bash -l -c "nvm alias default 9" +RUN /bin/bash -l -c "nvm install 4 && npm config set cache /tmp/npm-cache" +RUN /bin/bash -l -c "nvm install 5 && npm config set cache /tmp/npm-cache" +RUN /bin/bash -l -c "nvm install 6 && npm config set cache /tmp/npm-cache" +RUN /bin/bash -l -c "nvm install 8 && npm config set cache /tmp/npm-cache" +RUN /bin/bash -l -c "nvm install 9 && npm config set cache /tmp/npm-cache" +RUN /bin/bash -l -c "nvm install 10 && npm config set cache /tmp/npm-cache" +RUN /bin/bash -l -c "nvm alias default 10" #================= # PHP dependencies @@ -155,14 +156,6 @@ ENV PATH ~/.local/bin:$PATH # Install Mako to generate files in grpc/grpc-node RUN pip install Mako -# Prepare ccache -RUN ln -s /usr/bin/ccache /usr/local/bin/gcc -RUN ln -s /usr/bin/ccache /usr/local/bin/g++ -RUN ln -s /usr/bin/ccache /usr/local/bin/cc -RUN ln -s /usr/bin/ccache /usr/local/bin/c++ -RUN ln -s /usr/bin/ccache /usr/local/bin/clang -RUN ln -s /usr/bin/ccache /usr/local/bin/clang++ - RUN mkdir /var/local/jenkins diff --git a/tools/dockerfile/test/node_jessie_x64/Dockerfile b/tools/dockerfile/test/node_jessie_x64/Dockerfile index f32b437b75..1fe44b6dd7 100644 --- a/tools/dockerfile/test/node_jessie_x64/Dockerfile +++ b/tools/dockerfile/test/node_jessie_x64/Dockerfile @@ -86,20 +86,13 @@ RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.5.2.post1 six==1.10.0 t RUN touch .profile RUN curl -o- https://raw.githubusercontent.com/creationix/nvm/v0.25.4/install.sh | bash # Install all versions of node that we want to test -RUN /bin/bash -l -c "nvm install 4 && npm config set cache /tmp/npm-cache && npm install -g npm" -RUN /bin/bash -l -c "nvm install 5 && npm config set cache /tmp/npm-cache && npm install -g npm" -RUN /bin/bash -l -c "nvm install 6 && npm config set cache /tmp/npm-cache && npm install -g npm" -RUN /bin/bash -l -c "nvm install 8 && npm config set cache /tmp/npm-cache && npm install -g npm" -RUN /bin/bash -l -c "nvm install 9 && npm config set cache /tmp/npm-cache && npm install -g npm" -RUN /bin/bash -l -c "nvm alias default 9" -# Prepare ccache -RUN ln -s /usr/bin/ccache /usr/local/bin/gcc -RUN ln -s /usr/bin/ccache /usr/local/bin/g++ -RUN ln -s /usr/bin/ccache /usr/local/bin/cc -RUN ln -s /usr/bin/ccache /usr/local/bin/c++ -RUN ln -s /usr/bin/ccache /usr/local/bin/clang -RUN ln -s /usr/bin/ccache /usr/local/bin/clang++ - +RUN /bin/bash -l -c "nvm install 4 && npm config set cache /tmp/npm-cache" +RUN /bin/bash -l -c "nvm install 5 && npm config set cache /tmp/npm-cache" +RUN /bin/bash -l -c "nvm install 6 && npm config set cache /tmp/npm-cache" +RUN /bin/bash -l -c "nvm install 8 && npm config set cache /tmp/npm-cache" +RUN /bin/bash -l -c "nvm install 9 && npm config set cache /tmp/npm-cache" +RUN /bin/bash -l -c "nvm install 10 && npm config set cache /tmp/npm-cache" +RUN /bin/bash -l -c "nvm alias default 10" RUN mkdir /var/local/jenkins diff --git a/tools/dockerfile/test/php7_jessie_x64/Dockerfile b/tools/dockerfile/test/php7_jessie_x64/Dockerfile index e96be2769a..53ef7b3443 100644 --- a/tools/dockerfile/test/php7_jessie_x64/Dockerfile +++ b/tools/dockerfile/test/php7_jessie_x64/Dockerfile @@ -79,14 +79,6 @@ RUN pip install --upgrade pip==10.0.1 RUN pip install virtualenv RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.5.2.post1 six==1.10.0 twisted==17.5.0 -# Prepare ccache -RUN ln -s /usr/bin/ccache /usr/local/bin/gcc -RUN ln -s /usr/bin/ccache /usr/local/bin/g++ -RUN ln -s /usr/bin/ccache /usr/local/bin/cc -RUN ln -s /usr/bin/ccache /usr/local/bin/c++ -RUN ln -s /usr/bin/ccache /usr/local/bin/clang -RUN ln -s /usr/bin/ccache /usr/local/bin/clang++ - RUN mkdir /var/local/jenkins diff --git a/tools/dockerfile/test/php_jessie_x64/Dockerfile b/tools/dockerfile/test/php_jessie_x64/Dockerfile index 88ee267763..e884572640 100644 --- a/tools/dockerfile/test/php_jessie_x64/Dockerfile +++ b/tools/dockerfile/test/php_jessie_x64/Dockerfile @@ -76,14 +76,6 @@ RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.5.2.post1 six==1.10.0 t RUN apt-get update && apt-get install -y \ git php5 php5-dev phpunit unzip -# Prepare ccache -RUN ln -s /usr/bin/ccache /usr/local/bin/gcc -RUN ln -s /usr/bin/ccache /usr/local/bin/g++ -RUN ln -s /usr/bin/ccache /usr/local/bin/cc -RUN ln -s /usr/bin/ccache /usr/local/bin/c++ -RUN ln -s /usr/bin/ccache /usr/local/bin/clang -RUN ln -s /usr/bin/ccache /usr/local/bin/clang++ - RUN mkdir /var/local/jenkins diff --git a/tools/dockerfile/test/python_jessie_x64/Dockerfile b/tools/dockerfile/test/python_jessie_x64/Dockerfile index bd9d55ed8a..c2b4c1845b 100644 --- a/tools/dockerfile/test/python_jessie_x64/Dockerfile +++ b/tools/dockerfile/test/python_jessie_x64/Dockerfile @@ -72,14 +72,6 @@ RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.5.2.post1 six==1.10.0 t RUN curl https://bootstrap.pypa.io/get-pip.py | python3.4 RUN python3.4 -m pip install virtualenv -# Prepare ccache -RUN ln -s /usr/bin/ccache /usr/local/bin/gcc -RUN ln -s /usr/bin/ccache /usr/local/bin/g++ -RUN ln -s /usr/bin/ccache /usr/local/bin/cc -RUN ln -s /usr/bin/ccache /usr/local/bin/c++ -RUN ln -s /usr/bin/ccache /usr/local/bin/clang -RUN ln -s /usr/bin/ccache /usr/local/bin/clang++ - RUN mkdir /var/local/jenkins diff --git a/tools/dockerfile/test/python_pyenv_x64/Dockerfile b/tools/dockerfile/test/python_pyenv_x64/Dockerfile index 6d72a1cc40..43854aa689 100644 --- a/tools/dockerfile/test/python_pyenv_x64/Dockerfile +++ b/tools/dockerfile/test/python_pyenv_x64/Dockerfile @@ -95,14 +95,6 @@ RUN pyenv local 3.5-dev 3.6-dev pypy-5.3.1 RUN curl https://bootstrap.pypa.io/get-pip.py | python3.4 RUN python3.4 -m pip install virtualenv -# Prepare ccache -RUN ln -s /usr/bin/ccache /usr/local/bin/gcc -RUN ln -s /usr/bin/ccache /usr/local/bin/g++ -RUN ln -s /usr/bin/ccache /usr/local/bin/cc -RUN ln -s /usr/bin/ccache /usr/local/bin/c++ -RUN ln -s /usr/bin/ccache /usr/local/bin/clang -RUN ln -s /usr/bin/ccache /usr/local/bin/clang++ - RUN mkdir /var/local/jenkins diff --git a/tools/dockerfile/test/ruby_jessie_x64/Dockerfile b/tools/dockerfile/test/ruby_jessie_x64/Dockerfile index 37d909a038..d6f7459685 100644 --- a/tools/dockerfile/test/ruby_jessie_x64/Dockerfile +++ b/tools/dockerfile/test/ruby_jessie_x64/Dockerfile @@ -83,14 +83,6 @@ RUN /bin/bash -l -c "echo 'export PATH=/usr/local/rvm/bin:$PATH' >> ~/.bashrc" RUN /bin/bash -l -c "echo 'rvm --default use ruby-2.1' >> ~/.bashrc" RUN /bin/bash -l -c "gem install bundler --no-ri --no-rdoc" -# Prepare ccache -RUN ln -s /usr/bin/ccache /usr/local/bin/gcc -RUN ln -s /usr/bin/ccache /usr/local/bin/g++ -RUN ln -s /usr/bin/ccache /usr/local/bin/cc -RUN ln -s /usr/bin/ccache /usr/local/bin/c++ -RUN ln -s /usr/bin/ccache /usr/local/bin/clang -RUN ln -s /usr/bin/ccache /usr/local/bin/clang++ - RUN mkdir /var/local/jenkins diff --git a/tools/dockerfile/test/sanity/Dockerfile b/tools/dockerfile/test/sanity/Dockerfile index 4885843aa4..cb153012ea 100644 --- a/tools/dockerfile/test/sanity/Dockerfile +++ b/tools/dockerfile/test/sanity/Dockerfile @@ -85,26 +85,6 @@ RUN apt-get update && apt-get install -y \ shellcheck RUN pip install simplejson mako -#====================================== -# More sanity test dependencies (bazel) -RUN echo "deb http://http.debian.net/debian jessie-backports main" >> /etc/apt/sources.list -RUN apt-get update -RUN apt-get install -y -t jessie-backports openjdk-8-jdk - -#======================== -# Bazel installation -RUN echo "deb [arch=amd64] http://storage.googleapis.com/bazel-apt stable jdk1.8" > /etc/apt/sources.list.d/bazel.list -RUN curl https://bazel.build/bazel-release.pub.gpg | apt-key add - -RUN apt-get -y update -RUN apt-get -y install bazel - -# Pin Bazel to 0.9.0 -# Installing Bazel via apt-get first is required before installing 0.9.0 to -# allow gRPC to build without errors. See https://github.com/grpc/grpc/issues/10553 -RUN curl -fSsL -O https://github.com/bazelbuild/bazel/releases/download/0.9.0/bazel-0.9.0-installer-linux-x86_64.sh -RUN chmod +x ./bazel-0.9.0-installer-linux-x86_64.sh -RUN ./bazel-0.9.0-installer-linux-x86_64.sh - RUN apt-get update && apt-get -y install wget xz-utils RUN wget http://releases.llvm.org/5.0.0/clang+llvm-5.0.0-linux-x86_64-ubuntu14.04.tar.xz RUN tar xf clang+llvm-5.0.0-linux-x86_64-ubuntu14.04.tar.xz @@ -113,14 +93,6 @@ ENV CLANG_FORMAT=clang-format RUN ln -s /clang+llvm-5.0.0-linux-x86_64-ubuntu14.04/bin/clang-tidy /usr/local/bin/clang-tidy ENV CLANG_TIDY=clang-tidy -# Prepare ccache -RUN ln -s /usr/bin/ccache /usr/local/bin/gcc -RUN ln -s /usr/bin/ccache /usr/local/bin/g++ -RUN ln -s /usr/bin/ccache /usr/local/bin/cc -RUN ln -s /usr/bin/ccache /usr/local/bin/c++ -RUN ln -s /usr/bin/ccache /usr/local/bin/clang -RUN ln -s /usr/bin/ccache /usr/local/bin/clang++ - RUN mkdir /var/local/jenkins diff --git a/tools/internal_ci/helper_scripts/delete_nonartifacts.sh b/tools/internal_ci/helper_scripts/delete_nonartifacts.sh index 6c49b7377e..c7d6ba6d44 100755 --- a/tools/internal_ci/helper_scripts/delete_nonartifacts.sh +++ b/tools/internal_ci/helper_scripts/delete_nonartifacts.sh @@ -24,4 +24,4 @@ cd "$(dirname "$0")/../../.." # after finishing each build. We only leave files we want to keep: # - reports and artifacts # - directory containing the kokoro scripts to prevent deleting a script while being executed. -time find . -type f -not -iname "*sponge_log.xml" -not -path "./reports/*" -not -path "./artifacts/*" -not -path "./tools/internal_ci/*" -exec rm -f {} +
\ No newline at end of file +time find . -type f -not -iname "*sponge_log.xml" -not -path "./reports/*" -not -path "./artifacts/*" -not -path "./tools/internal_ci/*" -exec rm -f {} + diff --git a/tools/internal_ci/linux/grpc_coverage.sh b/tools/internal_ci/linux/grpc_coverage.sh index fd927a85fd..97166372ab 100755 --- a/tools/internal_ci/linux/grpc_coverage.sh +++ b/tools/internal_ci/linux/grpc_coverage.sh @@ -26,4 +26,16 @@ python tools/run_tests/run_tests.py \ -l all \ -c gcov \ -x sponge_log.xml \ - -j 16 + -j 16 || FAILED="true" + +# HTML reports can't be easily displayed in GCS, so create a zip archive +# and put it under reports directory to get it uploaded as an artifact. +zip -q -r coverage_report.zip reports || true +rm -rf reports || true +mkdir reports || true +mv coverage_report.zip reports || true + +if [ "$FAILED" != "" ] +then + exit 1 +fi diff --git a/tools/internal_ci/linux/grpc_run_tests_matrix.sh b/tools/internal_ci/linux/grpc_run_tests_matrix.sh index 1018708f96..4e7515227b 100755 --- a/tools/internal_ci/linux/grpc_run_tests_matrix.sh +++ b/tools/internal_ci/linux/grpc_run_tests_matrix.sh @@ -29,9 +29,6 @@ fi tools/run_tests/run_tests_matrix.py $RUN_TESTS_FLAGS || FAILED="true" -# Reveal leftover processes that might be left behind by the build -ps aux | grep -i kbuilder - echo 'Exiting gRPC main test script.' if [ "$FAILED" != "" ] diff --git a/tools/internal_ci/macos/grpc_run_tests_matrix.sh b/tools/internal_ci/macos/grpc_run_tests_matrix.sh index 6e0c2bb487..9a43e4869b 100755 --- a/tools/internal_ci/macos/grpc_run_tests_matrix.sh +++ b/tools/internal_ci/macos/grpc_run_tests_matrix.sh @@ -25,14 +25,6 @@ tools/run_tests/run_tests_matrix.py $RUN_TESTS_FLAGS || FAILED="true" # kill port_server.py to prevent the build from hanging ps aux | grep port_server\\.py | awk '{print $2}' | xargs kill -9 -# Reveal leftover processes that might be left behind by the build -ps aux | grep -i kbuilder - -# TODO(jtattermusch): better debugging of clock skew, remove once not needed -date - -echo 'Exiting gRPC main test script.' - if [ "$FAILED" != "" ] then exit 1 diff --git a/tools/interop_matrix/client_matrix.py b/tools/interop_matrix/client_matrix.py index 2fe45d5573..22e522bf6e 100644 --- a/tools/interop_matrix/client_matrix.py +++ b/tools/interop_matrix/client_matrix.py @@ -85,10 +85,13 @@ LANG_RELEASE_MATRIX = { 'v1.9.1': None }, { - 'v1.10.0': None + 'v1.10.1': None }, { - 'v1.11.0': None + 'v1.11.1': None + }, + { + 'v1.12.0': None }, ], 'go': [ @@ -199,10 +202,13 @@ LANG_RELEASE_MATRIX = { 'v1.9.1': None }, { - 'v1.10.0': None + 'v1.10.1': None }, { - 'v1.11.0': None + 'v1.11.1': None + }, + { + 'v1.12.0': None }, ], 'node': [ @@ -272,10 +278,13 @@ LANG_RELEASE_MATRIX = { 'v1.9.1': None }, { - 'v1.10.0': None + 'v1.10.1': None }, { - 'v1.11.0': None + 'v1.11.1': None + }, + { + 'v1.12.0': None }, ], 'php': [ @@ -307,10 +316,13 @@ LANG_RELEASE_MATRIX = { 'v1.9.1': None }, { - 'v1.10.0': None + 'v1.10.1': None }, { - 'v1.11.0': None + 'v1.11.1': None + }, + { + 'v1.12.0': None }, ], 'csharp': [ @@ -340,10 +352,13 @@ LANG_RELEASE_MATRIX = { 'v1.9.1': None }, { - 'v1.10.0': None + 'v1.10.1': None }, { - 'v1.11.0': None + 'v1.11.1': None + }, + { + 'v1.12.0': None }, ], } diff --git a/tools/run_tests/dockerize/build_and_run_docker.sh b/tools/run_tests/dockerize/build_and_run_docker.sh index 3ea3ca3c0b..4ef74085f9 100755 --- a/tools/run_tests/dockerize/build_and_run_docker.sh +++ b/tools/run_tests/dockerize/build_and_run_docker.sh @@ -56,6 +56,7 @@ CONTAINER_NAME="build_and_run_docker_$(uuidgen)" # shellcheck disable=SC2086 docker run \ "$@" \ + --cap-add SYS_PTRACE \ -e EXTERNAL_GIT_ROOT="/var/local/jenkins/grpc" \ -e THIS_IS_REALLY_NEEDED='see https://github.com/docker/docker/issues/14203 for why docker is awful' \ -e "KOKORO_BUILD_ID=$KOKORO_BUILD_ID" \ diff --git a/tools/run_tests/dockerize/build_docker_and_run_tests.sh b/tools/run_tests/dockerize/build_docker_and_run_tests.sh index 21eccbaac9..614049cae5 100755 --- a/tools/run_tests/dockerize/build_docker_and_run_tests.sh +++ b/tools/run_tests/dockerize/build_docker_and_run_tests.sh @@ -53,6 +53,7 @@ DOCKER_EXIT_CODE=0 # TODO: silence complaint about $TTY_FLAG expansion in some other way # shellcheck disable=SC2086 docker run \ + --cap-add SYS_PTRACE \ -e "RUN_TESTS_COMMAND=$RUN_TESTS_COMMAND" \ -e "config=$config" \ -e "arch=$arch" \ diff --git a/tools/run_tests/dockerize/build_interop_image.sh b/tools/run_tests/dockerize/build_interop_image.sh index 2bef4a3310..fcfcdeb4e4 100755 --- a/tools/run_tests/dockerize/build_interop_image.sh +++ b/tools/run_tests/dockerize/build_interop_image.sh @@ -104,6 +104,7 @@ CONTAINER_NAME="build_${BASE_NAME}_$(uuidgen)" # Same for $TTY_FLAG # shellcheck disable=SC2086 (docker run \ + --cap-add SYS_PTRACE \ -e CCACHE_DIR=/tmp/ccache \ -e THIS_IS_REALLY_NEEDED='see https://github.com/docker/docker/issues/14203 for why docker is awful' \ -e THIS_IS_REALLY_NEEDED_ONCE_AGAIN='For issue 4835. See https://github.com/docker/docker/issues/14203 for why docker is awful' \ diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index 5aabf9c161..ea4c7c3c65 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -490,6 +490,14 @@ class CLanguage(object): return 'Makefile' def _clang_make_options(self, version_suffix=''): + if self.args.config == 'ubsan': + return [ + 'CC=clang%s' % version_suffix, + 'CXX=clang++%s' % version_suffix, + 'LD=clang++%s' % version_suffix, + 'LDXX=clang++%s' % version_suffix + ] + return [ 'CC=clang%s' % version_suffix, 'CXX=clang++%s' % version_suffix, @@ -531,6 +539,10 @@ class CLanguage(object): elif compiler == 'clang3.7': return ('ubuntu1604', self._clang_make_options(version_suffix='-3.7')) + elif compiler == 'clang7.0': + # clang++-7.0 alias doesn't exist and there are no other clang versions + # installed. + return ('sanitizers_jessie', self._clang_make_options()) else: raise Exception('Compiler %s not supported.' % compiler) @@ -1343,10 +1355,10 @@ argp.add_argument( '--compiler', choices=[ 'default', 'gcc4.4', 'gcc4.6', 'gcc4.8', 'gcc4.9', 'gcc5.3', 'gcc7.2', - 'gcc_musl', 'clang3.4', 'clang3.5', 'clang3.6', 'clang3.7', 'python2.7', - 'python3.4', 'python3.5', 'python3.6', 'pypy', 'pypy3', 'python_alpine', - 'all_the_cpythons', 'electron1.3', 'electron1.6', 'coreclr', 'cmake', - 'cmake_vs2015', 'cmake_vs2017' + 'gcc_musl', 'clang3.4', 'clang3.5', 'clang3.6', 'clang3.7', 'clang7.0', + 'python2.7', 'python3.4', 'python3.5', 'python3.6', 'pypy', 'pypy3', + 'python_alpine', 'all_the_cpythons', 'electron1.3', 'electron1.6', + 'coreclr', 'cmake', 'cmake_vs2015', 'cmake_vs2017' ], default='default', help= diff --git a/tools/run_tests/run_tests_matrix.py b/tools/run_tests/run_tests_matrix.py index 64d80ab721..0af9e0cbbe 100755 --- a/tools/run_tests/run_tests_matrix.py +++ b/tools/run_tests/run_tests_matrix.py @@ -245,6 +245,8 @@ def _create_test_jobs(extra_args=[], inner_jobs=_DEFAULT_INNER_JOBS): languages=['c'], configs=['msan', 'asan', 'tsan', 'ubsan'], platforms=['linux'], + arch='x64', + compiler='clang7.0', labels=['sanitizers', 'corelang'], extra_args=extra_args, inner_jobs=inner_jobs, @@ -253,6 +255,8 @@ def _create_test_jobs(extra_args=[], inner_jobs=_DEFAULT_INNER_JOBS): languages=['c++'], configs=['asan'], platforms=['linux'], + arch='x64', + compiler='clang7.0', labels=['sanitizers', 'corelang'], extra_args=extra_args, inner_jobs=inner_jobs, @@ -261,6 +265,8 @@ def _create_test_jobs(extra_args=[], inner_jobs=_DEFAULT_INNER_JOBS): languages=['c++'], configs=['tsan'], platforms=['linux'], + arch='x64', + compiler='clang7.0', labels=['sanitizers', 'corelang'], extra_args=extra_args, inner_jobs=inner_jobs, @@ -286,7 +292,7 @@ def _create_portability_test_jobs(extra_args=[], # portability C and C++ on x64 for compiler in [ 'gcc4.8', 'gcc5.3', 'gcc7.2', 'gcc_musl', 'clang3.5', 'clang3.6', - 'clang3.7' + 'clang3.7', 'clang7.0' ]: test_jobs += _generate_jobs( languages=['c', 'c++'], |