diff options
32 files changed, 544 insertions, 143 deletions
diff --git a/src/core/lib/debug/stats_data.c b/src/core/lib/debug/stats_data.c index a18efcb524..fc1ac223de 100644 --- a/src/core/lib/debug/stats_data.c +++ b/src/core/lib/debug/stats_data.c @@ -27,6 +27,12 @@ const char *grpc_stats_counter_name[GRPC_STATS_COUNTER_COUNT] = { "server_calls_created", "syscall_poll", "syscall_wait", + "pollset_kick", + "pollset_kicked_without_poller", + "pollset_kicked_again", + "pollset_kick_wakeup_fd", + "pollset_kick_wakeup_cv", + "pollset_kick_own_thread", "histogram_slow_lookups", "syscall_write", "syscall_read", @@ -64,6 +70,16 @@ const char *grpc_stats_counter_doc[GRPC_STATS_COUNTER_COUNT] = { "Number of server side calls created by this process", "Number of polling syscalls (epoll_wait, poll, etc) made by this process", "Number of sleeping syscalls made by this process", + "How many polling wakeups were performed by the process", + "How many times was a polling wakeup requested without an active poller", + "How many times was the same polling worker awoken repeatedly before " + "waking up", + "How many times was an eventfd used as the wakeup vector for a polling " + "wakeup", + "How many times was a condition variable used as the wakeup vector for a " + "polling wakeup", + "How many times could a polling wakeup be satisfied by keeping the waking " + "thread awake?", "Number of times histogram increments went through the slow (binary " "search) path", "Number of write syscalls (or equivalent - eg sendmsg) made by this " @@ -105,6 +121,8 @@ const char *grpc_stats_counter_doc[GRPC_STATS_COUNTER_COUNT] = { "outstanding requests)", }; const char *grpc_stats_histogram_name[GRPC_STATS_HISTOGRAM_COUNT] = { + "call_initial_size", + "poll_events_returned", "tcp_write_size", "tcp_write_iov_size", "tcp_read_size", @@ -118,6 +136,8 @@ const char *grpc_stats_histogram_name[GRPC_STATS_HISTOGRAM_COUNT] = { "server_cqs_checked", }; const char *grpc_stats_histogram_doc[GRPC_STATS_HISTOGRAM_COUNT] = { + "Initial size of the grpc_call arena created at call start", + "How many events are called for each syscall_poll", "Number of bytes offered to each syscall_write", "Number of byte segments offered to each syscall_write", "Number of bytes received by each syscall_read", @@ -132,6 +152,42 @@ const char *grpc_stats_histogram_doc[GRPC_STATS_HISTOGRAM_COUNT] = { "requested the incoming call", }; const int grpc_stats_table_0[65] = { + 0, 1, 2, 3, 4, 5, 7, 9, 11, 14, + 17, 21, 26, 32, 39, 47, 57, 68, 82, 98, + 117, 140, 167, 199, 238, 284, 339, 404, 482, 575, + 685, 816, 972, 1158, 1380, 1644, 1959, 2334, 2780, 3312, + 3945, 4699, 5597, 6667, 7941, 9459, 11267, 13420, 15984, 19038, + 22676, 27009, 32169, 38315, 45635, 54353, 64737, 77104, 91834, 109378, + 130273, 155159, 184799, 220100, 262144}; +const uint8_t grpc_stats_table_1[124] = { + 0, 0, 0, 1, 1, 1, 2, 2, 3, 3, 3, 4, 4, 5, 5, 6, 6, 6, + 7, 7, 7, 8, 9, 9, 10, 10, 10, 11, 11, 12, 12, 13, 13, 14, 14, 15, + 15, 16, 16, 17, 17, 18, 18, 19, 19, 20, 20, 21, 21, 22, 22, 22, 23, 24, + 24, 25, 25, 26, 26, 26, 27, 27, 28, 29, 29, 30, 30, 30, 31, 31, 32, 33, + 33, 34, 34, 34, 35, 35, 36, 37, 37, 37, 38, 38, 39, 39, 40, 40, 41, 41, + 42, 42, 43, 43, 44, 44, 45, 45, 46, 46, 47, 47, 48, 48, 49, 49, 50, 50, + 51, 51, 52, 52, 53, 53, 54, 54, 55, 55, 56, 56, 57, 57, 58, 58}; +const int grpc_stats_table_2[129] = { + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, + 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 30, + 32, 34, 36, 38, 40, 42, 44, 46, 48, 50, 52, 54, 56, 58, 60, + 63, 66, 69, 72, 75, 78, 81, 84, 87, 90, 94, 98, 102, 106, 110, + 114, 118, 122, 126, 131, 136, 141, 146, 151, 156, 162, 168, 174, 180, 186, + 192, 199, 206, 213, 220, 228, 236, 244, 252, 260, 269, 278, 287, 297, 307, + 317, 327, 338, 349, 360, 372, 384, 396, 409, 422, 436, 450, 464, 479, 494, + 510, 526, 543, 560, 578, 596, 615, 634, 654, 674, 695, 717, 739, 762, 785, + 809, 834, 859, 885, 912, 939, 967, 996, 1024}; +const uint8_t grpc_stats_table_3[166] = { + 0, 0, 0, 1, 1, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, + 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14, 14, 15, 15, 16, 16, 16, + 17, 17, 18, 19, 19, 20, 21, 21, 22, 23, 23, 24, 25, 25, 26, 26, 27, 27, 28, + 28, 29, 29, 30, 30, 31, 31, 32, 32, 33, 33, 34, 34, 35, 36, 36, 37, 38, 39, + 40, 40, 41, 42, 42, 43, 44, 44, 45, 46, 46, 47, 48, 48, 49, 49, 50, 50, 51, + 51, 52, 52, 53, 53, 54, 54, 55, 56, 57, 58, 59, 59, 60, 61, 62, 63, 63, 64, + 65, 65, 66, 67, 67, 68, 69, 69, 70, 71, 71, 72, 72, 73, 73, 74, 75, 75, 76, + 76, 77, 78, 79, 79, 80, 81, 82, 83, 84, 85, 85, 86, 87, 88, 88, 89, 90, 90, + 91, 92, 92, 93, 94, 94, 95, 95, 96, 97, 97, 98, 98, 99}; +const int grpc_stats_table_4[65] = { 0, 1, 2, 3, 4, 6, 8, 11, 15, 20, 26, 34, 44, 57, 73, 94, 121, 155, 199, 255, 327, 419, 537, 688, @@ -141,27 +197,78 @@ const int grpc_stats_table_0[65] = { 326126, 417200, 533707, 682750, 873414, 1117323, 1429345, 1828502, 2339127, 2992348, 3827987, 4896985, 6264509, 8013925, 10251880, 13114801, 16777216}; -const uint8_t grpc_stats_table_1[87] = { +const uint8_t grpc_stats_table_5[87] = { 0, 0, 1, 1, 2, 3, 3, 4, 4, 5, 6, 6, 7, 8, 8, 9, 10, 11, 11, 12, 13, 13, 14, 15, 15, 16, 17, 17, 18, 19, 20, 20, 21, 22, 22, 23, 24, 25, 25, 26, 27, 27, 28, 29, 29, 30, 31, 31, 32, 33, 34, 34, 35, 36, 36, 37, 38, 39, 39, 40, 41, 41, 42, 43, 44, 44, 45, 45, 46, 47, 48, 48, 49, 50, 51, 51, 52, 53, 53, 54, 55, 56, 56, 57, 58, 58, 59}; -const int grpc_stats_table_2[65] = { +const int grpc_stats_table_6[65] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 14, 16, 18, 20, 22, 24, 27, 30, 33, 36, 39, 43, 47, 51, 56, 61, 66, 72, 78, 85, 92, 100, 109, 118, 128, 139, 151, 164, 178, 193, 209, 226, 244, 264, 285, 308, 333, 359, 387, 418, 451, 486, 524, 565, 609, 656, 707, 762, 821, 884, 952, 1024}; -const uint8_t grpc_stats_table_3[102] = { +const uint8_t grpc_stats_table_7[102] = { 0, 0, 0, 1, 1, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 6, 7, 7, 7, 8, 8, 9, 9, 10, 11, 11, 12, 12, 13, 13, 14, 14, 14, 15, 15, 16, 16, 17, 17, 18, 19, 19, 20, 20, 21, 21, 22, 22, 23, 23, 24, 24, 24, 25, 26, 27, 27, 28, 28, 29, 29, 30, 30, 31, 31, 32, 32, 33, 33, 34, 35, 35, 36, 37, 37, 38, 38, 39, 39, 40, 40, 41, 41, 42, 42, 43, 44, 44, 45, 46, 46, 47, 48, 48, 49, 49, 50, 50, 51, 51}; -const int grpc_stats_table_4[9] = {0, 1, 2, 4, 7, 13, 23, 39, 64}; -const uint8_t grpc_stats_table_5[9] = {0, 0, 1, 2, 2, 3, 4, 4, 5}; +const int grpc_stats_table_8[9] = {0, 1, 2, 4, 7, 13, 23, 39, 64}; +const uint8_t grpc_stats_table_9[9] = {0, 0, 1, 2, 2, 3, 4, 4, 5}; +void grpc_stats_inc_call_initial_size(grpc_exec_ctx *exec_ctx, int value) { + value = GPR_CLAMP(value, 0, 262144); + if (value < 6) { + GRPC_STATS_INC_HISTOGRAM((exec_ctx), GRPC_STATS_HISTOGRAM_CALL_INITIAL_SIZE, + value); + return; + } + union { + double dbl; + uint64_t uint; + } _val, _bkt; + _val.dbl = value; + if (_val.uint < 4651092515166879744ull) { + int bucket = + grpc_stats_table_1[((_val.uint - 4618441417868443648ull) >> 49)] + 6; + _bkt.dbl = grpc_stats_table_0[bucket]; + bucket -= (_val.uint < _bkt.uint); + GRPC_STATS_INC_HISTOGRAM((exec_ctx), GRPC_STATS_HISTOGRAM_CALL_INITIAL_SIZE, + bucket); + return; + } + GRPC_STATS_INC_HISTOGRAM((exec_ctx), GRPC_STATS_HISTOGRAM_CALL_INITIAL_SIZE, + grpc_stats_histo_find_bucket_slow( + (exec_ctx), value, grpc_stats_table_0, 64)); +} +void grpc_stats_inc_poll_events_returned(grpc_exec_ctx *exec_ctx, int value) { + value = GPR_CLAMP(value, 0, 1024); + if (value < 29) { + GRPC_STATS_INC_HISTOGRAM((exec_ctx), + GRPC_STATS_HISTOGRAM_POLL_EVENTS_RETURNED, value); + return; + } + union { + double dbl; + uint64_t uint; + } _val, _bkt; + _val.dbl = value; + if (_val.uint < 4642789003353915392ull) { + int bucket = + grpc_stats_table_3[((_val.uint - 4628855992006737920ull) >> 47)] + 29; + _bkt.dbl = grpc_stats_table_2[bucket]; + bucket -= (_val.uint < _bkt.uint); + GRPC_STATS_INC_HISTOGRAM((exec_ctx), + GRPC_STATS_HISTOGRAM_POLL_EVENTS_RETURNED, bucket); + return; + } + GRPC_STATS_INC_HISTOGRAM((exec_ctx), + GRPC_STATS_HISTOGRAM_POLL_EVENTS_RETURNED, + grpc_stats_histo_find_bucket_slow( + (exec_ctx), value, grpc_stats_table_2, 128)); +} void grpc_stats_inc_tcp_write_size(grpc_exec_ctx *exec_ctx, int value) { value = GPR_CLAMP(value, 0, 16777216); if (value < 5) { @@ -176,8 +283,8 @@ void grpc_stats_inc_tcp_write_size(grpc_exec_ctx *exec_ctx, int value) { _val.dbl = value; if (_val.uint < 4683743612465315840ull) { int bucket = - grpc_stats_table_1[((_val.uint - 4617315517961601024ull) >> 50)] + 5; - _bkt.dbl = grpc_stats_table_0[bucket]; + grpc_stats_table_5[((_val.uint - 4617315517961601024ull) >> 50)] + 5; + _bkt.dbl = grpc_stats_table_4[bucket]; bucket -= (_val.uint < _bkt.uint); GRPC_STATS_INC_HISTOGRAM((exec_ctx), GRPC_STATS_HISTOGRAM_TCP_WRITE_SIZE, bucket); @@ -185,7 +292,7 @@ void grpc_stats_inc_tcp_write_size(grpc_exec_ctx *exec_ctx, int value) { } GRPC_STATS_INC_HISTOGRAM((exec_ctx), GRPC_STATS_HISTOGRAM_TCP_WRITE_SIZE, grpc_stats_histo_find_bucket_slow( - (exec_ctx), value, grpc_stats_table_0, 64)); + (exec_ctx), value, grpc_stats_table_4, 64)); } void grpc_stats_inc_tcp_write_iov_size(grpc_exec_ctx *exec_ctx, int value) { value = GPR_CLAMP(value, 0, 1024); @@ -201,8 +308,8 @@ void grpc_stats_inc_tcp_write_iov_size(grpc_exec_ctx *exec_ctx, int value) { _val.dbl = value; if (_val.uint < 4637863191261478912ull) { int bucket = - grpc_stats_table_3[((_val.uint - 4623507967449235456ull) >> 48)] + 13; - _bkt.dbl = grpc_stats_table_2[bucket]; + grpc_stats_table_7[((_val.uint - 4623507967449235456ull) >> 48)] + 13; + _bkt.dbl = grpc_stats_table_6[bucket]; bucket -= (_val.uint < _bkt.uint); GRPC_STATS_INC_HISTOGRAM((exec_ctx), GRPC_STATS_HISTOGRAM_TCP_WRITE_IOV_SIZE, bucket); @@ -210,7 +317,7 @@ void grpc_stats_inc_tcp_write_iov_size(grpc_exec_ctx *exec_ctx, int value) { } GRPC_STATS_INC_HISTOGRAM((exec_ctx), GRPC_STATS_HISTOGRAM_TCP_WRITE_IOV_SIZE, grpc_stats_histo_find_bucket_slow( - (exec_ctx), value, grpc_stats_table_2, 64)); + (exec_ctx), value, grpc_stats_table_6, 64)); } void grpc_stats_inc_tcp_read_size(grpc_exec_ctx *exec_ctx, int value) { value = GPR_CLAMP(value, 0, 16777216); @@ -226,8 +333,8 @@ void grpc_stats_inc_tcp_read_size(grpc_exec_ctx *exec_ctx, int value) { _val.dbl = value; if (_val.uint < 4683743612465315840ull) { int bucket = - grpc_stats_table_1[((_val.uint - 4617315517961601024ull) >> 50)] + 5; - _bkt.dbl = grpc_stats_table_0[bucket]; + grpc_stats_table_5[((_val.uint - 4617315517961601024ull) >> 50)] + 5; + _bkt.dbl = grpc_stats_table_4[bucket]; bucket -= (_val.uint < _bkt.uint); GRPC_STATS_INC_HISTOGRAM((exec_ctx), GRPC_STATS_HISTOGRAM_TCP_READ_SIZE, bucket); @@ -235,7 +342,7 @@ void grpc_stats_inc_tcp_read_size(grpc_exec_ctx *exec_ctx, int value) { } GRPC_STATS_INC_HISTOGRAM((exec_ctx), GRPC_STATS_HISTOGRAM_TCP_READ_SIZE, grpc_stats_histo_find_bucket_slow( - (exec_ctx), value, grpc_stats_table_0, 64)); + (exec_ctx), value, grpc_stats_table_4, 64)); } void grpc_stats_inc_tcp_read_offer(grpc_exec_ctx *exec_ctx, int value) { value = GPR_CLAMP(value, 0, 16777216); @@ -251,8 +358,8 @@ void grpc_stats_inc_tcp_read_offer(grpc_exec_ctx *exec_ctx, int value) { _val.dbl = value; if (_val.uint < 4683743612465315840ull) { int bucket = - grpc_stats_table_1[((_val.uint - 4617315517961601024ull) >> 50)] + 5; - _bkt.dbl = grpc_stats_table_0[bucket]; + grpc_stats_table_5[((_val.uint - 4617315517961601024ull) >> 50)] + 5; + _bkt.dbl = grpc_stats_table_4[bucket]; bucket -= (_val.uint < _bkt.uint); GRPC_STATS_INC_HISTOGRAM((exec_ctx), GRPC_STATS_HISTOGRAM_TCP_READ_OFFER, bucket); @@ -260,7 +367,7 @@ void grpc_stats_inc_tcp_read_offer(grpc_exec_ctx *exec_ctx, int value) { } GRPC_STATS_INC_HISTOGRAM((exec_ctx), GRPC_STATS_HISTOGRAM_TCP_READ_OFFER, grpc_stats_histo_find_bucket_slow( - (exec_ctx), value, grpc_stats_table_0, 64)); + (exec_ctx), value, grpc_stats_table_4, 64)); } void grpc_stats_inc_tcp_read_offer_iov_size(grpc_exec_ctx *exec_ctx, int value) { @@ -277,8 +384,8 @@ void grpc_stats_inc_tcp_read_offer_iov_size(grpc_exec_ctx *exec_ctx, _val.dbl = value; if (_val.uint < 4637863191261478912ull) { int bucket = - grpc_stats_table_3[((_val.uint - 4623507967449235456ull) >> 48)] + 13; - _bkt.dbl = grpc_stats_table_2[bucket]; + grpc_stats_table_7[((_val.uint - 4623507967449235456ull) >> 48)] + 13; + _bkt.dbl = grpc_stats_table_6[bucket]; bucket -= (_val.uint < _bkt.uint); GRPC_STATS_INC_HISTOGRAM( (exec_ctx), GRPC_STATS_HISTOGRAM_TCP_READ_OFFER_IOV_SIZE, bucket); @@ -287,7 +394,7 @@ void grpc_stats_inc_tcp_read_offer_iov_size(grpc_exec_ctx *exec_ctx, GRPC_STATS_INC_HISTOGRAM((exec_ctx), GRPC_STATS_HISTOGRAM_TCP_READ_OFFER_IOV_SIZE, grpc_stats_histo_find_bucket_slow( - (exec_ctx), value, grpc_stats_table_2, 64)); + (exec_ctx), value, grpc_stats_table_6, 64)); } void grpc_stats_inc_http2_send_message_size(grpc_exec_ctx *exec_ctx, int value) { @@ -304,8 +411,8 @@ void grpc_stats_inc_http2_send_message_size(grpc_exec_ctx *exec_ctx, _val.dbl = value; if (_val.uint < 4683743612465315840ull) { int bucket = - grpc_stats_table_1[((_val.uint - 4617315517961601024ull) >> 50)] + 5; - _bkt.dbl = grpc_stats_table_0[bucket]; + grpc_stats_table_5[((_val.uint - 4617315517961601024ull) >> 50)] + 5; + _bkt.dbl = grpc_stats_table_4[bucket]; bucket -= (_val.uint < _bkt.uint); GRPC_STATS_INC_HISTOGRAM( (exec_ctx), GRPC_STATS_HISTOGRAM_HTTP2_SEND_MESSAGE_SIZE, bucket); @@ -314,7 +421,7 @@ void grpc_stats_inc_http2_send_message_size(grpc_exec_ctx *exec_ctx, GRPC_STATS_INC_HISTOGRAM((exec_ctx), GRPC_STATS_HISTOGRAM_HTTP2_SEND_MESSAGE_SIZE, grpc_stats_histo_find_bucket_slow( - (exec_ctx), value, grpc_stats_table_0, 64)); + (exec_ctx), value, grpc_stats_table_4, 64)); } void grpc_stats_inc_http2_send_initial_metadata_per_write( grpc_exec_ctx *exec_ctx, int value) { @@ -332,8 +439,8 @@ void grpc_stats_inc_http2_send_initial_metadata_per_write( _val.dbl = value; if (_val.uint < 4637863191261478912ull) { int bucket = - grpc_stats_table_3[((_val.uint - 4623507967449235456ull) >> 48)] + 13; - _bkt.dbl = grpc_stats_table_2[bucket]; + grpc_stats_table_7[((_val.uint - 4623507967449235456ull) >> 48)] + 13; + _bkt.dbl = grpc_stats_table_6[bucket]; bucket -= (_val.uint < _bkt.uint); GRPC_STATS_INC_HISTOGRAM( (exec_ctx), GRPC_STATS_HISTOGRAM_HTTP2_SEND_INITIAL_METADATA_PER_WRITE, @@ -342,7 +449,7 @@ void grpc_stats_inc_http2_send_initial_metadata_per_write( } GRPC_STATS_INC_HISTOGRAM( (exec_ctx), GRPC_STATS_HISTOGRAM_HTTP2_SEND_INITIAL_METADATA_PER_WRITE, - grpc_stats_histo_find_bucket_slow((exec_ctx), value, grpc_stats_table_2, + grpc_stats_histo_find_bucket_slow((exec_ctx), value, grpc_stats_table_6, 64)); } void grpc_stats_inc_http2_send_message_per_write(grpc_exec_ctx *exec_ctx, @@ -360,8 +467,8 @@ void grpc_stats_inc_http2_send_message_per_write(grpc_exec_ctx *exec_ctx, _val.dbl = value; if (_val.uint < 4637863191261478912ull) { int bucket = - grpc_stats_table_3[((_val.uint - 4623507967449235456ull) >> 48)] + 13; - _bkt.dbl = grpc_stats_table_2[bucket]; + grpc_stats_table_7[((_val.uint - 4623507967449235456ull) >> 48)] + 13; + _bkt.dbl = grpc_stats_table_6[bucket]; bucket -= (_val.uint < _bkt.uint); GRPC_STATS_INC_HISTOGRAM( (exec_ctx), GRPC_STATS_HISTOGRAM_HTTP2_SEND_MESSAGE_PER_WRITE, bucket); @@ -370,7 +477,7 @@ void grpc_stats_inc_http2_send_message_per_write(grpc_exec_ctx *exec_ctx, GRPC_STATS_INC_HISTOGRAM((exec_ctx), GRPC_STATS_HISTOGRAM_HTTP2_SEND_MESSAGE_PER_WRITE, grpc_stats_histo_find_bucket_slow( - (exec_ctx), value, grpc_stats_table_2, 64)); + (exec_ctx), value, grpc_stats_table_6, 64)); } void grpc_stats_inc_http2_send_trailing_metadata_per_write( grpc_exec_ctx *exec_ctx, int value) { @@ -388,8 +495,8 @@ void grpc_stats_inc_http2_send_trailing_metadata_per_write( _val.dbl = value; if (_val.uint < 4637863191261478912ull) { int bucket = - grpc_stats_table_3[((_val.uint - 4623507967449235456ull) >> 48)] + 13; - _bkt.dbl = grpc_stats_table_2[bucket]; + grpc_stats_table_7[((_val.uint - 4623507967449235456ull) >> 48)] + 13; + _bkt.dbl = grpc_stats_table_6[bucket]; bucket -= (_val.uint < _bkt.uint); GRPC_STATS_INC_HISTOGRAM( (exec_ctx), GRPC_STATS_HISTOGRAM_HTTP2_SEND_TRAILING_METADATA_PER_WRITE, @@ -398,7 +505,7 @@ void grpc_stats_inc_http2_send_trailing_metadata_per_write( } GRPC_STATS_INC_HISTOGRAM( (exec_ctx), GRPC_STATS_HISTOGRAM_HTTP2_SEND_TRAILING_METADATA_PER_WRITE, - grpc_stats_histo_find_bucket_slow((exec_ctx), value, grpc_stats_table_2, + grpc_stats_histo_find_bucket_slow((exec_ctx), value, grpc_stats_table_6, 64)); } void grpc_stats_inc_http2_send_flowctl_per_write(grpc_exec_ctx *exec_ctx, @@ -416,8 +523,8 @@ void grpc_stats_inc_http2_send_flowctl_per_write(grpc_exec_ctx *exec_ctx, _val.dbl = value; if (_val.uint < 4637863191261478912ull) { int bucket = - grpc_stats_table_3[((_val.uint - 4623507967449235456ull) >> 48)] + 13; - _bkt.dbl = grpc_stats_table_2[bucket]; + grpc_stats_table_7[((_val.uint - 4623507967449235456ull) >> 48)] + 13; + _bkt.dbl = grpc_stats_table_6[bucket]; bucket -= (_val.uint < _bkt.uint); GRPC_STATS_INC_HISTOGRAM( (exec_ctx), GRPC_STATS_HISTOGRAM_HTTP2_SEND_FLOWCTL_PER_WRITE, bucket); @@ -426,7 +533,7 @@ void grpc_stats_inc_http2_send_flowctl_per_write(grpc_exec_ctx *exec_ctx, GRPC_STATS_INC_HISTOGRAM((exec_ctx), GRPC_STATS_HISTOGRAM_HTTP2_SEND_FLOWCTL_PER_WRITE, grpc_stats_histo_find_bucket_slow( - (exec_ctx), value, grpc_stats_table_2, 64)); + (exec_ctx), value, grpc_stats_table_6, 64)); } void grpc_stats_inc_server_cqs_checked(grpc_exec_ctx *exec_ctx, int value) { value = GPR_CLAMP(value, 0, 64); @@ -442,8 +549,8 @@ void grpc_stats_inc_server_cqs_checked(grpc_exec_ctx *exec_ctx, int value) { _val.dbl = value; if (_val.uint < 4625196817309499392ull) { int bucket = - grpc_stats_table_5[((_val.uint - 4613937818241073152ull) >> 51)] + 3; - _bkt.dbl = grpc_stats_table_4[bucket]; + grpc_stats_table_9[((_val.uint - 4613937818241073152ull) >> 51)] + 3; + _bkt.dbl = grpc_stats_table_8[bucket]; bucket -= (_val.uint < _bkt.uint); GRPC_STATS_INC_HISTOGRAM((exec_ctx), GRPC_STATS_HISTOGRAM_SERVER_CQS_CHECKED, bucket); @@ -451,18 +558,21 @@ void grpc_stats_inc_server_cqs_checked(grpc_exec_ctx *exec_ctx, int value) { } GRPC_STATS_INC_HISTOGRAM((exec_ctx), GRPC_STATS_HISTOGRAM_SERVER_CQS_CHECKED, grpc_stats_histo_find_bucket_slow( - (exec_ctx), value, grpc_stats_table_4, 8)); + (exec_ctx), value, grpc_stats_table_8, 8)); } -const int grpc_stats_histo_buckets[11] = {64, 64, 64, 64, 64, 64, - 64, 64, 64, 64, 8}; -const int grpc_stats_histo_start[11] = {0, 64, 128, 192, 256, 320, - 384, 448, 512, 576, 640}; -const int *const grpc_stats_histo_bucket_boundaries[11] = { - grpc_stats_table_0, grpc_stats_table_2, grpc_stats_table_0, - grpc_stats_table_0, grpc_stats_table_2, grpc_stats_table_0, - grpc_stats_table_2, grpc_stats_table_2, grpc_stats_table_2, - grpc_stats_table_2, grpc_stats_table_4}; -void (*const grpc_stats_inc_histogram[11])(grpc_exec_ctx *exec_ctx, int x) = { +const int grpc_stats_histo_buckets[13] = {64, 128, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 8}; +const int grpc_stats_histo_start[13] = {0, 64, 192, 256, 320, 384, 448, + 512, 576, 640, 704, 768, 832}; +const int *const grpc_stats_histo_bucket_boundaries[13] = { + grpc_stats_table_0, grpc_stats_table_2, grpc_stats_table_4, + grpc_stats_table_6, grpc_stats_table_4, grpc_stats_table_4, + grpc_stats_table_6, grpc_stats_table_4, grpc_stats_table_6, + grpc_stats_table_6, grpc_stats_table_6, grpc_stats_table_6, + grpc_stats_table_8}; +void (*const grpc_stats_inc_histogram[13])(grpc_exec_ctx *exec_ctx, int x) = { + grpc_stats_inc_call_initial_size, + grpc_stats_inc_poll_events_returned, grpc_stats_inc_tcp_write_size, grpc_stats_inc_tcp_write_iov_size, grpc_stats_inc_tcp_read_size, diff --git a/src/core/lib/debug/stats_data.h b/src/core/lib/debug/stats_data.h index 479c9520b6..48f60fc8c9 100644 --- a/src/core/lib/debug/stats_data.h +++ b/src/core/lib/debug/stats_data.h @@ -29,6 +29,12 @@ typedef enum { GRPC_STATS_COUNTER_SERVER_CALLS_CREATED, GRPC_STATS_COUNTER_SYSCALL_POLL, GRPC_STATS_COUNTER_SYSCALL_WAIT, + GRPC_STATS_COUNTER_POLLSET_KICK, + GRPC_STATS_COUNTER_POLLSET_KICKED_WITHOUT_POLLER, + GRPC_STATS_COUNTER_POLLSET_KICKED_AGAIN, + GRPC_STATS_COUNTER_POLLSET_KICK_WAKEUP_FD, + GRPC_STATS_COUNTER_POLLSET_KICK_WAKEUP_CV, + GRPC_STATS_COUNTER_POLLSET_KICK_OWN_THREAD, GRPC_STATS_COUNTER_HISTOGRAM_SLOW_LOOKUPS, GRPC_STATS_COUNTER_SYSCALL_WRITE, GRPC_STATS_COUNTER_SYSCALL_READ, @@ -65,6 +71,8 @@ typedef enum { extern const char *grpc_stats_counter_name[GRPC_STATS_COUNTER_COUNT]; extern const char *grpc_stats_counter_doc[GRPC_STATS_COUNTER_COUNT]; typedef enum { + GRPC_STATS_HISTOGRAM_CALL_INITIAL_SIZE, + GRPC_STATS_HISTOGRAM_POLL_EVENTS_RETURNED, GRPC_STATS_HISTOGRAM_TCP_WRITE_SIZE, GRPC_STATS_HISTOGRAM_TCP_WRITE_IOV_SIZE, GRPC_STATS_HISTOGRAM_TCP_READ_SIZE, @@ -81,29 +89,33 @@ typedef enum { extern const char *grpc_stats_histogram_name[GRPC_STATS_HISTOGRAM_COUNT]; extern const char *grpc_stats_histogram_doc[GRPC_STATS_HISTOGRAM_COUNT]; typedef enum { - GRPC_STATS_HISTOGRAM_TCP_WRITE_SIZE_FIRST_SLOT = 0, + GRPC_STATS_HISTOGRAM_CALL_INITIAL_SIZE_FIRST_SLOT = 0, + GRPC_STATS_HISTOGRAM_CALL_INITIAL_SIZE_BUCKETS = 64, + GRPC_STATS_HISTOGRAM_POLL_EVENTS_RETURNED_FIRST_SLOT = 64, + GRPC_STATS_HISTOGRAM_POLL_EVENTS_RETURNED_BUCKETS = 128, + GRPC_STATS_HISTOGRAM_TCP_WRITE_SIZE_FIRST_SLOT = 192, GRPC_STATS_HISTOGRAM_TCP_WRITE_SIZE_BUCKETS = 64, - GRPC_STATS_HISTOGRAM_TCP_WRITE_IOV_SIZE_FIRST_SLOT = 64, + GRPC_STATS_HISTOGRAM_TCP_WRITE_IOV_SIZE_FIRST_SLOT = 256, GRPC_STATS_HISTOGRAM_TCP_WRITE_IOV_SIZE_BUCKETS = 64, - GRPC_STATS_HISTOGRAM_TCP_READ_SIZE_FIRST_SLOT = 128, + GRPC_STATS_HISTOGRAM_TCP_READ_SIZE_FIRST_SLOT = 320, GRPC_STATS_HISTOGRAM_TCP_READ_SIZE_BUCKETS = 64, - GRPC_STATS_HISTOGRAM_TCP_READ_OFFER_FIRST_SLOT = 192, + GRPC_STATS_HISTOGRAM_TCP_READ_OFFER_FIRST_SLOT = 384, GRPC_STATS_HISTOGRAM_TCP_READ_OFFER_BUCKETS = 64, - GRPC_STATS_HISTOGRAM_TCP_READ_OFFER_IOV_SIZE_FIRST_SLOT = 256, + GRPC_STATS_HISTOGRAM_TCP_READ_OFFER_IOV_SIZE_FIRST_SLOT = 448, GRPC_STATS_HISTOGRAM_TCP_READ_OFFER_IOV_SIZE_BUCKETS = 64, - GRPC_STATS_HISTOGRAM_HTTP2_SEND_MESSAGE_SIZE_FIRST_SLOT = 320, + GRPC_STATS_HISTOGRAM_HTTP2_SEND_MESSAGE_SIZE_FIRST_SLOT = 512, GRPC_STATS_HISTOGRAM_HTTP2_SEND_MESSAGE_SIZE_BUCKETS = 64, - GRPC_STATS_HISTOGRAM_HTTP2_SEND_INITIAL_METADATA_PER_WRITE_FIRST_SLOT = 384, + GRPC_STATS_HISTOGRAM_HTTP2_SEND_INITIAL_METADATA_PER_WRITE_FIRST_SLOT = 576, GRPC_STATS_HISTOGRAM_HTTP2_SEND_INITIAL_METADATA_PER_WRITE_BUCKETS = 64, - GRPC_STATS_HISTOGRAM_HTTP2_SEND_MESSAGE_PER_WRITE_FIRST_SLOT = 448, + GRPC_STATS_HISTOGRAM_HTTP2_SEND_MESSAGE_PER_WRITE_FIRST_SLOT = 640, GRPC_STATS_HISTOGRAM_HTTP2_SEND_MESSAGE_PER_WRITE_BUCKETS = 64, - GRPC_STATS_HISTOGRAM_HTTP2_SEND_TRAILING_METADATA_PER_WRITE_FIRST_SLOT = 512, + GRPC_STATS_HISTOGRAM_HTTP2_SEND_TRAILING_METADATA_PER_WRITE_FIRST_SLOT = 704, GRPC_STATS_HISTOGRAM_HTTP2_SEND_TRAILING_METADATA_PER_WRITE_BUCKETS = 64, - GRPC_STATS_HISTOGRAM_HTTP2_SEND_FLOWCTL_PER_WRITE_FIRST_SLOT = 576, + GRPC_STATS_HISTOGRAM_HTTP2_SEND_FLOWCTL_PER_WRITE_FIRST_SLOT = 768, GRPC_STATS_HISTOGRAM_HTTP2_SEND_FLOWCTL_PER_WRITE_BUCKETS = 64, - GRPC_STATS_HISTOGRAM_SERVER_CQS_CHECKED_FIRST_SLOT = 640, + GRPC_STATS_HISTOGRAM_SERVER_CQS_CHECKED_FIRST_SLOT = 832, GRPC_STATS_HISTOGRAM_SERVER_CQS_CHECKED_BUCKETS = 8, - GRPC_STATS_HISTOGRAM_BUCKETS = 648 + GRPC_STATS_HISTOGRAM_BUCKETS = 840 } grpc_stats_histogram_constants; #define GRPC_STATS_INC_CLIENT_CALLS_CREATED(exec_ctx) \ GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_CLIENT_CALLS_CREATED) @@ -113,6 +125,19 @@ typedef enum { GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_SYSCALL_POLL) #define GRPC_STATS_INC_SYSCALL_WAIT(exec_ctx) \ GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_SYSCALL_WAIT) +#define GRPC_STATS_INC_POLLSET_KICK(exec_ctx) \ + GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_POLLSET_KICK) +#define GRPC_STATS_INC_POLLSET_KICKED_WITHOUT_POLLER(exec_ctx) \ + GRPC_STATS_INC_COUNTER((exec_ctx), \ + GRPC_STATS_COUNTER_POLLSET_KICKED_WITHOUT_POLLER) +#define GRPC_STATS_INC_POLLSET_KICKED_AGAIN(exec_ctx) \ + GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_POLLSET_KICKED_AGAIN) +#define GRPC_STATS_INC_POLLSET_KICK_WAKEUP_FD(exec_ctx) \ + GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_POLLSET_KICK_WAKEUP_FD) +#define GRPC_STATS_INC_POLLSET_KICK_WAKEUP_CV(exec_ctx) \ + GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_POLLSET_KICK_WAKEUP_CV) +#define GRPC_STATS_INC_POLLSET_KICK_OWN_THREAD(exec_ctx) \ + GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_POLLSET_KICK_OWN_THREAD) #define GRPC_STATS_INC_HISTOGRAM_SLOW_LOOKUPS(exec_ctx) \ GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_HISTOGRAM_SLOW_LOOKUPS) #define GRPC_STATS_INC_SYSCALL_WRITE(exec_ctx) \ @@ -189,6 +214,12 @@ typedef enum { #define GRPC_STATS_INC_SERVER_SLOWPATH_REQUESTS_QUEUED(exec_ctx) \ GRPC_STATS_INC_COUNTER((exec_ctx), \ GRPC_STATS_COUNTER_SERVER_SLOWPATH_REQUESTS_QUEUED) +#define GRPC_STATS_INC_CALL_INITIAL_SIZE(exec_ctx, value) \ + grpc_stats_inc_call_initial_size((exec_ctx), (int)(value)) +void grpc_stats_inc_call_initial_size(grpc_exec_ctx *exec_ctx, int x); +#define GRPC_STATS_INC_POLL_EVENTS_RETURNED(exec_ctx, value) \ + grpc_stats_inc_poll_events_returned((exec_ctx), (int)(value)) +void grpc_stats_inc_poll_events_returned(grpc_exec_ctx *exec_ctx, int x); #define GRPC_STATS_INC_TCP_WRITE_SIZE(exec_ctx, value) \ grpc_stats_inc_tcp_write_size((exec_ctx), (int)(value)) void grpc_stats_inc_tcp_write_size(grpc_exec_ctx *exec_ctx, int x); @@ -227,10 +258,10 @@ void grpc_stats_inc_http2_send_flowctl_per_write(grpc_exec_ctx *exec_ctx, #define GRPC_STATS_INC_SERVER_CQS_CHECKED(exec_ctx, value) \ grpc_stats_inc_server_cqs_checked((exec_ctx), (int)(value)) void grpc_stats_inc_server_cqs_checked(grpc_exec_ctx *exec_ctx, int x); -extern const int grpc_stats_histo_buckets[11]; -extern const int grpc_stats_histo_start[11]; -extern const int *const grpc_stats_histo_bucket_boundaries[11]; -extern void (*const grpc_stats_inc_histogram[11])(grpc_exec_ctx *exec_ctx, +extern const int grpc_stats_histo_buckets[13]; +extern const int grpc_stats_histo_start[13]; +extern const int *const grpc_stats_histo_bucket_boundaries[13]; +extern void (*const grpc_stats_inc_histogram[13])(grpc_exec_ctx *exec_ctx, int x); #endif /* GRPC_CORE_LIB_DEBUG_STATS_DATA_H */ diff --git a/src/core/lib/debug/stats_data.yaml b/src/core/lib/debug/stats_data.yaml index 7cf82de96c..26c8e1b11d 100644 --- a/src/core/lib/debug/stats_data.yaml +++ b/src/core/lib/debug/stats_data.yaml @@ -20,11 +20,35 @@ doc: Number of client side calls created by this process - counter: server_calls_created doc: Number of server side calls created by this process +- histogram: call_initial_size + max: 262144 + buckets: 64 + doc: Initial size of the grpc_call arena created at call start # polling - counter: syscall_poll doc: Number of polling syscalls (epoll_wait, poll, etc) made by this process - counter: syscall_wait doc: Number of sleeping syscalls made by this process +- histogram: poll_events_returned + max: 1024 + buckets: 128 + doc: How many events are called for each syscall_poll +- counter: pollset_kick + doc: How many polling wakeups were performed by the process +- counter: pollset_kicked_without_poller + doc: How many times was a polling wakeup requested without an active poller +- counter: pollset_kicked_again + doc: How many times was the same polling worker awoken repeatedly before + waking up +- counter: pollset_kick_wakeup_fd + doc: How many times was an eventfd used as the wakeup vector for a polling + wakeup +- counter: pollset_kick_wakeup_cv + doc: How many times was a condition variable used as the wakeup vector for a + polling wakeup +- counter: pollset_kick_own_thread + doc: How many times could a polling wakeup be satisfied by keeping the waking + thread awake? # stats system - counter: histogram_slow_lookups doc: Number of times histogram increments went through the slow diff --git a/src/core/lib/debug/stats_data_bq_schema.sql b/src/core/lib/debug/stats_data_bq_schema.sql index 7291bbf070..70088e5bae 100644 --- a/src/core/lib/debug/stats_data_bq_schema.sql +++ b/src/core/lib/debug/stats_data_bq_schema.sql @@ -2,6 +2,12 @@ client_calls_created_per_iteration:INTEGER, server_calls_created_per_iteration:INTEGER, syscall_poll_per_iteration:INTEGER, syscall_wait_per_iteration:INTEGER, +pollset_kick_per_iteration:INTEGER, +pollset_kicked_without_poller_per_iteration:INTEGER, +pollset_kicked_again_per_iteration:INTEGER, +pollset_kick_wakeup_fd_per_iteration:INTEGER, +pollset_kick_wakeup_cv_per_iteration:INTEGER, +pollset_kick_own_thread_per_iteration:INTEGER, histogram_slow_lookups_per_iteration:INTEGER, syscall_write_per_iteration:INTEGER, syscall_read_per_iteration:INTEGER, diff --git a/src/core/lib/iomgr/ev_epoll1_linux.c b/src/core/lib/iomgr/ev_epoll1_linux.c index 01d28220c7..bf080fff0c 100644 --- a/src/core/lib/iomgr/ev_epoll1_linux.c +++ b/src/core/lib/iomgr/ev_epoll1_linux.c @@ -502,22 +502,27 @@ static void pollset_destroy(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset) { gpr_mu_destroy(&pollset->mu); } -static grpc_error *pollset_kick_all(grpc_pollset *pollset) { +static grpc_error *pollset_kick_all(grpc_exec_ctx *exec_ctx, + grpc_pollset *pollset) { GPR_TIMER_BEGIN("pollset_kick_all", 0); grpc_error *error = GRPC_ERROR_NONE; if (pollset->root_worker != NULL) { grpc_pollset_worker *worker = pollset->root_worker; do { + GRPC_STATS_INC_POLLSET_KICK(exec_ctx); switch (worker->kick_state) { case KICKED: + GRPC_STATS_INC_POLLSET_KICKED_AGAIN(exec_ctx); break; case UNKICKED: + GRPC_STATS_INC_POLLSET_KICK_WAKEUP_CV(exec_ctx); SET_KICK_STATE(worker, KICKED); if (worker->initialized_cv) { gpr_cv_signal(&worker->cv); } break; case DESIGNATED_POLLER: + GRPC_STATS_INC_POLLSET_KICK_WAKEUP_FD(exec_ctx); SET_KICK_STATE(worker, KICKED); append_error(&error, grpc_wakeup_fd_wakeup(&global_wakeup_fd), "pollset_kick_all"); @@ -550,7 +555,7 @@ static void pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, GPR_ASSERT(!pollset->shutting_down); pollset->shutdown_closure = closure; pollset->shutting_down = true; - GRPC_LOG_IF_ERROR("pollset_shutdown", pollset_kick_all(pollset)); + GRPC_LOG_IF_ERROR("pollset_shutdown", pollset_kick_all(exec_ctx, pollset)); pollset_maybe_finish_shutdown(exec_ctx, pollset); GPR_TIMER_END("pollset_shutdown", 0); } @@ -646,6 +651,8 @@ static grpc_error *do_epoll_wait(grpc_exec_ctx *exec_ctx, grpc_pollset *ps, if (r < 0) return GRPC_OS_ERROR(errno, "epoll_wait"); + GRPC_STATS_INC_POLL_EVENTS_RETURNED(exec_ctx, r); + if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_DEBUG, "ps: %p poll got %d events", ps, r); } @@ -979,9 +986,10 @@ static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *ps, return error; } -static grpc_error *pollset_kick(grpc_pollset *pollset, +static grpc_error *pollset_kick(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_pollset_worker *specific_worker) { GPR_TIMER_BEGIN("pollset_kick", 0); + GRPC_STATS_INC_POLLSET_KICK(exec_ctx); grpc_error *ret_err = GRPC_ERROR_NONE; if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_strvec log; @@ -1014,6 +1022,7 @@ static grpc_error *pollset_kick(grpc_pollset *pollset, if (gpr_tls_get(&g_current_thread_pollset) != (intptr_t)pollset) { grpc_pollset_worker *root_worker = pollset->root_worker; if (root_worker == NULL) { + GRPC_STATS_INC_POLLSET_KICKED_WITHOUT_POLLER(exec_ctx); pollset->kicked_without_poller = true; if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_ERROR, " .. kicked_without_poller"); @@ -1022,12 +1031,14 @@ static grpc_error *pollset_kick(grpc_pollset *pollset, } grpc_pollset_worker *next_worker = root_worker->next; if (root_worker->kick_state == KICKED) { + GRPC_STATS_INC_POLLSET_KICKED_AGAIN(exec_ctx); if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_ERROR, " .. already kicked %p", root_worker); } SET_KICK_STATE(root_worker, KICKED); goto done; } else if (next_worker->kick_state == KICKED) { + GRPC_STATS_INC_POLLSET_KICKED_AGAIN(exec_ctx); if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_ERROR, " .. already kicked %p", next_worker); } @@ -1038,6 +1049,7 @@ static grpc_error *pollset_kick(grpc_pollset *pollset, // there is no next worker root_worker == (grpc_pollset_worker *)gpr_atm_no_barrier_load( &g_active_poller)) { + GRPC_STATS_INC_POLLSET_KICK_WAKEUP_FD(exec_ctx); if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_ERROR, " .. kicked %p", root_worker); } @@ -1045,6 +1057,7 @@ static grpc_error *pollset_kick(grpc_pollset *pollset, ret_err = grpc_wakeup_fd_wakeup(&global_wakeup_fd); goto done; } else if (next_worker->kick_state == UNKICKED) { + GRPC_STATS_INC_POLLSET_KICK_WAKEUP_CV(exec_ctx); if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_ERROR, " .. kicked %p", next_worker); } @@ -1062,10 +1075,12 @@ static grpc_error *pollset_kick(grpc_pollset *pollset, } SET_KICK_STATE(root_worker, KICKED); if (root_worker->initialized_cv) { + GRPC_STATS_INC_POLLSET_KICK_WAKEUP_CV(exec_ctx); gpr_cv_signal(&root_worker->cv); } goto done; } else { + GRPC_STATS_INC_POLLSET_KICK_WAKEUP_FD(exec_ctx); if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_ERROR, " .. non-root poller %p (root=%p)", next_worker, root_worker); @@ -1075,11 +1090,13 @@ static grpc_error *pollset_kick(grpc_pollset *pollset, goto done; } } else { + GRPC_STATS_INC_POLLSET_KICKED_AGAIN(exec_ctx); GPR_ASSERT(next_worker->kick_state == KICKED); SET_KICK_STATE(next_worker, KICKED); goto done; } } else { + GRPC_STATS_INC_POLLSET_KICKED_AGAIN(exec_ctx); if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_ERROR, " .. kicked while waking up"); } @@ -1096,6 +1113,7 @@ static grpc_error *pollset_kick(grpc_pollset *pollset, goto done; } else if (gpr_tls_get(&g_current_thread_worker) == (intptr_t)specific_worker) { + GRPC_STATS_INC_POLLSET_KICK_OWN_THREAD(exec_ctx); if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_ERROR, " .. mark %p kicked", specific_worker); } @@ -1103,6 +1121,7 @@ static grpc_error *pollset_kick(grpc_pollset *pollset, goto done; } else if (specific_worker == (grpc_pollset_worker *)gpr_atm_no_barrier_load(&g_active_poller)) { + GRPC_STATS_INC_POLLSET_KICK_WAKEUP_FD(exec_ctx); if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_ERROR, " .. kick active poller"); } @@ -1110,6 +1129,7 @@ static grpc_error *pollset_kick(grpc_pollset *pollset, ret_err = grpc_wakeup_fd_wakeup(&global_wakeup_fd); goto done; } else if (specific_worker->initialized_cv) { + GRPC_STATS_INC_POLLSET_KICK_WAKEUP_CV(exec_ctx); if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_ERROR, " .. kick waiting worker"); } @@ -1117,6 +1137,7 @@ static grpc_error *pollset_kick(grpc_pollset *pollset, gpr_cv_signal(&specific_worker->cv); goto done; } else { + GRPC_STATS_INC_POLLSET_KICKED_AGAIN(exec_ctx); if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_ERROR, " .. kick non-waiting worker"); } diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index df69025f1a..f1a3b97f4d 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -561,6 +561,7 @@ static void do_kick_all(grpc_exec_ctx *exec_ctx, void *arg, if (pollset->root_worker != NULL) { grpc_pollset_worker *worker = pollset->root_worker; do { + GRPC_STATS_INC_POLLSET_KICK(exec_ctx); if (worker->pollable != &pollset->pollable) { gpr_mu_lock(&worker->pollable->po.mu); } @@ -599,7 +600,8 @@ static void pollset_kick_all(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset) { GRPC_ERROR_NONE); } -static grpc_error *pollset_kick_inner(grpc_pollset *pollset, pollable_t *p, +static grpc_error *pollset_kick_inner(grpc_exec_ctx *exec_ctx, + grpc_pollset *pollset, pollable_t *p, grpc_pollset_worker *specific_worker) { if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_DEBUG, @@ -662,13 +664,18 @@ static grpc_error *pollset_kick_inner(grpc_pollset *pollset, pollable_t *p, } /* p->po.mu must be held before calling this function */ -static grpc_error *pollset_kick(grpc_pollset *pollset, +static grpc_error *pollset_kick(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_pollset_worker *specific_worker) { +<<<<<<< HEAD + pollable *p = pollset->current_pollable; + GRPC_STATS_INC_POLLSET_KICK(exec_ctx); +======= pollable_t *p = pollset->current_pollable; +>>>>>>> d1fefe89dacf198a721053f2b1a3e173f1f24948 if (p != &pollset->pollable) { gpr_mu_lock(&p->po.mu); } - grpc_error *error = pollset_kick_inner(pollset, p, specific_worker); + grpc_error *error = pollset_kick_inner(exec_ctx, pollset, p, specific_worker); if (p != &pollset->pollable) { gpr_mu_unlock(&p->po.mu); } diff --git a/src/core/lib/iomgr/ev_epollsig_linux.c b/src/core/lib/iomgr/ev_epollsig_linux.c index b88c3ba111..f5d00cbc8c 100644 --- a/src/core/lib/iomgr/ev_epollsig_linux.c +++ b/src/core/lib/iomgr/ev_epollsig_linux.c @@ -1021,10 +1021,11 @@ static void push_front_worker(grpc_pollset *p, grpc_pollset_worker *worker) { } /* p->mu must be held before calling this function */ -static grpc_error *pollset_kick(grpc_pollset *p, +static grpc_error *pollset_kick(grpc_exec_ctx *exec_ctx, grpc_pollset *p, grpc_pollset_worker *specific_worker) { GPR_TIMER_BEGIN("pollset_kick", 0); grpc_error *error = GRPC_ERROR_NONE; + GRPC_STATS_INC_POLLSET_KICK(exec_ctx); const char *err_desc = "Kick Failure"; grpc_pollset_worker *worker = specific_worker; if (worker != NULL) { @@ -1158,7 +1159,7 @@ static void pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, GPR_ASSERT(!pollset->shutting_down); pollset->shutting_down = true; pollset->shutdown_done = closure; - pollset_kick(pollset, GRPC_POLLSET_KICK_BROADCAST); + pollset_kick(exec_ctx, pollset, GRPC_POLLSET_KICK_BROADCAST); /* If the pollset has any workers, we cannot call finish_shutdown_locked() because it would release the underlying polling island. In such a case, we diff --git a/src/core/lib/iomgr/ev_poll_posix.c b/src/core/lib/iomgr/ev_poll_posix.c index bcf1d9001b..15968d23ba 100644 --- a/src/core/lib/iomgr/ev_poll_posix.c +++ b/src/core/lib/iomgr/ev_poll_posix.c @@ -209,7 +209,7 @@ static int poll_deadline_to_millis_timeout(gpr_timespec deadline, #define GRPC_POLLSET_REEVALUATE_POLLING_ON_WAKEUP 2 /* As per pollset_kick, with an extended set of flags (defined above) -- mostly for fd_posix's use. */ -static grpc_error *pollset_kick_ext(grpc_pollset *p, +static grpc_error *pollset_kick_ext(grpc_exec_ctx *exec_ctx, grpc_pollset *p, grpc_pollset_worker *specific_worker, uint32_t flags) GRPC_MUST_USE_RESULT; @@ -365,36 +365,39 @@ static grpc_pollset *fd_get_read_notifier_pollset(grpc_exec_ctx *exec_ctx, return notifier; } -static grpc_error *pollset_kick_locked(grpc_fd_watcher *watcher) { +static grpc_error *pollset_kick_locked(grpc_exec_ctx *exec_ctx, + grpc_fd_watcher *watcher) { gpr_mu_lock(&watcher->pollset->mu); GPR_ASSERT(watcher->worker); - grpc_error *err = pollset_kick_ext(watcher->pollset, watcher->worker, - GRPC_POLLSET_REEVALUATE_POLLING_ON_WAKEUP); + grpc_error *err = + pollset_kick_ext(exec_ctx, watcher->pollset, watcher->worker, + GRPC_POLLSET_REEVALUATE_POLLING_ON_WAKEUP); gpr_mu_unlock(&watcher->pollset->mu); return err; } -static void maybe_wake_one_watcher_locked(grpc_fd *fd) { +static void maybe_wake_one_watcher_locked(grpc_exec_ctx *exec_ctx, + grpc_fd *fd) { if (fd->inactive_watcher_root.next != &fd->inactive_watcher_root) { - pollset_kick_locked(fd->inactive_watcher_root.next); + pollset_kick_locked(exec_ctx, fd->inactive_watcher_root.next); } else if (fd->read_watcher) { - pollset_kick_locked(fd->read_watcher); + pollset_kick_locked(exec_ctx, fd->read_watcher); } else if (fd->write_watcher) { - pollset_kick_locked(fd->write_watcher); + pollset_kick_locked(exec_ctx, fd->write_watcher); } } -static void wake_all_watchers_locked(grpc_fd *fd) { +static void wake_all_watchers_locked(grpc_exec_ctx *exec_ctx, grpc_fd *fd) { grpc_fd_watcher *watcher; for (watcher = fd->inactive_watcher_root.next; watcher != &fd->inactive_watcher_root; watcher = watcher->next) { - pollset_kick_locked(watcher); + pollset_kick_locked(exec_ctx, watcher); } if (fd->read_watcher) { - pollset_kick_locked(fd->read_watcher); + pollset_kick_locked(exec_ctx, fd->read_watcher); } if (fd->write_watcher && fd->write_watcher != fd->read_watcher) { - pollset_kick_locked(fd->write_watcher); + pollset_kick_locked(exec_ctx, fd->write_watcher); } } @@ -435,7 +438,7 @@ static void fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd, if (!has_watchers(fd)) { close_fd_locked(exec_ctx, fd); } else { - wake_all_watchers_locked(fd); + wake_all_watchers_locked(exec_ctx, fd); } gpr_mu_unlock(&fd->mu); UNREF_BY(fd, 2, reason); /* drop the reference */ @@ -479,7 +482,7 @@ static void notify_on_locked(grpc_exec_ctx *exec_ctx, grpc_fd *fd, /* already ready ==> queue the closure to run immediately */ *st = CLOSURE_NOT_READY; GRPC_CLOSURE_SCHED(exec_ctx, closure, fd_shutdown_error(fd)); - maybe_wake_one_watcher_locked(fd); + maybe_wake_one_watcher_locked(exec_ctx, fd); } else { /* upcallptr was set to a different closure. This is an error! */ gpr_log(GPR_ERROR, @@ -648,7 +651,7 @@ static void fd_end_poll(grpc_exec_ctx *exec_ctx, grpc_fd_watcher *watcher, } } if (kick) { - maybe_wake_one_watcher_locked(fd); + maybe_wake_one_watcher_locked(exec_ctx, fd); } if (fd_is_orphaned(fd) && !has_watchers(fd) && !fd->closed) { close_fd_locked(exec_ctx, fd); @@ -712,11 +715,12 @@ static void kick_append_error(grpc_error **composite, grpc_error *error) { *composite = grpc_error_add_child(*composite, error); } -static grpc_error *pollset_kick_ext(grpc_pollset *p, +static grpc_error *pollset_kick_ext(grpc_exec_ctx *exec_ctx, grpc_pollset *p, grpc_pollset_worker *specific_worker, uint32_t flags) { GPR_TIMER_BEGIN("pollset_kick_ext", 0); grpc_error *error = GRPC_ERROR_NONE; + GRPC_STATS_INC_POLLSET_KICK(exec_ctx); /* pollset->mu already held */ if (specific_worker != NULL) { @@ -782,9 +786,9 @@ static grpc_error *pollset_kick_ext(grpc_pollset *p, return error; } -static grpc_error *pollset_kick(grpc_pollset *p, +static grpc_error *pollset_kick(grpc_exec_ctx *exec_ctx, grpc_pollset *p, grpc_pollset_worker *specific_worker) { - return pollset_kick_ext(p, specific_worker, 0); + return pollset_kick_ext(exec_ctx, p, specific_worker, 0); } /* global state management */ @@ -847,7 +851,7 @@ static void pollset_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, } pollset->fds[pollset->fd_count++] = fd; GRPC_FD_REF(fd, "multipoller"); - pollset_kick(pollset, NULL); + pollset_kick(exec_ctx, pollset, NULL); exit: gpr_mu_unlock(&pollset->mu); } @@ -1083,7 +1087,7 @@ static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, /* check shutdown conditions */ if (pollset->shutting_down) { if (pollset_has_workers(pollset)) { - pollset_kick(pollset, NULL); + pollset_kick(exec_ctx, pollset, NULL); } else if (!pollset->called_shutdown && !pollset_has_observers(pollset)) { pollset->called_shutdown = 1; gpr_mu_unlock(&pollset->mu); @@ -1112,7 +1116,7 @@ static void pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, GPR_ASSERT(!pollset->shutting_down); pollset->shutting_down = 1; pollset->shutdown_done = closure; - pollset_kick(pollset, GRPC_POLLSET_KICK_BROADCAST); + pollset_kick(exec_ctx, pollset, GRPC_POLLSET_KICK_BROADCAST); if (!pollset_has_workers(pollset)) { GRPC_CLOSURE_LIST_SCHED(exec_ctx, &pollset->idle_jobs); } diff --git a/src/core/lib/iomgr/ev_posix.c b/src/core/lib/iomgr/ev_posix.c index d881e2d4dd..4d3ae2228e 100644 --- a/src/core/lib/iomgr/ev_posix.c +++ b/src/core/lib/iomgr/ev_posix.c @@ -210,9 +210,9 @@ grpc_error *grpc_pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, return g_event_engine->pollset_work(exec_ctx, pollset, worker, now, deadline); } -grpc_error *grpc_pollset_kick(grpc_pollset *pollset, +grpc_error *grpc_pollset_kick(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_pollset_worker *specific_worker) { - return g_event_engine->pollset_kick(pollset, specific_worker); + return g_event_engine->pollset_kick(exec_ctx, pollset, specific_worker); } void grpc_pollset_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, diff --git a/src/core/lib/iomgr/ev_posix.h b/src/core/lib/iomgr/ev_posix.h index 1108e46ef8..1ff2ff1413 100644 --- a/src/core/lib/iomgr/ev_posix.h +++ b/src/core/lib/iomgr/ev_posix.h @@ -54,7 +54,7 @@ typedef struct grpc_event_engine_vtable { grpc_error *(*pollset_work)(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_pollset_worker **worker, gpr_timespec now, gpr_timespec deadline); - grpc_error *(*pollset_kick)(grpc_pollset *pollset, + grpc_error *(*pollset_kick)(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_pollset_worker *specific_worker); void (*pollset_add_fd)(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, struct grpc_fd *fd); diff --git a/src/core/lib/iomgr/pollset.h b/src/core/lib/iomgr/pollset.h index a609a3877a..a0f6b3a9d3 100644 --- a/src/core/lib/iomgr/pollset.h +++ b/src/core/lib/iomgr/pollset.h @@ -76,7 +76,7 @@ grpc_error *grpc_pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, /* Break one polling thread out of polling work for this pollset. If specific_worker is non-NULL, then kick that worker. */ -grpc_error *grpc_pollset_kick(grpc_pollset *pollset, +grpc_error *grpc_pollset_kick(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_pollset_worker *specific_worker) GRPC_MUST_USE_RESULT; diff --git a/src/core/lib/security/credentials/google_default/google_default_credentials.c b/src/core/lib/security/credentials/google_default/google_default_credentials.c index a2a8e289ee..691d66df69 100644 --- a/src/core/lib/security/credentials/google_default/google_default_credentials.c +++ b/src/core/lib/security/credentials/google_default/google_default_credentials.c @@ -79,7 +79,8 @@ static void on_compute_engine_detection_http_response(grpc_exec_ctx *exec_ctx, detector->is_done = 1; GRPC_LOG_IF_ERROR( "Pollset kick", - grpc_pollset_kick(grpc_polling_entity_pollset(&detector->pollent), NULL)); + grpc_pollset_kick(exec_ctx, + grpc_polling_entity_pollset(&detector->pollent), NULL)); gpr_mu_unlock(g_polling_mu); } diff --git a/src/core/lib/surface/call.c b/src/core/lib/surface/call.c index de5aa7b6be..bb5b57a8bc 100644 --- a/src/core/lib/surface/call.c +++ b/src/core/lib/surface/call.c @@ -330,8 +330,9 @@ grpc_error *grpc_call_create(grpc_exec_ctx *exec_ctx, grpc_channel_get_channel_stack(args->channel); grpc_call *call; GPR_TIMER_BEGIN("grpc_call_create", 0); - gpr_arena *arena = - gpr_arena_create(grpc_channel_get_call_size_estimate(args->channel)); + size_t initial_size = grpc_channel_get_call_size_estimate(args->channel); + GRPC_STATS_INC_CALL_INITIAL_SIZE(exec_ctx, initial_size); + gpr_arena *arena = gpr_arena_create(initial_size); call = (grpc_call *)gpr_arena_alloc( arena, sizeof(grpc_call) + channel_stack->call_stack_size); gpr_ref_init(&call->ext_ref, 1); diff --git a/src/core/lib/surface/completion_queue.c b/src/core/lib/surface/completion_queue.c index 4726503994..bc57c6136b 100644 --- a/src/core/lib/surface/completion_queue.c +++ b/src/core/lib/surface/completion_queue.c @@ -54,7 +54,7 @@ typedef struct { bool can_listen; size_t (*size)(void); void (*init)(grpc_pollset *pollset, gpr_mu **mu); - grpc_error *(*kick)(grpc_pollset *pollset, + grpc_error *(*kick)(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_pollset_worker *specific_worker); grpc_error *(*work)(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_pollset_worker **worker, gpr_timespec now, @@ -130,7 +130,8 @@ static grpc_error *non_polling_poller_work(grpc_exec_ctx *exec_ctx, } static grpc_error *non_polling_poller_kick( - grpc_pollset *pollset, grpc_pollset_worker *specific_worker) { + grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, + grpc_pollset_worker *specific_worker) { non_polling_poller *p = (non_polling_poller *)pollset; if (specific_worker == NULL) specific_worker = (grpc_pollset_worker *)p->root; if (specific_worker != NULL) { @@ -646,7 +647,7 @@ static void cq_end_op_for_next(grpc_exec_ctx *exec_ctx, if (is_first) { gpr_mu_lock(cq->mu); grpc_error *kick_error = - cq->poller_vtable->kick(POLLSET_FROM_CQ(cq), NULL); + cq->poller_vtable->kick(exec_ctx, POLLSET_FROM_CQ(cq), NULL); gpr_mu_unlock(cq->mu); if (kick_error != GRPC_ERROR_NONE) { @@ -732,7 +733,7 @@ static void cq_end_op_for_pluck(grpc_exec_ctx *exec_ctx, } grpc_error *kick_error = - cq->poller_vtable->kick(POLLSET_FROM_CQ(cq), pluck_worker); + cq->poller_vtable->kick(exec_ctx, POLLSET_FROM_CQ(cq), pluck_worker); gpr_mu_unlock(cq->mu); @@ -931,7 +932,7 @@ static grpc_event cq_next(grpc_completion_queue *cq, gpr_timespec deadline, if (cq_event_queue_num_items(&cqd->queue) > 0 && gpr_atm_no_barrier_load(&cqd->pending_events) > 0) { gpr_mu_lock(cq->mu); - cq->poller_vtable->kick(POLLSET_FROM_CQ(cq), NULL); + cq->poller_vtable->kick(&exec_ctx, POLLSET_FROM_CQ(cq), NULL); gpr_mu_unlock(cq->mu); } diff --git a/test/core/http/httpcli_test.c b/test/core/http/httpcli_test.c index b8b96d673c..8a53903763 100644 --- a/test/core/http/httpcli_test.c +++ b/test/core/http/httpcli_test.c @@ -52,7 +52,7 @@ static void on_finish(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) { g_done = 1; GPR_ASSERT(GRPC_LOG_IF_ERROR( "pollset_kick", - grpc_pollset_kick(grpc_polling_entity_pollset(&g_pops), NULL))); + grpc_pollset_kick(exec_ctx, grpc_polling_entity_pollset(&g_pops), NULL))); gpr_mu_unlock(g_mu); } diff --git a/test/core/http/httpscli_test.c b/test/core/http/httpscli_test.c index a9d7abdcff..c7455bd8df 100644 --- a/test/core/http/httpscli_test.c +++ b/test/core/http/httpscli_test.c @@ -52,7 +52,7 @@ static void on_finish(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) { g_done = 1; GPR_ASSERT(GRPC_LOG_IF_ERROR( "pollset_kick", - grpc_pollset_kick(grpc_polling_entity_pollset(&g_pops), NULL))); + grpc_pollset_kick(exec_ctx, grpc_polling_entity_pollset(&g_pops), NULL))); gpr_mu_unlock(g_mu); } diff --git a/test/core/iomgr/endpoint_tests.c b/test/core/iomgr/endpoint_tests.c index 895cc0e158..f8570edde7 100644 --- a/test/core/iomgr/endpoint_tests.c +++ b/test/core/iomgr/endpoint_tests.c @@ -126,7 +126,8 @@ static void read_and_write_test_read_handler(grpc_exec_ctx *exec_ctx, gpr_log(GPR_INFO, "Read handler done"); gpr_mu_lock(g_mu); state->read_done = 1 + (error == GRPC_ERROR_NONE); - GRPC_LOG_IF_ERROR("pollset_kick", grpc_pollset_kick(g_pollset, NULL)); + GRPC_LOG_IF_ERROR("pollset_kick", + grpc_pollset_kick(exec_ctx, g_pollset, NULL)); gpr_mu_unlock(g_mu); } else if (error == GRPC_ERROR_NONE) { grpc_endpoint_read(exec_ctx, state->read_ep, &state->incoming, @@ -162,7 +163,8 @@ static void read_and_write_test_write_handler(grpc_exec_ctx *exec_ctx, gpr_log(GPR_INFO, "Write handler done"); gpr_mu_lock(g_mu); state->write_done = 1 + (error == GRPC_ERROR_NONE); - GRPC_LOG_IF_ERROR("pollset_kick", grpc_pollset_kick(g_pollset, NULL)); + GRPC_LOG_IF_ERROR("pollset_kick", + grpc_pollset_kick(exec_ctx, g_pollset, NULL)); gpr_mu_unlock(g_mu); } @@ -254,7 +256,8 @@ static void inc_on_failure(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) { gpr_mu_lock(g_mu); *(int *)arg += (error != GRPC_ERROR_NONE); - GPR_ASSERT(GRPC_LOG_IF_ERROR("kick", grpc_pollset_kick(g_pollset, NULL))); + GPR_ASSERT( + GRPC_LOG_IF_ERROR("kick", grpc_pollset_kick(exec_ctx, g_pollset, NULL))); gpr_mu_unlock(g_mu); } diff --git a/test/core/iomgr/fd_posix_test.c b/test/core/iomgr/fd_posix_test.c index 5791d17af6..881277a8d6 100644 --- a/test/core/iomgr/fd_posix_test.c +++ b/test/core/iomgr/fd_posix_test.c @@ -178,8 +178,8 @@ static void listen_shutdown_cb(grpc_exec_ctx *exec_ctx, void *arg /*server */, gpr_mu_lock(g_mu); sv->done = 1; - GPR_ASSERT( - GRPC_LOG_IF_ERROR("pollset_kick", grpc_pollset_kick(g_pollset, NULL))); + GPR_ASSERT(GRPC_LOG_IF_ERROR("pollset_kick", + grpc_pollset_kick(exec_ctx, g_pollset, NULL))); gpr_mu_unlock(g_mu); } @@ -297,8 +297,8 @@ static void client_session_shutdown_cb(grpc_exec_ctx *exec_ctx, grpc_fd_orphan(exec_ctx, cl->em_fd, NULL, NULL, false /* already_closed */, "c"); cl->done = 1; - GPR_ASSERT( - GRPC_LOG_IF_ERROR("pollset_kick", grpc_pollset_kick(g_pollset, NULL))); + GPR_ASSERT(GRPC_LOG_IF_ERROR("pollset_kick", + grpc_pollset_kick(exec_ctx, g_pollset, NULL))); } /* Write as much as possible, then register notify_on_write. */ @@ -417,8 +417,8 @@ static void first_read_callback(grpc_exec_ctx *exec_ctx, gpr_mu_lock(g_mu); fdc->cb_that_ran = first_read_callback; - GPR_ASSERT( - GRPC_LOG_IF_ERROR("pollset_kick", grpc_pollset_kick(g_pollset, NULL))); + GPR_ASSERT(GRPC_LOG_IF_ERROR("pollset_kick", + grpc_pollset_kick(exec_ctx, g_pollset, NULL))); gpr_mu_unlock(g_mu); } @@ -429,8 +429,8 @@ static void second_read_callback(grpc_exec_ctx *exec_ctx, gpr_mu_lock(g_mu); fdc->cb_that_ran = second_read_callback; - GPR_ASSERT( - GRPC_LOG_IF_ERROR("pollset_kick", grpc_pollset_kick(g_pollset, NULL))); + GPR_ASSERT(GRPC_LOG_IF_ERROR("pollset_kick", + grpc_pollset_kick(exec_ctx, g_pollset, NULL))); gpr_mu_unlock(g_mu); } diff --git a/test/core/iomgr/resolve_address_test.c b/test/core/iomgr/resolve_address_test.c index 7f5c4073d8..1110c04b6e 100644 --- a/test/core/iomgr/resolve_address_test.c +++ b/test/core/iomgr/resolve_address_test.c @@ -106,7 +106,8 @@ static void must_succeed(grpc_exec_ctx *exec_ctx, void *argsp, GPR_ASSERT(args->addrs->naddrs > 0); gpr_atm_rel_store(&args->done_atm, 1); gpr_mu_lock(args->mu); - GRPC_LOG_IF_ERROR("pollset_kick", grpc_pollset_kick(args->pollset, NULL)); + GRPC_LOG_IF_ERROR("pollset_kick", + grpc_pollset_kick(exec_ctx, args->pollset, NULL)); gpr_mu_unlock(args->mu); } @@ -115,7 +116,8 @@ static void must_fail(grpc_exec_ctx *exec_ctx, void *argsp, grpc_error *err) { GPR_ASSERT(err != GRPC_ERROR_NONE); gpr_atm_rel_store(&args->done_atm, 1); gpr_mu_lock(args->mu); - GRPC_LOG_IF_ERROR("pollset_kick", grpc_pollset_kick(args->pollset, NULL)); + GRPC_LOG_IF_ERROR("pollset_kick", + grpc_pollset_kick(exec_ctx, args->pollset, NULL)); gpr_mu_unlock(args->mu); } diff --git a/test/core/iomgr/tcp_client_posix_test.c b/test/core/iomgr/tcp_client_posix_test.c index 00ea495bbe..1032da942b 100644 --- a/test/core/iomgr/tcp_client_posix_test.c +++ b/test/core/iomgr/tcp_client_posix_test.c @@ -53,8 +53,10 @@ static gpr_timespec test_deadline(void) { static void finish_connection() { gpr_mu_lock(g_mu); g_connections_complete++; - GPR_ASSERT( - GRPC_LOG_IF_ERROR("pollset_kick", grpc_pollset_kick(g_pollset, NULL))); + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + GPR_ASSERT(GRPC_LOG_IF_ERROR("pollset_kick", + grpc_pollset_kick(&exec_ctx, g_pollset, NULL))); + grpc_exec_ctx_finish(&exec_ctx); gpr_mu_unlock(g_mu); } diff --git a/test/core/iomgr/tcp_posix_test.c b/test/core/iomgr/tcp_posix_test.c index cdaa2ce2af..cfb3cf897c 100644 --- a/test/core/iomgr/tcp_posix_test.c +++ b/test/core/iomgr/tcp_posix_test.c @@ -147,7 +147,8 @@ static void read_cb(grpc_exec_ctx *exec_ctx, void *user_data, gpr_log(GPR_INFO, "Read %" PRIuPTR " bytes of %" PRIuPTR, read_bytes, state->target_read_bytes); if (state->read_bytes >= state->target_read_bytes) { - GPR_ASSERT(GRPC_LOG_IF_ERROR("kick", grpc_pollset_kick(g_pollset, NULL))); + GPR_ASSERT(GRPC_LOG_IF_ERROR("kick", + grpc_pollset_kick(exec_ctx, g_pollset, NULL))); gpr_mu_unlock(g_mu); } else { grpc_endpoint_read(exec_ctx, state->ep, &state->incoming, &state->read_cb); @@ -295,8 +296,8 @@ static void write_done(grpc_exec_ctx *exec_ctx, gpr_mu_lock(g_mu); gpr_log(GPR_INFO, "Signalling write done"); state->write_done = 1; - GPR_ASSERT( - GRPC_LOG_IF_ERROR("pollset_kick", grpc_pollset_kick(g_pollset, NULL))); + GPR_ASSERT(GRPC_LOG_IF_ERROR("pollset_kick", + grpc_pollset_kick(exec_ctx, g_pollset, NULL))); gpr_mu_unlock(g_mu); } @@ -406,8 +407,8 @@ static void write_test(size_t num_bytes, size_t slice_size) { void on_fd_released(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *errors) { int *done = (int *)arg; *done = 1; - GPR_ASSERT( - GRPC_LOG_IF_ERROR("pollset_kick", grpc_pollset_kick(g_pollset, NULL))); + GPR_ASSERT(GRPC_LOG_IF_ERROR("pollset_kick", + grpc_pollset_kick(exec_ctx, g_pollset, NULL))); } /* Do a read_test, then release fd and try to read/write again. Verify that diff --git a/test/core/iomgr/tcp_server_posix_test.c b/test/core/iomgr/tcp_server_posix_test.c index 2371721a60..4d84608376 100644 --- a/test/core/iomgr/tcp_server_posix_test.c +++ b/test/core/iomgr/tcp_server_posix_test.c @@ -159,8 +159,8 @@ static void on_connect(grpc_exec_ctx *exec_ctx, void *arg, grpc_endpoint *tcp, gpr_mu_lock(g_mu); g_result = temp_result; g_nconnects++; - GPR_ASSERT( - GRPC_LOG_IF_ERROR("pollset_kick", grpc_pollset_kick(g_pollset, NULL))); + GPR_ASSERT(GRPC_LOG_IF_ERROR("pollset_kick", + grpc_pollset_kick(exec_ctx, g_pollset, NULL))); gpr_mu_unlock(g_mu); } diff --git a/test/core/iomgr/udp_server_test.c b/test/core/iomgr/udp_server_test.c index aa34857dbd..1d051bea62 100644 --- a/test/core/iomgr/udp_server_test.c +++ b/test/core/iomgr/udp_server_test.c @@ -61,8 +61,8 @@ static void on_read(grpc_exec_ctx *exec_ctx, grpc_fd *emfd, void *user_data) { g_number_of_reads++; g_number_of_bytes_read += (int)byte_count; - GPR_ASSERT( - GRPC_LOG_IF_ERROR("pollset_kick", grpc_pollset_kick(g_pollset, NULL))); + GPR_ASSERT(GRPC_LOG_IF_ERROR("pollset_kick", + grpc_pollset_kick(exec_ctx, g_pollset, NULL))); gpr_mu_unlock(g_mu); } @@ -70,8 +70,8 @@ static void on_write(grpc_exec_ctx *exec_ctx, grpc_fd *emfd, void *user_data) { gpr_mu_lock(g_mu); g_number_of_writes++; - GPR_ASSERT( - GRPC_LOG_IF_ERROR("pollset_kick", grpc_pollset_kick(g_pollset, NULL))); + GPR_ASSERT(GRPC_LOG_IF_ERROR("pollset_kick", + grpc_pollset_kick(exec_ctx, g_pollset, NULL))); gpr_mu_unlock(g_mu); } diff --git a/test/core/security/oauth2_utils.c b/test/core/security/oauth2_utils.c index fdbc6ea741..d240403a29 100644 --- a/test/core/security/oauth2_utils.c +++ b/test/core/security/oauth2_utils.c @@ -60,7 +60,8 @@ static void on_oauth2_response(grpc_exec_ctx *exec_ctx, void *arg, request->token = token; GRPC_LOG_IF_ERROR( "pollset_kick", - grpc_pollset_kick(grpc_polling_entity_pollset(&request->pops), NULL)); + grpc_pollset_kick(exec_ctx, grpc_polling_entity_pollset(&request->pops), + NULL)); gpr_mu_unlock(request->mu); } diff --git a/test/core/security/print_google_default_creds_token.c b/test/core/security/print_google_default_creds_token.c index e1385a80ca..3144717a85 100644 --- a/test/core/security/print_google_default_creds_token.c +++ b/test/core/security/print_google_default_creds_token.c @@ -57,7 +57,8 @@ static void on_metadata_response(grpc_exec_ctx *exec_ctx, void *arg, sync->is_done = true; GRPC_LOG_IF_ERROR( "pollset_kick", - grpc_pollset_kick(grpc_polling_entity_pollset(&sync->pops), NULL)); + grpc_pollset_kick(exec_ctx, grpc_polling_entity_pollset(&sync->pops), + NULL)); gpr_mu_unlock(sync->mu); } diff --git a/test/core/security/verify_jwt.c b/test/core/security/verify_jwt.c index 259038f4d6..5faa6352a8 100644 --- a/test/core/security/verify_jwt.c +++ b/test/core/security/verify_jwt.c @@ -66,7 +66,8 @@ static void on_jwt_verification_done(grpc_exec_ctx *exec_ctx, void *user_data, gpr_mu_lock(sync->mu); sync->is_done = 1; - GRPC_LOG_IF_ERROR("pollset_kick", grpc_pollset_kick(sync->pollset, NULL)); + GRPC_LOG_IF_ERROR("pollset_kick", + grpc_pollset_kick(exec_ctx, sync->pollset, NULL)); gpr_mu_unlock(sync->mu); } diff --git a/test/core/surface/concurrent_connectivity_test.c b/test/core/surface/concurrent_connectivity_test.c index 08079b6091..ec2cd8610b 100644 --- a/test/core/surface/concurrent_connectivity_test.c +++ b/test/core/surface/concurrent_connectivity_test.c @@ -108,7 +108,8 @@ static void on_connect(grpc_exec_ctx *exec_ctx, void *vargs, grpc_endpoint *tcp, GRPC_ERROR_CREATE_FROM_STATIC_STRING("Connected")); grpc_endpoint_destroy(exec_ctx, tcp); gpr_mu_lock(args->mu); - GRPC_LOG_IF_ERROR("pollset_kick", grpc_pollset_kick(args->pollset, NULL)); + GRPC_LOG_IF_ERROR("pollset_kick", + grpc_pollset_kick(exec_ctx, args->pollset, NULL)); gpr_mu_unlock(args->mu); } diff --git a/test/core/util/port_server_client.c b/test/core/util/port_server_client.c index 851c50941e..ba4028dbee 100644 --- a/test/core/util/port_server_client.c +++ b/test/core/util/port_server_client.c @@ -54,7 +54,8 @@ static void freed_port_from_server(grpc_exec_ctx *exec_ctx, void *arg, pr->done = 1; GRPC_LOG_IF_ERROR( "pollset_kick", - grpc_pollset_kick(grpc_polling_entity_pollset(&pr->pops), NULL)); + grpc_pollset_kick(exec_ctx, grpc_polling_entity_pollset(&pr->pops), + NULL)); gpr_mu_unlock(pr->mu); } @@ -153,7 +154,8 @@ static void got_port_from_server(grpc_exec_ctx *exec_ctx, void *arg, pr->port = 0; GRPC_LOG_IF_ERROR( "pollset_kick", - grpc_pollset_kick(grpc_polling_entity_pollset(&pr->pops), NULL)); + grpc_pollset_kick(exec_ctx, grpc_polling_entity_pollset(&pr->pops), + NULL)); gpr_mu_unlock(pr->mu); return; } @@ -189,7 +191,8 @@ static void got_port_from_server(grpc_exec_ctx *exec_ctx, void *arg, pr->port = port; GRPC_LOG_IF_ERROR( "pollset_kick", - grpc_pollset_kick(grpc_polling_entity_pollset(&pr->pops), NULL)); + grpc_pollset_kick(exec_ctx, grpc_polling_entity_pollset(&pr->pops), + NULL)); gpr_mu_unlock(pr->mu); } diff --git a/test/cpp/microbenchmarks/bm_cq_multiple_threads.cc b/test/cpp/microbenchmarks/bm_cq_multiple_threads.cc index 9d71d3990d..5c9405f583 100644 --- a/test/cpp/microbenchmarks/bm_cq_multiple_threads.cc +++ b/test/cpp/microbenchmarks/bm_cq_multiple_threads.cc @@ -59,7 +59,8 @@ static void pollset_destroy(grpc_exec_ctx* exec_ctx, grpc_pollset* ps) { gpr_mu_destroy(&ps->mu); } -static grpc_error* pollset_kick(grpc_pollset* p, grpc_pollset_worker* worker) { +static grpc_error* pollset_kick(grpc_exec_ctx* exec_ctx, grpc_pollset* p, + grpc_pollset_worker* worker) { return GRPC_ERROR_NONE; } diff --git a/test/cpp/qps/report.cc b/test/cpp/qps/report.cc index 3c99bda144..42ebeff41d 100644 --- a/test/cpp/qps/report.cc +++ b/test/cpp/qps/report.cc @@ -107,8 +107,8 @@ void GprLogReporter::ReportCoreStats(const char* name, int idx, grpc_stats_counter_name[i], data.counters[i]); } for (int i = 0; i < GRPC_STATS_HISTOGRAM_COUNT; i++) { - gpr_log(GPR_DEBUG, "%s[%d].%s = %lf/%lf/%lf (50/95/99%%-ile)", name, idx, - grpc_stats_histogram_name[i], + gpr_log(GPR_DEBUG, "%s[%d].%s = %.1lf/%.1lf/%.1lf (50/95/99%%-ile)", name, + idx, grpc_stats_histogram_name[i], grpc_stats_histo_percentile(&data, (grpc_stats_histograms)i, 50), grpc_stats_histo_percentile(&data, (grpc_stats_histograms)i, 95), grpc_stats_histo_percentile(&data, (grpc_stats_histograms)i, 99)); diff --git a/tools/run_tests/performance/massage_qps_stats.py b/tools/run_tests/performance/massage_qps_stats.py index 30a94ca7cd..606660f700 100644 --- a/tools/run_tests/performance/massage_qps_stats.py +++ b/tools/run_tests/performance/massage_qps_stats.py @@ -24,6 +24,12 @@ def massage_qps_stats(scenario_result): stats["core_server_calls_created"] = massage_qps_stats_helpers.counter(core_stats, "server_calls_created") stats["core_syscall_poll"] = massage_qps_stats_helpers.counter(core_stats, "syscall_poll") stats["core_syscall_wait"] = massage_qps_stats_helpers.counter(core_stats, "syscall_wait") + stats["core_pollset_kick"] = massage_qps_stats_helpers.counter(core_stats, "pollset_kick") + stats["core_pollset_kicked_without_poller"] = massage_qps_stats_helpers.counter(core_stats, "pollset_kicked_without_poller") + stats["core_pollset_kicked_again"] = massage_qps_stats_helpers.counter(core_stats, "pollset_kicked_again") + stats["core_pollset_kick_wakeup_fd"] = massage_qps_stats_helpers.counter(core_stats, "pollset_kick_wakeup_fd") + stats["core_pollset_kick_wakeup_cv"] = massage_qps_stats_helpers.counter(core_stats, "pollset_kick_wakeup_cv") + stats["core_pollset_kick_own_thread"] = massage_qps_stats_helpers.counter(core_stats, "pollset_kick_own_thread") stats["core_histogram_slow_lookups"] = massage_qps_stats_helpers.counter(core_stats, "histogram_slow_lookups") stats["core_syscall_write"] = massage_qps_stats_helpers.counter(core_stats, "syscall_write") stats["core_syscall_read"] = massage_qps_stats_helpers.counter(core_stats, "syscall_read") @@ -55,6 +61,18 @@ def massage_qps_stats(scenario_result): stats["core_executor_push_retries"] = massage_qps_stats_helpers.counter(core_stats, "executor_push_retries") stats["core_server_requested_calls"] = massage_qps_stats_helpers.counter(core_stats, "server_requested_calls") stats["core_server_slowpath_requests_queued"] = massage_qps_stats_helpers.counter(core_stats, "server_slowpath_requests_queued") + h = massage_qps_stats_helpers.histogram(core_stats, "call_initial_size") + stats["core_call_initial_size"] = ",".join("%f" % x for x in h.buckets) + stats["core_call_initial_size_bkts"] = ",".join("%f" % x for x in h.boundaries) + stats["core_call_initial_size_50p"] = massage_qps_stats_helpers.percentile(h.buckets, 50, h.boundaries) + stats["core_call_initial_size_95p"] = massage_qps_stats_helpers.percentile(h.buckets, 95, h.boundaries) + stats["core_call_initial_size_99p"] = massage_qps_stats_helpers.percentile(h.buckets, 99, h.boundaries) + h = massage_qps_stats_helpers.histogram(core_stats, "poll_events_returned") + stats["core_poll_events_returned"] = ",".join("%f" % x for x in h.buckets) + stats["core_poll_events_returned_bkts"] = ",".join("%f" % x for x in h.boundaries) + stats["core_poll_events_returned_50p"] = massage_qps_stats_helpers.percentile(h.buckets, 50, h.boundaries) + stats["core_poll_events_returned_95p"] = massage_qps_stats_helpers.percentile(h.buckets, 95, h.boundaries) + stats["core_poll_events_returned_99p"] = massage_qps_stats_helpers.percentile(h.buckets, 99, h.boundaries) h = massage_qps_stats_helpers.histogram(core_stats, "tcp_write_size") stats["core_tcp_write_size"] = ",".join("%f" % x for x in h.buckets) stats["core_tcp_write_size_bkts"] = ",".join("%f" % x for x in h.boundaries) diff --git a/tools/run_tests/performance/scenario_result_schema.json b/tools/run_tests/performance/scenario_result_schema.json index 84b3cb7c0a..11f41ab012 100644 --- a/tools/run_tests/performance/scenario_result_schema.json +++ b/tools/run_tests/performance/scenario_result_schema.json @@ -132,6 +132,36 @@ }, { "mode": "NULLABLE", + "name": "core_pollset_kick", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_pollset_kicked_without_poller", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_pollset_kicked_again", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_pollset_kick_wakeup_fd", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_pollset_kick_wakeup_cv", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_pollset_kick_own_thread", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", "name": "core_histogram_slow_lookups", "type": "INTEGER" }, @@ -287,6 +317,56 @@ }, { "mode": "NULLABLE", + "name": "core_call_initial_size", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_call_initial_size_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_call_initial_size_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_call_initial_size_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_call_initial_size_99p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_poll_events_returned", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_poll_events_returned_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_poll_events_returned_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_poll_events_returned_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_poll_events_returned_99p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", "name": "core_tcp_write_size", "type": "STRING" }, @@ -609,6 +689,36 @@ }, { "mode": "NULLABLE", + "name": "core_pollset_kick", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_pollset_kicked_without_poller", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_pollset_kicked_again", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_pollset_kick_wakeup_fd", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_pollset_kick_wakeup_cv", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_pollset_kick_own_thread", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", "name": "core_histogram_slow_lookups", "type": "INTEGER" }, @@ -764,6 +874,56 @@ }, { "mode": "NULLABLE", + "name": "core_call_initial_size", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_call_initial_size_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_call_initial_size_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_call_initial_size_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_call_initial_size_99p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_poll_events_returned", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_poll_events_returned_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_poll_events_returned_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_poll_events_returned_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_poll_events_returned_99p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", "name": "core_tcp_write_size", "type": "STRING" }, |