Changes to exec_ctx/closure/combiner/workqueue interfaces

- make closures know where they should be executed (eg, on a workqueue,
  or a combiner, or on an exec_ctx)
- this allows removal of a large number of trampoline functions that
  were appearing whenever we used combiners, and should allow for a much
  easier interface to combiner locks

11 files changed, 77 insertions, 49 deletions

diff --git a/test/core/iomgr/combiner_test.c b/test/core/iomgr/combiner_test.c
index f7d5809be7..9b6d6ff9b4 100644
--- a/test/core/iomgr/combiner_test.c
+++ b/test/core/iomgr/combiner_test.c
@@ -59,9 +59,10 @@ static void test_execute_one(void) {
   grpc_combiner *lock = grpc_combiner_create(NULL);
   bool done = false;
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
-  grpc_combiner_execute(&exec_ctx, lock,
-                        grpc_closure_create(set_bool_to_true, &done),
-                        GRPC_ERROR_NONE, false);
+  grpc_closure_sched(&exec_ctx,
+                     grpc_closure_create(set_bool_to_true, &done,
+                                         grpc_combiner_scheduler(lock, false)),
+                     GRPC_ERROR_NONE);
   grpc_exec_ctx_flush(&exec_ctx);
   GPR_ASSERT(done);
   grpc_combiner_destroy(&exec_ctx, lock);
@@ -94,9 +95,10 @@ static void execute_many_loop(void *a) {
       ex_args *c = gpr_malloc(sizeof(*c));
       c->ctr = &args->ctr;
       c->value = n++;
-      grpc_combiner_execute(&exec_ctx, args->lock,
-                            grpc_closure_create(check_one, c), GRPC_ERROR_NONE,
-                            false);
+      grpc_closure_sched(
+          &exec_ctx, grpc_closure_create(check_one, c, grpc_combiner_scheduler(
+                                                           args->lock, false)),
+          GRPC_ERROR_NONE);
       grpc_exec_ctx_flush(&exec_ctx);
     }
     // sleep for a little bit, to test a combiner draining and another thread
@@ -134,9 +136,10 @@ static void in_finally(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) {
 }
 
 static void add_finally(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) {
-  grpc_combiner_execute_finally(exec_ctx, arg,
-                                grpc_closure_create(in_finally, NULL),
-                                GRPC_ERROR_NONE, false);
+  grpc_closure_sched(exec_ctx, grpc_closure_create(
+                                   in_finally, NULL,
+                                   grpc_combiner_finally_scheduler(arg, false)),
+                     GRPC_ERROR_NONE);
 }
 
 static void test_execute_finally(void) {
@@ -144,8 +147,10 @@ static void test_execute_finally(void) {
 
   grpc_combiner *lock = grpc_combiner_create(NULL);
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
-  grpc_combiner_execute(&exec_ctx, lock, grpc_closure_create(add_finally, lock),
-                        GRPC_ERROR_NONE, false);
+  grpc_closure_sched(&exec_ctx,
+                     grpc_closure_create(add_finally, lock,
+                                         grpc_combiner_scheduler(lock, false)),
+                     GRPC_ERROR_NONE);
   grpc_exec_ctx_flush(&exec_ctx);
   GPR_ASSERT(got_in_finally);
   grpc_combiner_destroy(&exec_ctx, lock);
diff --git a/test/core/iomgr/endpoint_pair_test.c b/test/core/iomgr/endpoint_pair_test.c
index 2a257a7cea..6899f6524a 100644
--- a/test/core/iomgr/endpoint_pair_test.c
+++ b/test/core/iomgr/endpoint_pair_test.c
@@ -81,7 +81,7 @@ int main(int argc, char **argv) {
   g_pollset = gpr_malloc(grpc_pollset_size());
   grpc_pollset_init(g_pollset, &g_mu);
   grpc_endpoint_tests(configs[0], g_pollset, g_mu);
-  grpc_closure_init(&destroyed, destroy_pollset, g_pollset);
+  grpc_closure_init(&destroyed, destroy_pollset, g_pollset, grpc_schedule_on_exec_ctx);
   grpc_pollset_shutdown(&exec_ctx, g_pollset, &destroyed);
   grpc_exec_ctx_finish(&exec_ctx);
   grpc_shutdown();
diff --git a/test/core/iomgr/endpoint_tests.c b/test/core/iomgr/endpoint_tests.c
index 8186ea7e85..87a9d79e9b 100644
--- a/test/core/iomgr/endpoint_tests.c
+++ b/test/core/iomgr/endpoint_tests.c
@@ -211,9 +211,10 @@ static void read_and_write_test(grpc_endpoint_test_config config,
   state.write_done = 0;
   state.current_read_data = 0;
   state.current_write_data = 0;
-  grpc_closure_init(&state.done_read, read_and_write_test_read_handler, &state);
+  grpc_closure_init(&state.done_read, read_and_write_test_read_handler, &state,
+                    grpc_schedule_on_exec_ctx);
   grpc_closure_init(&state.done_write, read_and_write_test_write_handler,
-                    &state);
+                    &state, grpc_schedule_on_exec_ctx);
   grpc_slice_buffer_init(&state.outgoing);
   grpc_slice_buffer_init(&state.incoming);
 
@@ -290,16 +291,19 @@ static void multiple_shutdown_test(grpc_endpoint_test_config config) {
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
   grpc_endpoint_add_to_pollset(&exec_ctx, f.client_ep, g_pollset);
   grpc_endpoint_read(&exec_ctx, f.client_ep, &slice_buffer,
-                     grpc_closure_create(inc_on_failure, &fail_count));
+                     grpc_closure_create(inc_on_failure, &fail_count,
+                                         grpc_schedule_on_exec_ctx));
   wait_for_fail_count(&exec_ctx, &fail_count, 0);
   grpc_endpoint_shutdown(&exec_ctx, f.client_ep);
   wait_for_fail_count(&exec_ctx, &fail_count, 1);
   grpc_endpoint_read(&exec_ctx, f.client_ep, &slice_buffer,
-                     grpc_closure_create(inc_on_failure, &fail_count));
+                     grpc_closure_create(inc_on_failure, &fail_count,
+                                         grpc_schedule_on_exec_ctx));
   wait_for_fail_count(&exec_ctx, &fail_count, 2);
   grpc_slice_buffer_add(&slice_buffer, grpc_slice_from_copied_string("a"));
   grpc_endpoint_write(&exec_ctx, f.client_ep, &slice_buffer,
-                      grpc_closure_create(inc_on_failure, &fail_count));
+                      grpc_closure_create(inc_on_failure, &fail_count,
+                                          grpc_schedule_on_exec_ctx));
   wait_for_fail_count(&exec_ctx, &fail_count, 3);
   grpc_endpoint_shutdown(&exec_ctx, f.client_ep);
   wait_for_fail_count(&exec_ctx, &fail_count, 3);
diff --git a/test/core/iomgr/ev_epoll_linux_test.c b/test/core/iomgr/ev_epoll_linux_test.c
index 564b05d7f4..4fee4cb8c3 100644
--- a/test/core/iomgr/ev_epoll_linux_test.c
+++ b/test/core/iomgr/ev_epoll_linux_test.c
@@ -102,7 +102,7 @@ static void test_pollset_cleanup(grpc_exec_ctx *exec_ctx,
   int i;
 
   for (i = 0; i < num_pollsets; i++) {
-    grpc_closure_init(&destroyed, destroy_pollset, pollsets[i].pollset);
+    grpc_closure_init(&destroyed, destroy_pollset, pollsets[i].pollset, grpc_schedule_on_exec_ctx);
     grpc_pollset_shutdown(exec_ctx, pollsets[i].pollset, &destroyed);
 
     grpc_exec_ctx_flush(exec_ctx);
diff --git a/test/core/iomgr/fd_posix_test.c b/test/core/iomgr/fd_posix_test.c
index 6166699fe6..e7abecdb46 100644
--- a/test/core/iomgr/fd_posix_test.c
+++ b/test/core/iomgr/fd_posix_test.c
@@ -546,7 +546,7 @@ int main(int argc, char **argv) {
   grpc_pollset_init(g_pollset, &g_mu);
   test_grpc_fd();
   test_grpc_fd_change();
-  grpc_closure_init(&destroyed, destroy_pollset, g_pollset);
+  grpc_closure_init(&destroyed, destroy_pollset, g_pollset, grpc_schedule_on_exec_ctx);
   grpc_pollset_shutdown(&exec_ctx, g_pollset, &destroyed);
   grpc_exec_ctx_finish(&exec_ctx);
   gpr_free(g_pollset);
diff --git a/test/core/iomgr/resolve_address_test.c b/test/core/iomgr/resolve_address_test.c
index e4136a7a7a..d844e6eceb 100644
--- a/test/core/iomgr/resolve_address_test.c
+++ b/test/core/iomgr/resolve_address_test.c
@@ -71,7 +71,8 @@ void args_finish(grpc_exec_ctx *exec_ctx, args_struct *args) {
   grpc_pollset_set_del_pollset(exec_ctx, args->pollset_set, args->pollset);
   grpc_pollset_set_destroy(args->pollset_set);
   grpc_closure do_nothing_cb;
-  grpc_closure_init(&do_nothing_cb, do_nothing, NULL);
+  grpc_closure_init(&do_nothing_cb, do_nothing, NULL,
+                    grpc_schedule_on_exec_ctx);
   grpc_pollset_shutdown(exec_ctx, args->pollset, &do_nothing_cb);
   // exec_ctx needs to be flushed before calling grpc_pollset_destroy()
   grpc_exec_ctx_flush(exec_ctx);
@@ -136,8 +137,10 @@ static void test_localhost(void) {
   args_struct args;
   args_init(&exec_ctx, &args);
   poll_pollset_until_request_done(&args);
-  grpc_resolve_address(&exec_ctx, "localhost:1", NULL, args.pollset_set,
-                       grpc_closure_create(must_succeed, &args), &args.addrs);
+  grpc_resolve_address(
+      &exec_ctx, "localhost:1", NULL, args.pollset_set,
+      grpc_closure_create(must_succeed, &args, grpc_schedule_on_exec_ctx),
+      &args.addrs);
   args_finish(&exec_ctx, &args);
   grpc_exec_ctx_finish(&exec_ctx);
 }
@@ -147,8 +150,10 @@ static void test_default_port(void) {
   args_struct args;
   args_init(&exec_ctx, &args);
   poll_pollset_until_request_done(&args);
-  grpc_resolve_address(&exec_ctx, "localhost", "1", args.pollset_set,
-                       grpc_closure_create(must_succeed, &args), &args.addrs);
+  grpc_resolve_address(
+      &exec_ctx, "localhost", "1", args.pollset_set,
+      grpc_closure_create(must_succeed, &args, grpc_schedule_on_exec_ctx),
+      &args.addrs);
   args_finish(&exec_ctx, &args);
   grpc_exec_ctx_finish(&exec_ctx);
 }
@@ -158,8 +163,10 @@ static void test_missing_default_port(void) {
   args_struct args;
   args_init(&exec_ctx, &args);
   poll_pollset_until_request_done(&args);
-  grpc_resolve_address(&exec_ctx, "localhost", NULL, args.pollset_set,
-                       grpc_closure_create(must_fail, &args), &args.addrs);
+  grpc_resolve_address(
+      &exec_ctx, "localhost", NULL, args.pollset_set,
+      grpc_closure_create(must_fail, &args, grpc_schedule_on_exec_ctx),
+      &args.addrs);
   args_finish(&exec_ctx, &args);
   grpc_exec_ctx_finish(&exec_ctx);
 }
@@ -169,8 +176,10 @@ static void test_ipv6_with_port(void) {
   args_struct args;
   args_init(&exec_ctx, &args);
   poll_pollset_until_request_done(&args);
-  grpc_resolve_address(&exec_ctx, "[2001:db8::1]:1", NULL, args.pollset_set,
-                       grpc_closure_create(must_succeed, &args), &args.addrs);
+  grpc_resolve_address(
+      &exec_ctx, "[2001:db8::1]:1", NULL, args.pollset_set,
+      grpc_closure_create(must_succeed, &args, grpc_schedule_on_exec_ctx),
+      &args.addrs);
   args_finish(&exec_ctx, &args);
   grpc_exec_ctx_finish(&exec_ctx);
 }
@@ -185,8 +194,10 @@ static void test_ipv6_without_port(void) {
     args_struct args;
     args_init(&exec_ctx, &args);
     poll_pollset_until_request_done(&args);
-    grpc_resolve_address(&exec_ctx, kCases[i], "80", args.pollset_set,
-                         grpc_closure_create(must_succeed, &args), &args.addrs);
+    grpc_resolve_address(
+        &exec_ctx, kCases[i], "80", args.pollset_set,
+        grpc_closure_create(must_succeed, &args, grpc_schedule_on_exec_ctx),
+        &args.addrs);
     args_finish(&exec_ctx, &args);
     grpc_exec_ctx_finish(&exec_ctx);
   }
@@ -202,8 +213,10 @@ static void test_invalid_ip_addresses(void) {
     args_struct args;
     args_init(&exec_ctx, &args);
     poll_pollset_until_request_done(&args);
-    grpc_resolve_address(&exec_ctx, kCases[i], NULL, args.pollset_set,
-                         grpc_closure_create(must_fail, &args), &args.addrs);
+    grpc_resolve_address(
+        &exec_ctx, kCases[i], NULL, args.pollset_set,
+        grpc_closure_create(must_fail, &args, grpc_schedule_on_exec_ctx),
+        &args.addrs);
     args_finish(&exec_ctx, &args);
     grpc_exec_ctx_finish(&exec_ctx);
   }
@@ -219,8 +232,10 @@ static void test_unparseable_hostports(void) {
     args_struct args;
     args_init(&exec_ctx, &args);
     poll_pollset_until_request_done(&args);
-    grpc_resolve_address(&exec_ctx, kCases[i], "1", args.pollset_set,
-                         grpc_closure_create(must_fail, &args), &args.addrs);
+    grpc_resolve_address(
+        &exec_ctx, kCases[i], "1", args.pollset_set,
+        grpc_closure_create(must_fail, &args, grpc_schedule_on_exec_ctx),
+        &args.addrs);
     args_finish(&exec_ctx, &args);
     grpc_exec_ctx_finish(&exec_ctx);
   }
diff --git a/test/core/iomgr/resource_quota_test.c b/test/core/iomgr/resource_quota_test.c
index a82d44f7f8..181776341f 100644
--- a/test/core/iomgr/resource_quota_test.c
+++ b/test/core/iomgr/resource_quota_test.c
@@ -45,7 +45,9 @@ static void inc_int_cb(grpc_exec_ctx *exec_ctx, void *a, grpc_error *error) {
 static void set_bool_cb(grpc_exec_ctx *exec_ctx, void *a, grpc_error *error) {
   *(bool *)a = true;
 }
-grpc_closure *set_bool(bool *p) { return grpc_closure_create(set_bool_cb, p); }
+grpc_closure *set_bool(bool *p) {
+  return grpc_closure_create(set_bool_cb, p, grpc_schedule_on_exec_ctx);
+}
 
 typedef struct {
   size_t size;
@@ -67,7 +69,7 @@ grpc_closure *make_reclaimer(grpc_resource_user *resource_user, size_t size,
   a->size = size;
   a->resource_user = resource_user;
   a->then = then;
-  return grpc_closure_create(reclaimer_cb, a);
+  return grpc_closure_create(reclaimer_cb, a, grpc_schedule_on_exec_ctx);
 }
 
 static void unused_reclaimer_cb(grpc_exec_ctx *exec_ctx, void *arg,
@@ -76,7 +78,8 @@ static void unused_reclaimer_cb(grpc_exec_ctx *exec_ctx, void *arg,
   grpc_closure_run(exec_ctx, arg, GRPC_ERROR_NONE);
 }
 grpc_closure *make_unused_reclaimer(grpc_closure *then) {
-  return grpc_closure_create(unused_reclaimer_cb, then);
+  return grpc_closure_create(unused_reclaimer_cb, then,
+                             grpc_schedule_on_exec_ctx);
 }
 
 static void destroy_user(grpc_resource_user *usr) {
diff --git a/test/core/iomgr/tcp_client_posix_test.c b/test/core/iomgr/tcp_client_posix_test.c
index 5fab826fb7..2100fd4c2f 100644
--- a/test/core/iomgr/tcp_client_posix_test.c
+++ b/test/core/iomgr/tcp_client_posix_test.c
@@ -113,7 +113,7 @@ void test_succeeds(void) {
   /* connect to it */
   GPR_ASSERT(getsockname(svr_fd, (struct sockaddr *)addr,
                          (socklen_t *)&resolved_addr.len) == 0);
-  grpc_closure_init(&done, must_succeed, NULL);
+  grpc_closure_init(&done, must_succeed, NULL, grpc_schedule_on_exec_ctx);
   grpc_tcp_client_connect(&exec_ctx, &done, &g_connecting, g_pollset_set, NULL,
                           &resolved_addr, gpr_inf_future(GPR_CLOCK_REALTIME));
 
@@ -163,7 +163,7 @@ void test_fails(void) {
   gpr_mu_unlock(g_mu);
 
   /* connect to a broken address */
-  grpc_closure_init(&done, must_fail, NULL);
+  grpc_closure_init(&done, must_fail, NULL, grpc_schedule_on_exec_ctx);
   grpc_tcp_client_connect(&exec_ctx, &done, &g_connecting, g_pollset_set, NULL,
                           &resolved_addr, gpr_inf_future(GPR_CLOCK_REALTIME));
 
@@ -207,7 +207,7 @@ int main(int argc, char **argv) {
   gpr_log(GPR_ERROR, "End of first test");
   test_fails();
   grpc_pollset_set_destroy(g_pollset_set);
-  grpc_closure_init(&destroyed, destroy_pollset, g_pollset);
+  grpc_closure_init(&destroyed, destroy_pollset, g_pollset, grpc_schedule_on_exec_ctx);
   grpc_pollset_shutdown(&exec_ctx, g_pollset, &destroyed);
   grpc_exec_ctx_finish(&exec_ctx);
   grpc_shutdown();
diff --git a/test/core/iomgr/tcp_posix_test.c b/test/core/iomgr/tcp_posix_test.c
index 5eafa570bb..340b807047 100644
--- a/test/core/iomgr/tcp_posix_test.c
+++ b/test/core/iomgr/tcp_posix_test.c
@@ -194,7 +194,7 @@ static void read_test(size_t num_bytes, size_t slice_size) {
   state.read_bytes = 0;
   state.target_read_bytes = written_bytes;
   grpc_slice_buffer_init(&state.incoming);
-  grpc_closure_init(&state.read_cb, read_cb, &state);
+  grpc_closure_init(&state.read_cb, read_cb, &state, grpc_schedule_on_exec_ctx);
 
   grpc_endpoint_read(&exec_ctx, ep, &state.incoming, &state.read_cb);
 
@@ -245,7 +245,7 @@ static void large_read_test(size_t slice_size) {
   state.read_bytes = 0;
   state.target_read_bytes = (size_t)written_bytes;
   grpc_slice_buffer_init(&state.incoming);
-  grpc_closure_init(&state.read_cb, read_cb, &state);
+  grpc_closure_init(&state.read_cb, read_cb, &state, grpc_schedule_on_exec_ctx);
 
   grpc_endpoint_read(&exec_ctx, ep, &state.incoming, &state.read_cb);
 
@@ -384,7 +384,7 @@ static void write_test(size_t num_bytes, size_t slice_size) {
 
   grpc_slice_buffer_init(&outgoing);
   grpc_slice_buffer_addn(&outgoing, slices, num_blocks);
-  grpc_closure_init(&write_done_closure, write_done, &state);
+  grpc_closure_init(&write_done_closure, write_done, &state, grpc_schedule_on_exec_ctx);
 
   grpc_endpoint_write(&exec_ctx, ep, &outgoing, &write_done_closure);
   drain_socket_blocking(sv[0], num_bytes, num_bytes);
@@ -429,7 +429,7 @@ static void release_fd_test(size_t num_bytes, size_t slice_size) {
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
   grpc_closure fd_released_cb;
   int fd_released_done = 0;
-  grpc_closure_init(&fd_released_cb, &on_fd_released, &fd_released_done);
+  grpc_closure_init(&fd_released_cb, &on_fd_released, &fd_released_done, grpc_schedule_on_exec_ctx);
 
   gpr_log(GPR_INFO,
           "Release fd read_test of size %" PRIuPTR ", slice size %" PRIuPTR,
@@ -452,7 +452,7 @@ static void release_fd_test(size_t num_bytes, size_t slice_size) {
   state.read_bytes = 0;
   state.target_read_bytes = written_bytes;
   grpc_slice_buffer_init(&state.incoming);
-  grpc_closure_init(&state.read_cb, read_cb, &state);
+  grpc_closure_init(&state.read_cb, read_cb, &state, grpc_schedule_on_exec_ctx);
 
   grpc_endpoint_read(&exec_ctx, ep, &state.incoming, &state.read_cb);
 
@@ -561,7 +561,7 @@ int main(int argc, char **argv) {
   grpc_pollset_init(g_pollset, &g_mu);
   grpc_endpoint_tests(configs[0], g_pollset, g_mu);
   run_tests();
-  grpc_closure_init(&destroyed, destroy_pollset, g_pollset);
+  grpc_closure_init(&destroyed, destroy_pollset, g_pollset, grpc_schedule_on_exec_ctx);
   grpc_pollset_shutdown(&exec_ctx, g_pollset, &destroyed);
   grpc_exec_ctx_finish(&exec_ctx);
   grpc_shutdown();
diff --git a/test/core/iomgr/tcp_server_posix_test.c b/test/core/iomgr/tcp_server_posix_test.c
index 9a7810e227..020f005980 100644
--- a/test/core/iomgr/tcp_server_posix_test.c
+++ b/test/core/iomgr/tcp_server_posix_test.c
@@ -104,7 +104,7 @@ static void server_weak_ref_shutdown(grpc_exec_ctx *exec_ctx, void *arg,
 static void server_weak_ref_init(server_weak_ref *weak_ref) {
   weak_ref->server = NULL;
   grpc_closure_init(&weak_ref->server_shutdown, server_weak_ref_shutdown,
-                    weak_ref);
+                    weak_ref, grpc_schedule_on_exec_ctx);
 }
 
 /* Make weak_ref->server_shutdown a shutdown_starting cb on server.
@@ -366,7 +366,8 @@ int main(int argc, char **argv) {
   test_connect(1);
   test_connect(10);
 
-  grpc_closure_init(&destroyed, destroy_pollset, g_pollset);
+  grpc_closure_init(&destroyed, destroy_pollset, g_pollset,
+                    grpc_schedule_on_exec_ctx);
   grpc_pollset_shutdown(&exec_ctx, g_pollset, &destroyed);
   grpc_exec_ctx_finish(&exec_ctx);
   grpc_shutdown();
diff --git a/test/core/iomgr/udp_server_test.c b/test/core/iomgr/udp_server_test.c
index 9bea229466..67ef81fab2 100644
--- a/test/core/iomgr/udp_server_test.c
+++ b/test/core/iomgr/udp_server_test.c
@@ -234,7 +234,7 @@ int main(int argc, char **argv) {
   test_receive(1);
   test_receive(10);
 
-  grpc_closure_init(&destroyed, destroy_pollset, g_pollset);
+  grpc_closure_init(&destroyed, destroy_pollset, g_pollset, grpc_schedule_on_exec_ctx);
   grpc_pollset_shutdown(&exec_ctx, g_pollset, &destroyed);
   grpc_exec_ctx_finish(&exec_ctx);
   gpr_free(g_pollset);