Merge pull request #3726 from dgquintas/iomgr_executor

Introducing grpc_executor, for all your threading needs

6 files changed, 240 insertions, 28 deletions

diff --git a/src/core/iomgr/closure.c b/src/core/iomgr/closure.c
index d91681990f..b4f1817de4 100644
--- a/src/core/iomgr/closure.c
+++ b/src/core/iomgr/closure.c
@@ -72,6 +72,16 @@ void grpc_closure_list_move(grpc_closure_list *src, grpc_closure_list *dst) {
   src->head = src->tail = NULL;
 }
 
+grpc_closure *grpc_closure_list_pop(grpc_closure_list *list) {
+  grpc_closure *head;
+  if (list->head == NULL) {
+    return NULL;
+  }
+  head = list->head;
+  list->head = list->head->next;
+  return head;
+}
+
 typedef struct {
   grpc_iomgr_cb_func cb;
   void *cb_arg;
diff --git a/src/core/iomgr/closure.h b/src/core/iomgr/closure.h
index d812659af0..7a9f7ccad0 100644
--- a/src/core/iomgr/closure.h
+++ b/src/core/iomgr/closure.h
@@ -83,9 +83,18 @@ grpc_closure *grpc_closure_create(grpc_iomgr_cb_func cb, void *cb_arg);
 #define GRPC_CLOSURE_LIST_INIT \
   { NULL, NULL }
 
+/** add \a closure to the end of \a list and set \a closure's success to \a
+ * success */
 void grpc_closure_list_add(grpc_closure_list *list, grpc_closure *closure,
                            int success);
+
+/** append all closures from \a src to \a dst and empty \a src. */
 void grpc_closure_list_move(grpc_closure_list *src, grpc_closure_list *dst);
+
+/** pop (return and remove) the head closure from \a list. */
+grpc_closure *grpc_closure_list_pop(grpc_closure_list *list);
+
+/** return whether \a list is empty. */
 int grpc_closure_list_empty(grpc_closure_list list);
 
 #endif /* GRPC_INTERNAL_CORE_IOMGR_CLOSURE_H */
diff --git a/src/core/iomgr/executor.c b/src/core/iomgr/executor.c
new file mode 100644
index 0000000000..457e5cdbac
--- /dev/null
+++ b/src/core/iomgr/executor.c
@@ -0,0 +1,148 @@
+/*
+ *
+ * Copyright 2015, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include "src/core/iomgr/executor.h"
+
+#include <string.h>
+
+#include <grpc/support/alloc.h>
+#include <grpc/support/log.h>
+#include <grpc/support/sync.h>
+#include <grpc/support/thd.h>
+#include "src/core/iomgr/exec_ctx.h"
+
+typedef struct grpc_executor_data {
+  int busy;          /**< is the thread currently running? */
+  int shutting_down; /**< has \a grpc_shutdown() been invoked? */
+  int pending_join;  /**< has the thread finished but not been joined? */
+  grpc_closure_list closures; /**< collection of pending work */
+  gpr_thd_id tid; /**< thread id of the thread, only valid if \a busy or \a
+                     pending_join are true */
+  gpr_thd_options options;
+  gpr_mu mu;
+} grpc_executor;
+
+static grpc_executor g_executor;
+
+void grpc_executor_init() {
+  memset(&g_executor, 0, sizeof(grpc_executor));
+  gpr_mu_init(&g_executor.mu);
+  g_executor.options = gpr_thd_options_default();
+  gpr_thd_options_set_joinable(&g_executor.options);
+}
+
+/* thread body */
+static void closure_exec_thread_func(void *ignored) {
+  grpc_closure *closure;
+
+  grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
+  while (1) {
+    gpr_mu_lock(&g_executor.mu);
+    if (g_executor.shutting_down != 0) {
+      gpr_mu_unlock(&g_executor.mu);
+      break;
+    }
+    closure = grpc_closure_list_pop(&g_executor.closures);
+    if (closure == NULL) {
+      /* no more work, time to die */
+      GPR_ASSERT(g_executor.busy == 1);
+      g_executor.busy = 0;
+      gpr_mu_unlock(&g_executor.mu);
+      break;
+    }
+    gpr_mu_unlock(&g_executor.mu);
+    closure->cb(&exec_ctx, closure->cb_arg, closure->success);
+    grpc_exec_ctx_flush(&exec_ctx);
+  }
+  grpc_exec_ctx_finish(&exec_ctx);
+}
+
+/* Spawn the thread if new work has arrived a no thread is up */
+static void maybe_spawn_locked() {
+  if (grpc_closure_list_empty(g_executor.closures) == 1) {
+    return;
+  }
+  if (g_executor.shutting_down == 1) {
+    return;
+  }
+
+  if (g_executor.busy != 0) {
+    /* Thread still working. New work will be picked up by already running
+     * thread. Not spawning anything. */
+    return;
+  } else if (g_executor.pending_join != 0) {
+    /* Pickup the remains of the previous incarnations of the thread. */
+    gpr_thd_join(g_executor.tid);
+    g_executor.pending_join = 0;
+  }
+
+  /* All previous instances of the thread should have been joined at this point.
+   * Spawn time! */
+  g_executor.busy = 1;
+  gpr_thd_new(&g_executor.tid, closure_exec_thread_func, NULL,
+              &g_executor.options);
+  g_executor.pending_join = 1;
+}
+
+void grpc_executor_enqueue(grpc_closure *closure, int success) {
+  gpr_mu_lock(&g_executor.mu);
+  if (g_executor.shutting_down == 0) {
+    grpc_closure_list_add(&g_executor.closures, closure, success);
+    maybe_spawn_locked();
+  }
+  gpr_mu_unlock(&g_executor.mu);
+}
+
+void grpc_executor_shutdown() {
+  int pending_join;
+  grpc_closure *closure;
+  grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
+
+  gpr_mu_lock(&g_executor.mu);
+  pending_join = g_executor.pending_join;
+  g_executor.shutting_down = 1;
+  gpr_mu_unlock(&g_executor.mu);
+  /* we can release the lock at this point despite the access to the closure
+   * list below because we aren't accepting new work */
+
+  /* Execute pending callbacks, some may be performing cleanups */
+  while ((closure = grpc_closure_list_pop(&g_executor.closures)) != NULL) {
+    closure->cb(&exec_ctx, closure->cb_arg, closure->success);
+  }
+  grpc_exec_ctx_finish(&exec_ctx);
+  GPR_ASSERT(grpc_closure_list_empty(g_executor.closures));
+  if (pending_join) {
+    gpr_thd_join(g_executor.tid);
+  }
+  gpr_mu_destroy(&g_executor.mu);
+}
diff --git a/src/core/iomgr/executor.h b/src/core/iomgr/executor.h
new file mode 100644
index 0000000000..6da446ae9c
--- /dev/null
+++ b/src/core/iomgr/executor.h
@@ -0,0 +1,53 @@
+/*
+ *
+ * Copyright 2015, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef GRPC_INTERNAL_CORE_IOMGR_EXECUTOR_H
+#define GRPC_INTERNAL_CORE_IOMGR_EXECUTOR_H
+
+#include "src/core/iomgr/closure.h"
+
+/** Initialize the global executor.
+ *
+ * This mechanism is meant to outsource work (grpc_closure instances) to a
+ * thread, for those cases where blocking isn't an option but there isn't a
+ * non-blocking solution available. */
+void grpc_executor_init();
+
+/** Enqueue \a closure for its eventual execution of \a f(arg) on a separate
+ * thread */
+void grpc_executor_enqueue(grpc_closure *closure, int success);
+
+/** Shutdown the executor, running all pending work as part of the call */
+void grpc_executor_shutdown();
+
+#endif /* GRPC_INTERNAL_CORE_IOMGR_EXECUTOR_H */
diff --git a/src/core/iomgr/resolve_address_posix.c b/src/core/iomgr/resolve_address_posix.c
index ed0a93fcc9..555c74ce7e 100644
--- a/src/core/iomgr/resolve_address_posix.c
+++ b/src/core/iomgr/resolve_address_posix.c
@@ -41,6 +41,7 @@
 #include <sys/un.h>
 #include <string.h>
 
+#include "src/core/iomgr/executor.h"
 #include "src/core/iomgr/iomgr_internal.h"
 #include "src/core/iomgr/sockaddr_utils.h"
 #include "src/core/support/block_annotate.h"
@@ -57,8 +58,8 @@ typedef struct {
   char *name;
   char *default_port;
   grpc_resolve_cb cb;
+  grpc_closure request_closure;
   void *arg;
-  grpc_iomgr_object iomgr_object;
 } request;
 
 grpc_resolved_addresses *grpc_blocking_resolve_address(
@@ -149,20 +150,18 @@ done:
   return addrs;
 }
 
-/* Thread function to asynch-ify grpc_blocking_resolve_address */
-static void do_request_thread(void *rp) {
+/* Callback to be passed to grpc_executor to asynch-ify
+ * grpc_blocking_resolve_address */
+static void do_request_thread(grpc_exec_ctx *exec_ctx, void *rp, int success) {
   request *r = rp;
-  grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
   grpc_resolved_addresses *resolved =
       grpc_blocking_resolve_address(r->name, r->default_port);
   void *arg = r->arg;
   grpc_resolve_cb cb = r->cb;
   gpr_free(r->name);
   gpr_free(r->default_port);
-  cb(&exec_ctx, arg, resolved);
-  grpc_iomgr_unregister_object(&r->iomgr_object);
+  cb(exec_ctx, arg, resolved);
   gpr_free(r);
-  grpc_exec_ctx_finish(&exec_ctx);
 }
 
 void grpc_resolved_addresses_destroy(grpc_resolved_addresses *addrs) {
@@ -173,17 +172,12 @@ void grpc_resolved_addresses_destroy(grpc_resolved_addresses *addrs) {
 void grpc_resolve_address(const char *name, const char *default_port,
                           grpc_resolve_cb cb, void *arg) {
   request *r = gpr_malloc(sizeof(request));
-  gpr_thd_id id;
-  char *tmp;
-  gpr_asprintf(&tmp, "resolve_address:name='%s':default_port='%s'", name,
-               default_port);
-  grpc_iomgr_register_object(&r->iomgr_object, tmp);
-  gpr_free(tmp);
+  grpc_closure_init(&r->request_closure, do_request_thread, r);
   r->name = gpr_strdup(name);
   r->default_port = gpr_strdup(default_port);
   r->cb = cb;
   r->arg = arg;
-  gpr_thd_new(&id, do_request_thread, r, NULL);
+  grpc_executor_enqueue(&r->request_closure, 1);
 }
 
 #endif
diff --git a/src/core/iomgr/resolve_address_windows.c b/src/core/iomgr/resolve_address_windows.c
index 82a5602996..007c855d10 100644
--- a/src/core/iomgr/resolve_address_windows.c
+++ b/src/core/iomgr/resolve_address_windows.c
@@ -40,6 +40,7 @@
 #include <sys/types.h>
 #include <string.h>
 
+#include "src/core/iomgr/executor.h"
 #include "src/core/iomgr/iomgr_internal.h"
 #include "src/core/iomgr/sockaddr_utils.h"
 #include "src/core/support/block_annotate.h"
@@ -47,6 +48,7 @@
 #include <grpc/support/alloc.h>
 #include <grpc/support/host_port.h>
 #include <grpc/support/log.h>
+#include <grpc/support/log_win32.h>
 #include <grpc/support/string_util.h>
 #include <grpc/support/thd.h>
 #include <grpc/support/time.h>
@@ -55,8 +57,8 @@ typedef struct {
   char *name;
   char *default_port;
   grpc_resolve_cb cb;
+  grpc_closure request_closure;
   void *arg;
-  grpc_iomgr_object iomgr_object;
 } request;
 
 grpc_resolved_addresses *grpc_blocking_resolve_address(
@@ -93,7 +95,9 @@ grpc_resolved_addresses *grpc_blocking_resolve_address(
   s = getaddrinfo(host, port, &hints, &result);
   GRPC_SCHEDULING_END_BLOCKING_REGION;
   if (s != 0) {
-    gpr_log(GPR_ERROR, "getaddrinfo: %s", gai_strerror(s));
+    char *error_message = gpr_format_message(s);
+    gpr_log(GPR_ERROR, "getaddrinfo: %s", error_message);
+    gpr_free(error_message);
     goto done;
   }
 
@@ -129,9 +133,9 @@ done:
   return addrs;
 }
 
-/* Thread function to asynch-ify grpc_blocking_resolve_address */
-static void do_request(void *rp) {
-  grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
+/* Callback to be passed to grpc_executor to asynch-ify
+ * grpc_blocking_resolve_address */
+static void do_request_thread(grpc_exec_ctx *exec_ctx, void *rp, int success) {
   request *r = rp;
   grpc_resolved_addresses *resolved =
       grpc_blocking_resolve_address(r->name, r->default_port);
@@ -139,10 +143,8 @@ static void do_request(void *rp) {
   grpc_resolve_cb cb = r->cb;
   gpr_free(r->name);
   gpr_free(r->default_port);
-  grpc_iomgr_unregister_object(&r->iomgr_object);
+  cb(exec_ctx, arg, resolved);
   gpr_free(r);
-  cb(&exec_ctx, arg, resolved);
-  grpc_exec_ctx_finish(&exec_ctx);
 }
 
 void grpc_resolved_addresses_destroy(grpc_resolved_addresses *addrs) {
@@ -153,16 +155,12 @@ void grpc_resolved_addresses_destroy(grpc_resolved_addresses *addrs) {
 void grpc_resolve_address(const char *name, const char *default_port,
                           grpc_resolve_cb cb, void *arg) {
   request *r = gpr_malloc(sizeof(request));
-  gpr_thd_id id;
-  char *label;
-  gpr_asprintf(&label, "resolve:%s", name);
-  grpc_iomgr_register_object(&r->iomgr_object, label);
-  gpr_free(label);
+  grpc_closure_init(&r->request_closure, do_request_thread, r);
   r->name = gpr_strdup(name);
   r->default_port = gpr_strdup(default_port);
   r->cb = cb;
   r->arg = arg;
-  gpr_thd_new(&id, do_request, r, NULL);
+  grpc_executor_enqueue(&r->request_closure, 1);
 }
 
 #endif