1 files changed, 26 insertions, 390 deletions

diff --git a/src/core/lib/iomgr/tcp_server_posix.c b/src/core/lib/iomgr/tcp_server_posix.c
index 5f286a6723..e242631fc0 100644
--- a/src/core/lib/iomgr/tcp_server_posix.c
+++ b/src/core/lib/iomgr/tcp_server_posix.c
@@ -44,11 +44,8 @@
 
 #include <errno.h>
 #include <fcntl.h>
-#include <ifaddrs.h>
-#include <limits.h>
 #include <netinet/in.h>
 #include <netinet/tcp.h>
-#include <stdio.h>
 #include <string.h>
 #include <sys/socket.h>
 #include <sys/stat.h>
@@ -67,82 +64,10 @@
 #include "src/core/lib/iomgr/sockaddr_utils.h"
 #include "src/core/lib/iomgr/socket_utils_posix.h"
 #include "src/core/lib/iomgr/tcp_posix.h"
+#include "src/core/lib/iomgr/tcp_server_utils_posix.h"
 #include "src/core/lib/iomgr/unix_sockets_posix.h"
 #include "src/core/lib/support/string.h"
 
-#define MIN_SAFE_ACCEPT_QUEUE_SIZE 100
-
-static gpr_once s_init_max_accept_queue_size;
-static int s_max_accept_queue_size;
-
-/* one listening port */
-typedef struct grpc_tcp_listener grpc_tcp_listener;
-struct grpc_tcp_listener {
-  int fd;
-  grpc_fd *emfd;
-  grpc_tcp_server *server;
-  grpc_resolved_address addr;
-  int port;
-  unsigned port_index;
-  unsigned fd_index;
-  grpc_closure read_closure;
-  grpc_closure destroyed_closure;
-  struct grpc_tcp_listener *next;
-  /* sibling is a linked list of all listeners for a given port. add_port and
-     clone_port place all new listeners in the same sibling list. A member of
-     the 'sibling' list is also a member of the 'next' list. The head of each
-     sibling list has is_sibling==0, and subsequent members of sibling lists
-     have is_sibling==1. is_sibling allows separate sibling lists to be
-     identified while iterating through 'next'. */
-  struct grpc_tcp_listener *sibling;
-  int is_sibling;
-};
-
-/* the overall server */
-struct grpc_tcp_server {
-  gpr_refcount refs;
-  /* Called whenever accept() succeeds on a server port. */
-  grpc_tcp_server_cb on_accept_cb;
-  void *on_accept_cb_arg;
-
-  gpr_mu mu;
-
-  /* active port count: how many ports are actually still listening */
-  size_t active_ports;
-  /* destroyed port count: how many ports are completely destroyed */
-  size_t destroyed_ports;
-
-  /* is this server shutting down? */
-  bool shutdown;
-  /* have listeners been shutdown? */
-  bool shutdown_listeners;
-  /* use SO_REUSEPORT */
-  bool so_reuseport;
-  /* expand wildcard addresses to a list of all local addresses */
-  bool expand_wildcard_addrs;
-
-  /* linked list of server ports */
-  grpc_tcp_listener *head;
-  grpc_tcp_listener *tail;
-  unsigned nports;
-
-  /* List of closures passed to shutdown_starting_add(). */
-  grpc_closure_list shutdown_starting;
-
-  /* shutdown callback */
-  grpc_closure *shutdown_complete;
-
-  /* all pollsets interested in new connections */
-  grpc_pollset **pollsets;
-  /* number of pollsets in the pollsets array */
-  size_t pollset_count;
-
-  /* next pollset to assign a channel to */
-  gpr_atm next_pollset_to_assign;
-
-  grpc_resource_quota *resource_quota;
-};
-
 static gpr_once check_init = GPR_ONCE_INIT;
 static bool has_so_reuseport = false;
 
@@ -301,99 +226,6 @@ static void tcp_server_destroy(grpc_exec_ctx *exec_ctx, grpc_tcp_server *s) {
   }
 }
 
-/* get max listen queue size on linux */
-static void init_max_accept_queue_size(void) {
-  int n = SOMAXCONN;
-  char buf[64];
-  FILE *fp = fopen("/proc/sys/net/core/somaxconn", "r");
-  if (fp == NULL) {
-    /* 2.4 kernel. */
-    s_max_accept_queue_size = SOMAXCONN;
-    return;
-  }
-  if (fgets(buf, sizeof buf, fp)) {
-    char *end;
-    long i = strtol(buf, &end, 10);
-    if (i > 0 && i <= INT_MAX && end && *end == 0) {
-      n = (int)i;
-    }
-  }
-  fclose(fp);
-  s_max_accept_queue_size = n;
-
-  if (s_max_accept_queue_size < MIN_SAFE_ACCEPT_QUEUE_SIZE) {
-    gpr_log(GPR_INFO,
-            "Suspiciously small accept queue (%d) will probably lead to "
-            "connection drops",
-            s_max_accept_queue_size);
-  }
-}
-
-static int get_max_accept_queue_size(void) {
-  gpr_once_init(&s_init_max_accept_queue_size, init_max_accept_queue_size);
-  return s_max_accept_queue_size;
-}
-
-/* Prepare a recently-created socket for listening. */
-static grpc_error *prepare_socket(int fd, const grpc_resolved_address *addr,
-                                  bool so_reuseport, int *port) {
-  grpc_resolved_address sockname_temp;
-  grpc_error *err = GRPC_ERROR_NONE;
-
-  GPR_ASSERT(fd >= 0);
-
-  if (so_reuseport && !grpc_is_unix_socket(addr)) {
-    err = grpc_set_socket_reuse_port(fd, 1);
-    if (err != GRPC_ERROR_NONE) goto error;
-  }
-
-  err = grpc_set_socket_nonblocking(fd, 1);
-  if (err != GRPC_ERROR_NONE) goto error;
-  err = grpc_set_socket_cloexec(fd, 1);
-  if (err != GRPC_ERROR_NONE) goto error;
-  if (!grpc_is_unix_socket(addr)) {
-    err = grpc_set_socket_low_latency(fd, 1);
-    if (err != GRPC_ERROR_NONE) goto error;
-    err = grpc_set_socket_reuse_addr(fd, 1);
-    if (err != GRPC_ERROR_NONE) goto error;
-  }
-  err = grpc_set_socket_no_sigpipe_if_possible(fd);
-  if (err != GRPC_ERROR_NONE) goto error;
-
-  GPR_ASSERT(addr->len < ~(socklen_t)0);
-  if (bind(fd, (struct sockaddr *)addr->addr, (socklen_t)addr->len) < 0) {
-    err = GRPC_OS_ERROR(errno, "bind");
-    goto error;
-  }
-
-  if (listen(fd, get_max_accept_queue_size()) < 0) {
-    err = GRPC_OS_ERROR(errno, "listen");
-    goto error;
-  }
-
-  sockname_temp.len = sizeof(struct sockaddr_storage);
-
-  if (getsockname(fd, (struct sockaddr *)sockname_temp.addr,
-                  (socklen_t *)&sockname_temp.len) < 0) {
-    err = GRPC_OS_ERROR(errno, "getsockname");
-    goto error;
-  }
-
-  *port = grpc_sockaddr_get_port(&sockname_temp);
-  return GRPC_ERROR_NONE;
-
-error:
-  GPR_ASSERT(err != GRPC_ERROR_NONE);
-  if (fd >= 0) {
-    close(fd);
-  }
-  grpc_error *ret = grpc_error_set_int(
-      GRPC_ERROR_CREATE_REFERENCING("Unable to configure socket", &err, 1),
-      GRPC_ERROR_INT_FD, fd);
-  GRPC_ERROR_UNREF(err);
-  return ret;
-}
-
 /* event manager callback when reads are ready */
 static void on_read(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *err) {
   grpc_tcp_listener *sp = arg;
@@ -477,216 +309,6 @@ error:
   }
 }
 
-static grpc_error *add_socket_to_server(grpc_tcp_server *s, int fd,
-                                        const grpc_resolved_address *addr,
-                                        unsigned port_index, unsigned fd_index,
-                                        grpc_tcp_listener **listener) {
-  grpc_tcp_listener *sp = NULL;
-  int port = -1;
-  char *addr_str;
-  char *name;
-
-  grpc_error *err = prepare_socket(fd, addr, s->so_reuseport, &port);
-  if (err == GRPC_ERROR_NONE) {
-    GPR_ASSERT(port > 0);
-    grpc_sockaddr_to_string(&addr_str, addr, 1);
-    gpr_asprintf(&name, "tcp-server-listener:%s", addr_str);
-    gpr_mu_lock(&s->mu);
-    s->nports++;
-    GPR_ASSERT(!s->on_accept_cb && "must add ports before starting server");
-    sp = gpr_malloc(sizeof(grpc_tcp_listener));
-    sp->next = NULL;
-    if (s->head == NULL) {
-      s->head = sp;
-    } else {
-      s->tail->next = sp;
-    }
-    s->tail = sp;
-    sp->server = s;
-    sp->fd = fd;
-    sp->emfd = grpc_fd_create(fd, name);
-    memcpy(&sp->addr, addr, sizeof(grpc_resolved_address));
-    sp->port = port;
-    sp->port_index = port_index;
-    sp->fd_index = fd_index;
-    sp->is_sibling = 0;
-    sp->sibling = NULL;
-    GPR_ASSERT(sp->emfd);
-    gpr_mu_unlock(&s->mu);
-    gpr_free(addr_str);
-    gpr_free(name);
-  }
-
-  *listener = sp;
-  return err;
-}
-
-/* If successful, add a listener to s for addr, set *dsmode for the socket, and
-   return the *listener. */
-static grpc_error *add_addr_to_server(grpc_tcp_server *s,
-                                      const grpc_resolved_address *addr,
-                                      unsigned port_index, unsigned fd_index,
-                                      grpc_dualstack_mode *dsmode,
-                                      grpc_tcp_listener **listener) {
-  grpc_resolved_address addr4_copy;
-  int fd;
-  grpc_error *err =
-      grpc_create_dualstack_socket(addr, SOCK_STREAM, 0, dsmode, &fd);
-  if (err != GRPC_ERROR_NONE) {
-    return err;
-  }
-  if (*dsmode == GRPC_DSMODE_IPV4 &&
-      grpc_sockaddr_is_v4mapped(addr, &addr4_copy)) {
-    addr = &addr4_copy;
-  }
-  return add_socket_to_server(s, fd, addr, port_index, fd_index, listener);
-}
-
-/* Bind to "::" to get a port number not used by any address. */
-static grpc_error *get_unused_port(int *port) {
-  grpc_resolved_address wild;
-  grpc_sockaddr_make_wildcard6(0, &wild);
-  grpc_dualstack_mode dsmode;
-  int fd;
-  grpc_error *err =
-      grpc_create_dualstack_socket(&wild, SOCK_STREAM, 0, &dsmode, &fd);
-  if (err != GRPC_ERROR_NONE) {
-    return err;
-  }
-  if (dsmode == GRPC_DSMODE_IPV4) {
-    grpc_sockaddr_make_wildcard4(0, &wild);
-  }
-  if (bind(fd, (const struct sockaddr *)wild.addr, (socklen_t)wild.len) != 0) {
-    err = GRPC_OS_ERROR(errno, "bind");
-    close(fd);
-    return err;
-  }
-  if (getsockname(fd, (struct sockaddr *)wild.addr, (socklen_t *)&wild.len) !=
-      0) {
-    err = GRPC_OS_ERROR(errno, "getsockname");
-    close(fd);
-    return err;
-  }
-  close(fd);
-  *port = grpc_sockaddr_get_port(&wild);
-  return *port <= 0 ? GRPC_ERROR_CREATE("Bad port") : GRPC_ERROR_NONE;
-}
-
-/* Return the listener in s with address addr or NULL. */
-static grpc_tcp_listener *find_listener_with_addr(grpc_tcp_server *s,
-                                                  grpc_resolved_address *addr) {
-  grpc_tcp_listener *l;
-  gpr_mu_lock(&s->mu);
-  for (l = s->head; l != NULL; l = l->next) {
-    if (l->addr.len != addr->len) {
-      continue;
-    }
-    if (memcmp(l->addr.addr, addr->addr, addr->len) == 0) {
-      break;
-    }
-  }
-  gpr_mu_unlock(&s->mu);
-  return l;
-}
-
-/* Get all addresses assigned to network interfaces on the machine and create a
-   listener for each. requested_port is the port to use for every listener, or 0
-   to select one random port that will be used for every listener. Set *out_port
-   to the port selected. Return GRPC_ERROR_NONE only if all listeners were
-   added. */
-static grpc_error *add_all_local_addrs_to_server(grpc_tcp_server *s,
-                                                 unsigned port_index,
-                                                 int requested_port,
-                                                 int *out_port) {
-  struct ifaddrs *ifa = NULL;
-  struct ifaddrs *ifa_it;
-  unsigned fd_index = 0;
-  grpc_tcp_listener *sp = NULL;
-  grpc_error *err = GRPC_ERROR_NONE;
-  if (requested_port == 0) {
-    /* Note: There could be a race where some local addrs can listen on the
-       selected port and some can't. The sane way to handle this would be to
-       retry by recreating the whole grpc_tcp_server. Backing out individual
-       listeners and orphaning the FDs looks like too much trouble. */
-    if ((err = get_unused_port(&requested_port)) != GRPC_ERROR_NONE) {
-      return err;
-    } else if (requested_port <= 0) {
-      return GRPC_ERROR_CREATE("Bad get_unused_port()");
-    }
-    gpr_log(GPR_DEBUG, "Picked unused port %d", requested_port);
-  }
-  if (getifaddrs(&ifa) != 0 || ifa == NULL) {
-    return GRPC_OS_ERROR(errno, "getifaddrs");
-  }
-  for (ifa_it = ifa; ifa_it != NULL; ifa_it = ifa_it->ifa_next) {
-    grpc_resolved_address addr;
-    char *addr_str = NULL;
-    grpc_dualstack_mode dsmode;
-    grpc_tcp_listener *new_sp = NULL;
-    const char *ifa_name = (ifa_it->ifa_name ? ifa_it->ifa_name : "<unknown>");
-    if (ifa_it->ifa_addr == NULL) {
-      continue;
-    } else if (ifa_it->ifa_addr->sa_family == AF_INET) {
-      addr.len = sizeof(struct sockaddr_in);
-    } else if (ifa_it->ifa_addr->sa_family == AF_INET6) {
-      addr.len = sizeof(struct sockaddr_in6);
-    } else {
-      continue;
-    }
-    memcpy(addr.addr, ifa_it->ifa_addr, addr.len);
-    if (!grpc_sockaddr_set_port(&addr, requested_port)) {
-      /* Should never happen, because we check sa_family above. */
-      err = GRPC_ERROR_CREATE("Failed to set port");
-      break;
-    }
-    if (grpc_sockaddr_to_string(&addr_str, &addr, 0) < 0) {
-      addr_str = gpr_strdup("<error>");
-    }
-    gpr_log(GPR_DEBUG,
-            "Adding local addr from interface %s flags 0x%x to server: %s",
-            ifa_name, ifa_it->ifa_flags, addr_str);
-    /* We could have multiple interfaces with the same address (e.g., bonding),
-       so look for duplicates. */
-    if (find_listener_with_addr(s, &addr) != NULL) {
-      gpr_log(GPR_DEBUG, "Skipping duplicate addr %s on interface %s", addr_str,
-              ifa_name);
-      gpr_free(addr_str);
-      continue;
-    }
-    if ((err = add_addr_to_server(s, &addr, port_index, fd_index, &dsmode,
-                                  &new_sp)) != GRPC_ERROR_NONE) {
-      char *err_str = NULL;
-      grpc_error *root_err;
-      if (gpr_asprintf(&err_str, "Failed to add listener: %s", addr_str) < 0) {
-        err_str = gpr_strdup("Failed to add listener");
-      }
-      root_err = GRPC_ERROR_CREATE(err_str);
-      gpr_free(err_str);
-      gpr_free(addr_str);
-      err = grpc_error_add_child(root_err, err);
-      break;
-    } else {
-      GPR_ASSERT(requested_port == new_sp->port);
-      ++fd_index;
-      if (sp != NULL) {
-        new_sp->is_sibling = 1;
-        sp->sibling = new_sp;
-      }
-      sp = new_sp;
-    }
-    gpr_free(addr_str);
-  }
-  freeifaddrs(ifa);
-  if (err != GRPC_ERROR_NONE) {
-    return err;
-  } else if (sp == NULL) {
-    return GRPC_ERROR_CREATE("No local addresses");
-  } else {
-    *out_port = sp->port;
-    return GRPC_ERROR_NONE;
-  }
-}
-
 /* Treat :: or 0.0.0.0 as a family-agnostic wildcard. */
 static grpc_error *add_wildcard_addrs_to_server(grpc_tcp_server *s,
                                                 unsigned port_index,
@@ -701,14 +323,16 @@ static grpc_error *add_wildcard_addrs_to_server(grpc_tcp_server *s,
   grpc_error *v6_err = GRPC_ERROR_NONE;
   grpc_error *v4_err = GRPC_ERROR_NONE;
   *out_port = -1;
-  if (s->expand_wildcard_addrs) {
-    return add_all_local_addrs_to_server(s, port_index, requested_port,
-                                         out_port);
+
+  if (grpc_tcp_server_have_ifaddrs() && s->expand_wildcard_addrs) {
+    return grpc_tcp_server_add_all_local_addrs(s, port_index, requested_port,
+                                               out_port);
   }
+
   grpc_sockaddr_make_wildcards(requested_port, &wild4, &wild6);
   /* Try listening on IPv6 first. */
-  if ((v6_err = add_addr_to_server(s, &wild6, port_index, fd_index, &dsmode,
-                                   &sp)) == GRPC_ERROR_NONE) {
+  if ((v6_err = grpc_tcp_server_add_addr(s, &wild6, port_index, fd_index,
+                                         &dsmode, &sp)) == GRPC_ERROR_NONE) {
     ++fd_index;
     requested_port = *out_port = sp->port;
     if (dsmode == GRPC_DSMODE_DUALSTACK || dsmode == GRPC_DSMODE_IPV4) {
@@ -717,8 +341,8 @@ static grpc_error *add_wildcard_addrs_to_server(grpc_tcp_server *s,
   }
   /* If we got a v6-only socket or nothing, try adding 0.0.0.0. */
   grpc_sockaddr_set_port(&wild4, requested_port);
-  if ((v4_err = add_addr_to_server(s, &wild4, port_index, fd_index, &dsmode,
-                                   &sp2)) == GRPC_ERROR_NONE) {
+  if ((v4_err = grpc_tcp_server_add_addr(s, &wild4, port_index, fd_index,
+                                         &dsmode, &sp2)) == GRPC_ERROR_NONE) {
     *out_port = sp2->port;
     if (sp != NULL) {
       sp2->is_sibling = 1;
@@ -726,8 +350,20 @@ static grpc_error *add_wildcard_addrs_to_server(grpc_tcp_server *s,
     }
   }
   if (*out_port > 0) {
-    GRPC_LOG_IF_ERROR("Failed to add :: listener", v6_err);
-    GRPC_LOG_IF_ERROR("Failed to add 0.0.0.0 listener", v4_err);
+    if (v6_err != GRPC_ERROR_NONE) {
+      gpr_log(GPR_INFO,
+              "Failed to add :: listener, "
+              "the environment may not support IPv6: %s",
+              grpc_error_string(v6_err));
+      GRPC_ERROR_UNREF(v6_err);
+    }
+    if (v4_err != GRPC_ERROR_NONE) {
+      gpr_log(GPR_INFO,
+              "Failed to add 0.0.0.0 listener, "
+              "the environment may not support IPv4: %s",
+              grpc_error_string(v4_err));
+      GRPC_ERROR_UNREF(v4_err);
+    }
     return GRPC_ERROR_NONE;
   } else {
     grpc_error *root_err =
@@ -756,7 +392,7 @@ static grpc_error *clone_port(grpc_tcp_listener *listener, unsigned count) {
     err = grpc_create_dualstack_socket(&listener->addr, SOCK_STREAM, 0, &dsmode,
                                        &fd);
     if (err != GRPC_ERROR_NONE) return err;
-    err = prepare_socket(fd, &listener->addr, true, &port);
+    err = grpc_tcp_server_prepare_socket(fd, &listener->addr, true, &port);
     if (err != GRPC_ERROR_NONE) return err;
     listener->server->nports++;
     grpc_sockaddr_to_string(&addr_str, &listener->addr, 1);
@@ -828,7 +464,7 @@ grpc_error *grpc_tcp_server_add_port(grpc_tcp_server *s,
   if (grpc_sockaddr_to_v4mapped(addr, &addr6_v4mapped)) {
     addr = &addr6_v4mapped;
   }
-  if ((err = add_addr_to_server(s, addr, port_index, 0, &dsmode, &sp)) ==
+  if ((err = grpc_tcp_server_add_addr(s, addr, port_index, 0, &dsmode, &sp)) ==
       GRPC_ERROR_NONE) {
     *out_port = sp->port;
   }