Merge pull request #7047 from daniel-j-born/server_expand_addrs

Add a flag that causes wildcard IP addresses on servers to be expanded

2 files changed, 236 insertions, 85 deletions

diff --git a/test/core/handshake/client_ssl.c b/test/core/handshake/client_ssl.c
index a22133e629..1a06fd6255 100644
--- a/test/core/handshake/client_ssl.c
+++ b/test/core/handshake/client_ssl.c
@@ -59,13 +59,17 @@ typedef struct {
   char *alpn_preferred;
 } server_args;
 
-// From https://wiki.openssl.org/index.php/Simple_TLS_Server.
-static int create_socket(int port) {
+// Based on https://wiki.openssl.org/index.php/Simple_TLS_Server.
+// Pick an arbitrary unused port and return it in *out_port. Return
+// an fd>=0 on success.
+static int create_socket(int *out_port) {
   int s;
   struct sockaddr_in addr;
+  socklen_t addr_len;
+  *out_port = -1;
 
   addr.sin_family = AF_INET;
-  addr.sin_port = htons((uint16_t)port);
+  addr.sin_port = 0;
   addr.sin_addr.s_addr = htonl(INADDR_ANY);
 
   s = socket(AF_INET, SOCK_STREAM, 0);
@@ -76,7 +80,7 @@ static int create_socket(int port) {
 
   if (bind(s, (struct sockaddr *)&addr, sizeof(addr)) < 0) {
     perror("Unable to bind");
-    gpr_log(GPR_ERROR, "Unable to bind to %d", port);
+    gpr_log(GPR_ERROR, "%s", "Unable to bind to any port");
     close(s);
     return -1;
   }
@@ -87,6 +91,16 @@ static int create_socket(int port) {
     return -1;
   }
 
+  addr_len = sizeof(addr);
+  if (getsockname(s, (struct sockaddr *)&addr, &addr_len) != 0 ||
+      addr_len > sizeof(addr)) {
+    perror("getsockname");
+    gpr_log(GPR_ERROR, "%s", "Unable to get socket local address");
+    close(s);
+    return -1;
+  }
+
+  *out_port = ntohs(addr.sin_port);
   return s;
 }
 
@@ -215,13 +229,12 @@ static bool client_ssl_test(char *server_alpn_preferred) {
   int server_socket = -1;
   int socket_retries = 30;
   while (server_socket == -1 && socket_retries-- > 0) {
-    port = grpc_pick_unused_port_or_die();
-    server_socket = create_socket(port);
+    server_socket = create_socket(&port);
     if (server_socket == -1) {
       sleep(1);
     }
   }
-  GPR_ASSERT(server_socket > 0);
+  GPR_ASSERT(server_socket > 0 && port > 0);
 
   // Launch the TLS server thread.
   gpr_thd_options thdopt = gpr_thd_options_default();
diff --git a/test/core/iomgr/tcp_server_posix_test.c b/test/core/iomgr/tcp_server_posix_test.c
index 7122cd1e86..4a6570015f 100644
--- a/test/core/iomgr/tcp_server_posix_test.c
+++ b/test/core/iomgr/tcp_server_posix_test.c
@@ -39,9 +39,12 @@
 #include "src/core/lib/iomgr/tcp_server.h"
 
 #include <errno.h>
+#include <ifaddrs.h>
 #include <netinet/in.h>
+#include <stdio.h>
 #include <string.h>
 #include <sys/socket.h>
+#include <sys/types.h>
 #include <unistd.h>
 
 #include <grpc/grpc.h>
@@ -50,6 +53,7 @@
 #include <grpc/support/sync.h>
 #include <grpc/support/time.h>
 
+#include "src/core/lib/iomgr/error.h"
 #include "src/core/lib/iomgr/iomgr.h"
 #include "src/core/lib/iomgr/resolve_address.h"
 #include "src/core/lib/iomgr/sockaddr_utils.h"
@@ -62,7 +66,7 @@ static gpr_mu *g_mu;
 static grpc_pollset *g_pollset;
 static int g_nconnects = 0;
 
-typedef struct on_connect_result {
+typedef struct {
   /* Owns a ref to server. */
   grpc_tcp_server *server;
   unsigned port_index;
@@ -70,15 +74,41 @@ typedef struct on_connect_result {
   int server_fd;
 } on_connect_result;
 
-typedef struct server_weak_ref {
+typedef struct {
   grpc_tcp_server *server;
 
   /* arg is this server_weak_ref. */
   grpc_closure server_shutdown;
 } server_weak_ref;
 
+#define MAX_URI 1024
+typedef struct {
+  grpc_resolved_address addr;
+  char str[MAX_URI];
+} test_addr;
+
+#define MAX_ADDRS 100
+typedef struct {
+  size_t naddrs;
+  test_addr addrs[MAX_ADDRS];
+} test_addrs;
+
 static on_connect_result g_result = {NULL, 0, 0, -1};
 
+static char family_name_buf[1024];
+static const char *sock_family_name(int family) {
+  if (family == AF_INET) {
+    return "AF_INET";
+  } else if (family == AF_INET6) {
+    return "AF_INET6";
+  } else if (family == AF_UNSPEC) {
+    return "AF_UNSPEC";
+  } else {
+    sprintf(family_name_buf, "%d", family);
+    return family_name_buf;
+  }
+}
+
 static void on_connect_result_init(on_connect_result *result) {
   result->server = NULL;
   result->port_index = 0;
@@ -118,6 +148,18 @@ static void server_weak_ref_set(server_weak_ref *weak_ref,
   weak_ref->server = server;
 }
 
+static void test_addr_init_str(test_addr *addr) {
+  char *str = NULL;
+  if (grpc_sockaddr_to_string(&str, &addr->addr, 0) != -1) {
+    size_t str_len;
+    memcpy(addr->str, str, (str_len = strnlen(str, sizeof(addr->str) - 1)));
+    addr->str[str_len] = '\0';
+    gpr_free(str);
+  } else {
+    addr->str[0] = '\0';
+  }
+}
+
 static void on_connect(grpc_exec_ctx *exec_ctx, void *arg, grpc_endpoint *tcp,
                        grpc_pollset *pollset,
                        grpc_tcp_server_acceptor *acceptor) {
@@ -168,7 +210,7 @@ static void test_no_op_with_port(void) {
   memset(&resolved_addr, 0, sizeof(resolved_addr));
   resolved_addr.len = sizeof(struct sockaddr_in);
   addr->sin_family = AF_INET;
-  int port;
+  int port = -1;
   GPR_ASSERT(grpc_tcp_server_add_port(s, &resolved_addr, &port) ==
                  GRPC_ERROR_NONE &&
              port > 0);
@@ -185,7 +227,7 @@ static void test_no_op_with_port_and_start(void) {
   GPR_ASSERT(GRPC_ERROR_NONE ==
              grpc_tcp_server_create(&exec_ctx, NULL, NULL, &s));
   LOG_TEST("test_no_op_with_port_and_start");
-  int port;
+  int port = -1;
 
   memset(&resolved_addr, 0, sizeof(resolved_addr));
   resolved_addr.len = sizeof(struct sockaddr_in);
@@ -200,74 +242,115 @@ static void test_no_op_with_port_and_start(void) {
   grpc_exec_ctx_finish(&exec_ctx);
 }
 
-static void tcp_connect(grpc_exec_ctx *exec_ctx, const struct sockaddr *remote,
-                        socklen_t remote_len, on_connect_result *result) {
+static grpc_error *tcp_connect(grpc_exec_ctx *exec_ctx, const test_addr *remote,
+                               on_connect_result *result) {
   gpr_timespec deadline = grpc_timeout_seconds_to_deadline(10);
-  int clifd = socket(remote->sa_family, SOCK_STREAM, 0);
+  int clifd;
   int nconnects_before;
+  const struct sockaddr *remote_addr =
+      (const struct sockaddr *)remote->addr.addr;
 
+  gpr_log(GPR_INFO, "Connecting to %s", remote->str);
   gpr_mu_lock(g_mu);
   nconnects_before = g_nconnects;
   on_connect_result_init(&g_result);
-  GPR_ASSERT(clifd >= 0);
-  gpr_log(GPR_DEBUG, "start connect");
-  GPR_ASSERT(connect(clifd, remote, remote_len) == 0);
+  clifd = socket(remote_addr->sa_family, SOCK_STREAM, 0);
+  if (clifd < 0) {
+    gpr_mu_unlock(g_mu);
+    return GRPC_OS_ERROR(errno, "Failed to create socket");
+  }
+  gpr_log(GPR_DEBUG, "start connect to %s", remote->str);
+  if (connect(clifd, remote_addr, (socklen_t)remote->addr.len) != 0) {
+    gpr_mu_unlock(g_mu);
+    close(clifd);
+    return GRPC_OS_ERROR(errno, "connect");
+  }
   gpr_log(GPR_DEBUG, "wait");
   while (g_nconnects == nconnects_before &&
          gpr_time_cmp(deadline, gpr_now(deadline.clock_type)) > 0) {
     grpc_pollset_worker *worker = NULL;
-    GPR_ASSERT(GRPC_LOG_IF_ERROR(
-        "pollset_work",
-        grpc_pollset_work(exec_ctx, g_pollset, &worker,
-                          gpr_now(GPR_CLOCK_MONOTONIC), deadline)));
+    grpc_error *err;
+    if ((err = grpc_pollset_work(exec_ctx, g_pollset, &worker,
+                                 gpr_now(GPR_CLOCK_MONOTONIC), deadline)) !=
+        GRPC_ERROR_NONE) {
+      gpr_mu_unlock(g_mu);
+      close(clifd);
+      return err;
+    }
     gpr_mu_unlock(g_mu);
     grpc_exec_ctx_finish(exec_ctx);
     gpr_mu_lock(g_mu);
   }
   gpr_log(GPR_DEBUG, "wait done");
-  GPR_ASSERT(g_nconnects == nconnects_before + 1);
+  if (g_nconnects != nconnects_before + 1) {
+    gpr_mu_unlock(g_mu);
+    close(clifd);
+    return GRPC_ERROR_CREATE("Didn't connect");
+  }
   close(clifd);
   *result = g_result;
 
   gpr_mu_unlock(g_mu);
+  gpr_log(GPR_INFO, "Result (%d, %d) fd %d", result->port_index,
+          result->fd_index, result->server_fd);
+  grpc_tcp_server_unref(exec_ctx, result->server);
+  return GRPC_ERROR_NONE;
 }
 
-/* Tests a tcp server with multiple ports. TODO(daniel-j-born): Multiple fds for
-   the same port should be tested. */
-static void test_connect(unsigned n) {
+/* Tests a tcp server on "::" listeners with multiple ports. If channel_args is
+   non-NULL, pass them to the server. If dst_addrs is non-NULL, use valid addrs
+   as destination addrs (port is not set). If dst_addrs is NULL, use listener
+   addrs as destination addrs. If test_dst_addrs is true, test connectivity with
+   each destination address, set grpc_resolved_address::len=0 for failures, but
+   don't fail the overall unitest. */
+static void test_connect(size_t num_connects,
+                         const grpc_channel_args *channel_args,
+                         test_addrs *dst_addrs, bool test_dst_addrs) {
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
   grpc_resolved_address resolved_addr;
   grpc_resolved_address resolved_addr1;
-  struct sockaddr_storage *addr = (struct sockaddr_storage *)resolved_addr.addr;
-  struct sockaddr_storage *addr1 =
+  struct sockaddr_storage *const addr =
+      (struct sockaddr_storage *)resolved_addr.addr;
+  struct sockaddr_storage *const addr1 =
       (struct sockaddr_storage *)resolved_addr1.addr;
   unsigned svr_fd_count;
+  int port;
   int svr_port;
   unsigned svr1_fd_count;
   int svr1_port;
   grpc_tcp_server *s;
+  const unsigned num_ports = 2;
   GPR_ASSERT(GRPC_ERROR_NONE ==
-             grpc_tcp_server_create(&exec_ctx, NULL, NULL, &s));
-  unsigned i;
+             grpc_tcp_server_create(&exec_ctx, NULL, channel_args, &s));
+  unsigned port_num;
   server_weak_ref weak_ref;
   server_weak_ref_init(&weak_ref);
+  server_weak_ref_set(&weak_ref, s);
   LOG_TEST("test_connect");
-  gpr_log(GPR_INFO, "clients=%d", n);
+  gpr_log(GPR_INFO,
+          "clients=%lu, num chan args=%lu, remote IP=%s, test_dst_addrs=%d",
+          (unsigned long)num_connects,
+          (unsigned long)(channel_args != NULL ? channel_args->num_args : 0),
+          dst_addrs != NULL ? "<specific>" : "::", test_dst_addrs);
   memset(&resolved_addr, 0, sizeof(resolved_addr));
   memset(&resolved_addr1, 0, sizeof(resolved_addr1));
   resolved_addr.len = sizeof(struct sockaddr_storage);
   resolved_addr1.len = sizeof(struct sockaddr_storage);
   addr->ss_family = addr1->ss_family = AF_INET;
-  GPR_ASSERT(GRPC_ERROR_NONE ==
-             grpc_tcp_server_add_port(s, &resolved_addr, &svr_port));
+  GPR_ASSERT(GRPC_LOG_IF_ERROR(
+      "grpc_tcp_server_add_port",
+      grpc_tcp_server_add_port(s, &resolved_addr, &svr_port)));
+  gpr_log(GPR_INFO, "Allocated port %d", svr_port);
   GPR_ASSERT(svr_port > 0);
   /* Cannot use wildcard (port==0), because add_port() will try to reuse the
      same port as a previous add_port(). */
   svr1_port = grpc_pick_unused_port_or_die();
+  GPR_ASSERT(svr1_port > 0);
+  gpr_log(GPR_INFO, "Picked unused port %d", svr1_port);
   grpc_sockaddr_set_port(&resolved_addr1, svr1_port);
-  GPR_ASSERT(grpc_tcp_server_add_port(s, &resolved_addr1, &svr_port) ==
+  GPR_ASSERT(grpc_tcp_server_add_port(s, &resolved_addr1, &port) ==
                  GRPC_ERROR_NONE &&
-             svr_port == svr1_port);
+             port == svr1_port);
 
   /* Bad port_index. */
   GPR_ASSERT(grpc_tcp_server_port_fd_count(s, 2) == 0);
@@ -283,58 +366,70 @@ static void test_connect(unsigned n) {
   svr1_fd_count = grpc_tcp_server_port_fd_count(s, 1);
   GPR_ASSERT(svr1_fd_count >= 1);
 
-  for (i = 0; i < svr_fd_count; ++i) {
-    int fd = grpc_tcp_server_port_fd(s, 0, i);
-    GPR_ASSERT(fd >= 0);
-    if (i == 0) {
-      GPR_ASSERT(getsockname(fd, (struct sockaddr *)addr,
-                             (socklen_t *)&resolved_addr.len) == 0);
-      GPR_ASSERT(resolved_addr.len <= sizeof(*addr));
-    }
-  }
-  for (i = 0; i < svr1_fd_count; ++i) {
-    int fd = grpc_tcp_server_port_fd(s, 1, i);
-    GPR_ASSERT(fd >= 0);
-    if (i == 0) {
-      GPR_ASSERT(getsockname(fd, (struct sockaddr *)addr1,
-                             (socklen_t *)&resolved_addr1.len) == 0);
-      GPR_ASSERT(resolved_addr1.len <= sizeof(*addr1));
-    }
-  }
-
   grpc_tcp_server_start(&exec_ctx, s, &g_pollset, 1, on_connect, NULL);
 
-  for (i = 0; i < n; i++) {
-    on_connect_result result;
-    int svr_fd;
-    on_connect_result_init(&result);
-    tcp_connect(&exec_ctx, (struct sockaddr *)addr,
-                (socklen_t)resolved_addr.len, &result);
-    GPR_ASSERT(result.server_fd >= 0);
-    svr_fd = result.server_fd;
-    GPR_ASSERT(grpc_tcp_server_port_fd(s, result.port_index, result.fd_index) ==
-               result.server_fd);
-    GPR_ASSERT(result.port_index == 0);
-    GPR_ASSERT(result.fd_index < svr_fd_count);
-    GPR_ASSERT(result.server == s);
-    if (weak_ref.server == NULL) {
-      server_weak_ref_set(&weak_ref, result.server);
+  if (dst_addrs != NULL) {
+    int ports[] = {svr_port, svr1_port};
+    for (port_num = 0; port_num < num_ports; ++port_num) {
+      size_t dst_idx;
+      size_t num_tested = 0;
+      for (dst_idx = 0; dst_idx < dst_addrs->naddrs; ++dst_idx) {
+        test_addr dst = dst_addrs->addrs[dst_idx];
+        on_connect_result result;
+        grpc_error *err;
+        if (dst.addr.len == 0) {
+          gpr_log(GPR_DEBUG, "Skipping test of non-functional local IP %s",
+                  dst.str);
+          continue;
+        }
+        GPR_ASSERT(grpc_sockaddr_set_port(&dst.addr, ports[port_num]));
+        test_addr_init_str(&dst);
+        ++num_tested;
+        on_connect_result_init(&result);
+        if ((err = tcp_connect(&exec_ctx, &dst, &result)) == GRPC_ERROR_NONE &&
+            result.server_fd >= 0 && result.server == s) {
+          continue;
+        }
+        gpr_log(GPR_ERROR, "Failed to connect to %s: %s", dst.str,
+                grpc_error_string(err));
+        GPR_ASSERT(test_dst_addrs);
+        dst_addrs->addrs[dst_idx].addr.len = 0;
+        GRPC_ERROR_UNREF(err);
+      }
+      GPR_ASSERT(num_tested > 0);
+    }
+  } else {
+    for (port_num = 0; port_num < num_ports; ++port_num) {
+      const unsigned num_fds = grpc_tcp_server_port_fd_count(s, port_num);
+      unsigned fd_num;
+      for (fd_num = 0; fd_num < num_fds; ++fd_num) {
+        int fd = grpc_tcp_server_port_fd(s, port_num, fd_num);
+        size_t connect_num;
+        test_addr dst;
+        GPR_ASSERT(fd >= 0);
+        dst.addr.len = sizeof(dst.addr.addr);
+        GPR_ASSERT(getsockname(fd, (struct sockaddr *)dst.addr.addr,
+                               (socklen_t *)&dst.addr.len) == 0);
+        GPR_ASSERT(dst.addr.len <= sizeof(dst.addr.addr));
+        test_addr_init_str(&dst);
+        gpr_log(GPR_INFO, "(%d, %d) fd %d family %s listening on %s", port_num,
+                fd_num, fd, sock_family_name(addr->ss_family), dst.str);
+        for (connect_num = 0; connect_num < num_connects; ++connect_num) {
+          on_connect_result result;
+          on_connect_result_init(&result);
+          GPR_ASSERT(GRPC_LOG_IF_ERROR("tcp_connect",
+                                       tcp_connect(&exec_ctx, &dst, &result)));
+          GPR_ASSERT(result.server_fd == fd);
+          GPR_ASSERT(result.port_index == port_num);
+          GPR_ASSERT(result.fd_index == fd_num);
+          GPR_ASSERT(result.server == s);
+          GPR_ASSERT(
+              grpc_tcp_server_port_fd(s, result.port_index, result.fd_index) ==
+              result.server_fd);
+        }
+      }
     }
-    grpc_tcp_server_unref(&exec_ctx, result.server);
-
-    on_connect_result_init(&result);
-    tcp_connect(&exec_ctx, (struct sockaddr *)addr1,
-                (socklen_t)resolved_addr1.len, &result);
-    GPR_ASSERT(result.server_fd >= 0);
-    GPR_ASSERT(result.server_fd != svr_fd);
-    GPR_ASSERT(grpc_tcp_server_port_fd(s, result.port_index, result.fd_index) ==
-               result.server_fd);
-    GPR_ASSERT(result.port_index == 1);
-    GPR_ASSERT(result.fd_index < svr_fd_count);
-    GPR_ASSERT(result.server == s);
-    grpc_tcp_server_unref(&exec_ctx, result.server);
   }
-
   /* Weak ref to server valid until final unref. */
   GPR_ASSERT(weak_ref.server != NULL);
   GPR_ASSERT(grpc_tcp_server_port_fd(s, 0, 0) >= 0);
@@ -354,6 +449,12 @@ static void destroy_pollset(grpc_exec_ctx *exec_ctx, void *p,
 int main(int argc, char **argv) {
   grpc_closure destroyed;
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
+  grpc_arg chan_args[] = {
+      {GRPC_ARG_INTEGER, GRPC_ARG_EXPAND_WILDCARD_ADDRS, {.integer = 1}}};
+  const grpc_channel_args channel_args = {1, chan_args};
+  struct ifaddrs *ifa = NULL;
+  struct ifaddrs *ifa_it;
+  test_addrs dst_addrs;
   grpc_test_init(argc, argv);
   grpc_init();
   g_pollset = gpr_malloc(grpc_pollset_size());
@@ -363,8 +464,45 @@ int main(int argc, char **argv) {
   test_no_op_with_start();
   test_no_op_with_port();
   test_no_op_with_port_and_start();
-  test_connect(1);
-  test_connect(10);
+
+  if (getifaddrs(&ifa) != 0 || ifa == NULL) {
+    gpr_log(GPR_ERROR, "getifaddrs: %s", strerror(errno));
+    return EXIT_FAILURE;
+  }
+  dst_addrs.naddrs = 0;
+  for (ifa_it = ifa; ifa_it != NULL && dst_addrs.naddrs < MAX_ADDRS;
+       ifa_it = ifa_it->ifa_next) {
+    if (ifa_it->ifa_addr == NULL) {
+      continue;
+    } else if (ifa_it->ifa_addr->sa_family == AF_INET) {
+      dst_addrs.addrs[dst_addrs.naddrs].addr.len = sizeof(struct sockaddr_in);
+    } else if (ifa_it->ifa_addr->sa_family == AF_INET6) {
+      dst_addrs.addrs[dst_addrs.naddrs].addr.len = sizeof(struct sockaddr_in6);
+    } else {
+      continue;
+    }
+    memcpy(dst_addrs.addrs[dst_addrs.naddrs].addr.addr, ifa_it->ifa_addr,
+           dst_addrs.addrs[dst_addrs.naddrs].addr.len);
+    GPR_ASSERT(
+        grpc_sockaddr_set_port(&dst_addrs.addrs[dst_addrs.naddrs].addr, 0));
+    test_addr_init_str(&dst_addrs.addrs[dst_addrs.naddrs]);
+    ++dst_addrs.naddrs;
+  }
+  freeifaddrs(ifa);
+  ifa = NULL;
+
+  /* Connect to same addresses as listeners. */
+  test_connect(1, NULL, NULL, false);
+  test_connect(10, NULL, NULL, false);
+
+  /* Set dst_addrs.addrs[i].len=0 for dst_addrs that are unreachable with a "::"
+     listener. */
+  test_connect(1, NULL, &dst_addrs, true);
+
+  /* Test connect(2) with dst_addrs. */
+  test_connect(1, &channel_args, &dst_addrs, false);
+  /* Test connect(2) with dst_addrs. */
+  test_connect(10, &channel_args, &dst_addrs, false);
 
   grpc_closure_init(&destroyed, destroy_pollset, g_pollset,
                     grpc_schedule_on_exec_ctx);
@@ -372,7 +510,7 @@ int main(int argc, char **argv) {
   grpc_exec_ctx_finish(&exec_ctx);
   grpc_shutdown();
   gpr_free(g_pollset);
-  return 0;
+  return EXIT_SUCCESS;
 }
 
 #else /* GRPC_POSIX_SOCKET */