bundle: use userland resolver

- Fix SNI handling
- Add keylog fd to seccomp policy
- Add userland resolver to seccomp policy

Author: riptl

src/disco/bundle/fd_bundle_client.c

@@ -17,7 +17,7 @@
 #include <unistd.h> /* close */
 #include <poll.h> /* poll */
 #include <sys/socket.h> /* socket */
-#include <netinet/in.h> /* IPPROTO_TCP */
+#include <netinet/in.h>
 
 static void
 fd_bundle_client_reset( fd_bundle_tile_t * ctx ) {
@@ -58,10 +58,11 @@ fd_bundle_client_reset( fd_bundle_tile_t * ctx ) {
 }
 
 static int
-fd_bundle_client_do_connect( fd_bundle_tile_t const * ctx ) {
+fd_bundle_client_do_connect( fd_bundle_tile_t const * ctx,
+                             uint                     ip4_addr ) {
   struct sockaddr_in addr = {
     .sin_family      = AF_INET,
-    .sin_addr.s_addr = ctx->server_ip4_addr,
+    .sin_addr.s_addr = ip4_addr,
     .sin_port        = fd_ushort_bswap( ctx->server_tcp_port )
   };
   errno = 0;
@@ -76,9 +77,24 @@ static void
 fd_bundle_client_create_conn( fd_bundle_tile_t * ctx ) {
   fd_bundle_client_reset( ctx );
 
-  int tcp_sock = socket( AF_INET, SOCK_STREAM|SOCK_CLOEXEC, IPPROTO_TCP );
+  /* FIXME IPv6 support */
+  fd_addrinfo_t hints = {0};
+  hints.ai_family = AF_INET;
+  fd_addrinfo_t * res = NULL;
+  uchar scratch[ 4096 ];
+  void * pscratch = scratch;
+  int err = fd_getaddrinfo( ctx->server_fqdn, &hints, &res, &pscratch, sizeof(scratch) );
+  if( FD_UNLIKELY( err ) ) {
+    FD_LOG_WARNING(( "fd_getaddrinfo `%s` failed (%d-%s)", ctx->server_fqdn, err, fd_gai_strerror( err ) ));
+    fd_bundle_client_reset( ctx );
+    ctx->metrics.transport_fail_cnt++;
+    return;
+  }
+  uint const ip4_addr = ((struct sockaddr_in *)res->ai_addr)->sin_addr.s_addr;
+
+  int tcp_sock = socket( AF_INET, SOCK_STREAM|SOCK_CLOEXEC, 0 );
   if( FD_UNLIKELY( tcp_sock<0 ) ) {
-    FD_LOG_ERR(( "socket(AF_INET,SOCK_STREAM|SOCK_CLOEXEC,IPPROTO_TCP) failed (%i-%s)", errno, fd_io_strerror( errno ) ));
+    FD_LOG_ERR(( "socket(AF_INET,SOCK_STREAM|SOCK_CLOEXEC,0) failed (%i-%s)", errno, fd_io_strerror( errno ) ));
   }
   ctx->tcp_sock = tcp_sock;
 
@@ -97,14 +113,14 @@ fd_bundle_client_create_conn( fd_bundle_tile_t * ctx ) {
 
   FD_LOG_INFO(( "Connecting to %s://" FD_IP4_ADDR_FMT ":%hu (%.*s)",
                 scheme,
-                FD_IP4_ADDR_FMT_ARGS( ctx->server_ip4_addr ), ctx->server_tcp_port,
-                (int)ctx->server_fqdn_len, ctx->server_fqdn ));
+                FD_IP4_ADDR_FMT_ARGS( ip4_addr ), ctx->server_tcp_port,
+                (int)ctx->server_sni_len, ctx->server_sni ));
 
-  int connect_err = fd_bundle_client_do_connect( ctx );
+  int connect_err = fd_bundle_client_do_connect( ctx, ip4_addr );
   if( FD_UNLIKELY( connect_err ) ) {
     if( FD_UNLIKELY( connect_err!=EINPROGRESS ) ) {
       FD_LOG_WARNING(( "connect(tcp_sock," FD_IP4_ADDR_FMT ":%u) failed (%i-%s)",
-                      FD_IP4_ADDR_FMT_ARGS( ctx->server_ip4_addr ), ctx->server_tcp_port,
+                      FD_IP4_ADDR_FMT_ARGS( ip4_addr ), ctx->server_tcp_port,
                       connect_err, fd_io_strerror( connect_err ) ));
       fd_bundle_client_reset( ctx );
       ctx->metrics.transport_fail_cnt++;
@@ -127,7 +143,7 @@ fd_bundle_client_create_conn( fd_bundle_tile_t * ctx ) {
     SSL_set_bio( ssl, bio, bio ); /* moves ownership of bio */
     SSL_set_connect_state( ssl );
 
-    if( FD_UNLIKELY( !SSL_set_tlsext_host_name( ssl, ctx->server_fqdn ) ) ) {
+    if( FD_UNLIKELY( !SSL_set_tlsext_host_name( ssl, ctx->server_sni ) ) ) {
       FD_LOG_ERR(( "SSL_set_tlsext_host_name failed" ));
     }
 
@@ -162,7 +178,7 @@ fd_bundle_client_request_builder_info( fd_bundle_tile_t * ctx ) {
   static char const path[] = "/block_engine.BlockEngineValidator/GetBlockBuilderFeeInfo";
   int req_ok = fd_grpc_client_request_start(
       ctx->grpc_client,
-      ctx->server_fqdn, ctx->server_fqdn_len, ctx->server_tcp_port,
+      ctx->server_sni, ctx->server_sni_len, ctx->server_tcp_port,
       path, sizeof(path)-1,
       FD_BUNDLE_CLIENT_REQ_Bundle_GetBlockBuilderFeeInfo,
       &block_engine_BlockBuilderFeeInfoRequest_msg, &req,
@@ -181,7 +197,7 @@ fd_bundle_client_subscribe_packets( fd_bundle_tile_t * ctx ) {
   static char const path[] = "/block_engine.BlockEngineValidator/SubscribePackets";
   int req_ok = fd_grpc_client_request_start(
       ctx->grpc_client,
-      ctx->server_fqdn, ctx->server_fqdn_len, ctx->server_tcp_port,
+      ctx->server_sni, ctx->server_sni_len, ctx->server_tcp_port,
       path, sizeof(path)-1,
       FD_BUNDLE_CLIENT_REQ_Bundle_SubscribePackets,
       &block_engine_SubscribePacketsRequest_msg, &req,
@@ -200,7 +216,7 @@ fd_bundle_client_subscribe_bundles( fd_bundle_tile_t * ctx ) {
   static char const path[] = "/block_engine.BlockEngineValidator/SubscribeBundles";
   int req_ok = fd_grpc_client_request_start(
       ctx->grpc_client,
-      ctx->server_fqdn, ctx->server_fqdn_len, ctx->server_tcp_port,
+      ctx->server_sni, ctx->server_sni_len, ctx->server_tcp_port,
       path, sizeof(path)-1,
       FD_BUNDLE_CLIENT_REQ_Bundle_SubscribeBundles,
       &block_engine_SubscribeBundlesRequest_msg, &req,
@@ -253,7 +269,7 @@ fd_bundle_client_step( fd_bundle_tile_t * ctx,
     if( poll_res==0 ) return;
 
     if( pfds[0].revents & (POLLERR|POLLHUP) ) {
-      int connect_err = fd_bundle_client_do_connect( ctx );
+      int connect_err = fd_bundle_client_do_connect( ctx, 0 );
       FD_LOG_INFO(( "Bundle gRPC connect attempt failed (%i-%s)", connect_err, fd_io_strerror( connect_err ) ));
       fd_bundle_client_reset( ctx );
       ctx->metrics.transport_fail_cnt++;
@@ -296,7 +312,7 @@ fd_bundle_client_step( fd_bundle_tile_t * ctx,
 
   /* Drive auth */
   if( FD_UNLIKELY( ctx->auther.needs_poll ) ) {
-    fd_bundle_auther_poll( &ctx->auther, ctx->grpc_client, ctx->keyguard_client, ctx->server_fqdn, ctx->server_fqdn_len, ctx->server_tcp_port );
+    fd_bundle_auther_poll( &ctx->auther, ctx->grpc_client, ctx->keyguard_client, ctx->server_sni, ctx->server_sni_len, ctx->server_tcp_port );
     *charge_busy = 1;
     return;
   }

src/disco/bundle/fd_bundle_tile.c

@@ -11,7 +11,8 @@
 #include <sys/mman.h> /* PROT_READ (seccomp) */
 #include <sys/uio.h> /* writev */
 #include <netinet/in.h> /* AF_INET */
-#include <netdb.h> /* getaddrinfo */
+#include <netinet/tcp.h> /* TCP_FASTOPEN_CONNECT (seccomp) */
+#include "../../waltz/resolv/fd_netdb.h"
 
 #include "generated/fd_bundle_tile_seccomp.h"
 
@@ -123,12 +124,11 @@ after_credit( fd_bundle_tile_t *  ctx,
 }
 
 static void
-resolve_url( fd_url_t *   url_,
-             char const * url_str,
-             ulong        url_str_len,
-             uint *       ip4_addr,
-             ushort *     tcp_port,
-             _Bool *      is_ssl ) {
+parse_url( fd_url_t *   url_,
+           char const * url_str,
+           ulong        url_str_len,
+           ushort *     tcp_port,
+           _Bool *      is_ssl ) {
 
   /* Parse URL */
 
@@ -182,29 +182,16 @@ resolve_url( fd_url_t *   url_,
   }
   char host_cstr[ 256 ];
   fd_cstr_fini( fd_cstr_append_text( fd_cstr_init( host_cstr ), url->host, url->host_len ) );
-
-  struct addrinfo hints, *res;
-  memset( &hints, 0, sizeof(hints) );
-  hints.ai_family = AF_INET;
-
-  int err = getaddrinfo( host_cstr, NULL, &hints, &res );
-  if( FD_UNLIKELY( err ) ) {
-    FD_LOG_ERR(( "getaddrinfo `%s` failed (%d-%s)", host_cstr, err, gai_strerror( err ) ));
-  }
-
-  *ip4_addr = ((struct sockaddr_in *)res->ai_addr)->sin_addr.s_addr;
-  freeaddrinfo( res );
 }
 
 static void
-fd_bundle_tile_resolve_endpoint( fd_bundle_tile_t *     ctx,
-                                 fd_topo_tile_t const * tile ) {
+fd_bundle_tile_parse_endpoint( fd_bundle_tile_t *     ctx,
+                               fd_topo_tile_t const * tile ) {
   fd_url_t url[1];
   _Bool is_ssl = 0;
-  resolve_url(
+  parse_url(
       url,
       tile->bundle.url, tile->bundle.url_len,
-      &ctx->server_ip4_addr,
       &ctx->server_tcp_port,
       &is_ssl
   );
@@ -214,6 +201,14 @@ fd_bundle_tile_resolve_endpoint( fd_bundle_tile_t *     ctx,
   fd_cstr_fini( fd_cstr_append_text( fd_cstr_init( ctx->server_fqdn ), url->host, url->host_len ) );
   ctx->server_fqdn_len = url->host_len;
 
+  if( FD_UNLIKELY( tile->bundle.sni_len ) ) {
+    fd_cstr_fini( fd_cstr_append_text( fd_cstr_init( ctx->server_sni ), tile->bundle.sni, tile->bundle.sni_len ) );
+    ctx->server_sni_len = tile->bundle.sni_len;
+  } else {
+    fd_cstr_fini( fd_cstr_append_text( fd_cstr_init( ctx->server_sni ), url->host, url->host_len ) );
+    ctx->server_sni_len = url->host_len;
+  }
+
   ctx->is_ssl = !!is_ssl;
 #if !FD_HAS_OPENSSL
   if( FD_UNLIKELY( is_ssl ) ) {
@@ -379,19 +374,10 @@ privileged_init( fd_topo_t *      topo,
   uchar const * public_key = fd_keyload_load( tile->bundle.identity_key_path, 1 /* public key only */ );
   fd_memcpy( ctx->auther.pubkey, public_key, 32UL );
 
-  /* DNS resolution does arbitrary syscalls and system ops, therefore
-     has to be run outside the sandbox. */
-  fd_bundle_tile_resolve_endpoint( ctx, tile );
-
-  /* Override server name indication */
-  if( FD_UNLIKELY( tile->bundle.sni_len ) ) {
-    fd_cstr_fini( fd_cstr_append_text( fd_cstr_init( ctx->server_fqdn ), tile->bundle.sni, tile->bundle.sni_len ) );
-    ctx->server_fqdn_len = tile->bundle.sni_len;
-  }
+  ctx->keylog_fd = -1;
 
 # if FD_HAS_OPENSSL
 
-  ctx->keylog_fd = -1;
   if( FD_UNLIKELY( tile->bundle.key_log_path[0] ) ) {
     ctx->keylog_fd = open( tile->bundle.key_log_path, O_WRONLY|O_APPEND|O_CREAT, 0644 );
     if( FD_UNLIKELY( ctx->keylog_fd < 0 ) ) {
@@ -406,6 +392,11 @@ privileged_init( fd_topo_t *      topo,
 
 # endif /* FD_HAS_OPENSSL */
 
+  /* Init resolver */
+  if( FD_UNLIKELY( !fd_netdb_open_fds( ctx->netdb_fds ) ) ) {
+    FD_LOG_ERR(( "fd_netdb_open_fds failed" ));
+  }
+
   /* Random seed for header hashmap */
   if( FD_UNLIKELY( !fd_rng_secure( &ctx->map_seed, sizeof(ulong) ) ) ) {
     FD_LOG_CRIT(( "fd_rng_secure failed" ));
@@ -490,16 +481,23 @@ unprivileged_init( fd_topo_t *      topo,
   /* Force tile to output a plugin message on startup */
   ctx->bundle_status_plugin = 127;
   ctx->bundle_status_recent = FD_PLUGIN_MSG_BLOCK_ENGINE_UPDATE_STATUS_DISCONNECTED;
+
+  fd_bundle_tile_parse_endpoint( ctx, tile );
 }
 
 static ulong
 populate_allowed_seccomp( fd_topo_t const *      topo,
                           fd_topo_tile_t const * tile,
                           ulong                  out_cnt,
                           struct sock_filter *   out ) {
-  (void)topo; (void)tile;
+  fd_bundle_tile_t * ctx = fd_topo_obj_laddr( topo, tile->tile_obj_id );
   populate_sock_filter_policy_fd_bundle_tile(
-      out_cnt, out, (uint)fd_log_private_logfile_fd() );
+      out_cnt, out,
+      (uint)fd_log_private_logfile_fd(),
+      (uint)ctx->keylog_fd,
+      (uint)ctx->netdb_fds->etc_hosts,
+      (uint)ctx->netdb_fds->etc_resolv_conf
+  );
   return sock_filter_policy_fd_bundle_tile_instr_cnt;
 }
 
@@ -508,14 +506,19 @@ populate_allowed_fds( fd_topo_t const *      topo,
                       fd_topo_tile_t const * tile,
                       ulong                  out_fds_cnt,
                       int *                  out_fds ) {
-  (void)topo; (void)tile;
+  fd_bundle_tile_t * ctx = fd_topo_obj_laddr( topo, tile->tile_obj_id );
 
-  if( FD_UNLIKELY( out_fds_cnt<2UL ) ) FD_LOG_ERR(( "out_fds_cnt %lu", out_fds_cnt ));
+  if( FD_UNLIKELY( out_fds_cnt<5UL ) ) FD_LOG_ERR(( "out_fds_cnt %lu", out_fds_cnt ));
 
   ulong out_cnt = 0UL;
   out_fds[ out_cnt++ ] = 2; /* stderr */
   if( FD_LIKELY( -1!=fd_log_private_logfile_fd() ) )
     out_fds[ out_cnt++ ] = fd_log_private_logfile_fd(); /* logfile */
+  if( FD_LIKELY( ctx->netdb_fds->etc_hosts >= 0 ) )
+    out_fds[ out_cnt++ ] = ctx->netdb_fds->etc_hosts;
+  out_fds[ out_cnt++ ] = ctx->netdb_fds->etc_resolv_conf;
+  if( FD_UNLIKELY( ctx->keylog_fd>=0 ) )
+    out_fds[ out_cnt++ ] = ctx->keylog_fd;
   return out_cnt;
 }
 

src/disco/bundle/fd_bundle_tile.seccomppolicy

@@ -1,12 +1,21 @@
 # logfile_fd: It can be disabled by configuration, but typically tiles
 #             will open a log file on boot and write all messages there.
-unsigned int logfile_fd
+#
+# keylog_fd: Log HTTPS session keys to file (development option)
+#
+# etc_hosts_fd, etc_resolv_conf: Used to resolve DNS records.
+uint logfile_fd, uint keylog_fd, uint etc_hosts_fd, uint etc_resolv_conf
 
 # bundle: Read from TCP connection (HTTPS)
+#
+# resolv: Read DNS config
 read
 
 # bundle: Read from TCP connection (HTTP)
-recvmsg: (eq (arg 2) "MSG_NOSIGNAL|MSG_DONTWAIT")
+#
+# resolv: Receive DNS responses
+recvmsg: (or (eq (arg 2) "MSG_NOSIGNAL|MSG_DONTWAIT")
+             (eq (arg 2) 0))
 
 # bundle: Write to TCP connection (HTTPS)
 #
@@ -19,36 +28,73 @@ recvmsg: (eq (arg 2) "MSG_NOSIGNAL|MSG_DONTWAIT")
 # that descriptor 2 is always STDERR.
 write
 
+# OpenSSL: SSLKEYLOGFILE
+writev: (and (eq (arg 0) keylog_fd)
+             (eq (arg 2) 2))
+
 # bundle: Write to TCP connection (HTTP)
-sendmsg: (eq (arg 2) "MSG_NOSIGNAL|MSG_DONTWAIT")
+#
+# resolv: Send DNS queries via UDP or TCP
+sendmsg: (or (eq (arg 2) "MSG_NOSIGNAL|MSG_DONTWAIT")
+             (eq (arg 2) "MSG_FASTOPEN|MSG_NOSIGNAL")
+             (eq (arg 2) "MSG_NOSIGNAL"))
+
+# resolv: Send DNS queries via UDP or TCP
+sendto: (eq (arg 3) "MSG_NOSIGNAL")
 
 # logging: 'WARNING' and above fsync the logfile to disk immediately
 #
 # arg 0 is the file descriptor to fsync.
 fsync: (eq (arg 0) logfile_fd)
 
 # bundle: TCP connection to the bundle server needs to be established.
-socket: (and (eq (arg 0) "AF_INET")
-             (eq (arg 1) "SOCK_STREAM|SOCK_CLOEXEC")
-             (eq (arg 2) "IPPROTO_TCP"))
+#
+# resolv: Send DNS queries via UDP or TCP
+socket: (and (or (eq (arg 0) "AF_INET")
+                 (eq (arg 0) "AF_INET6"))
+             (or (and (or (eq (arg 1) "SOCK_STREAM|SOCK_CLOEXEC")
+                          (eq (arg 1) "SOCK_STREAM|SOCK_CLOEXEC|SOCK_NONBLOCK"))
+                      (eq (arg 2) 0))
+                 (and (eq (arg 1) "SOCK_DGRAM|SOCK_CLOEXEC|SOCK_NONBLOCK")
+                      (eq (arg 2) 0))
+                 (and (eq (arg 1) "SOCK_DGRAM|SOCK_CLOEXEC")
+                      (eq (arg 2) "IPPROTO_UDP"))))
 connect
 
 # bundle: TCP connection is closed if there is an error, and a new one
 #         is reopened
 shutdown: (eq (arg 1) "SHUT_WR")
+
+# bundle: TCP connection is closed if there is an error, and a new one
+#         is reopened
+# resolv: Close UDP/TCP sockets once done querying DNS
 close
 
 # bundle: Make TCP connection non-blocking
 fcntl: (and (eq (arg 1) "F_SETFL")
             (eq (arg 2) "O_NONBLOCK"))
 
+# resolv: Bind DNS request socket
+bind: (or (eq (arg 2) "sizeof(struct sockaddr_in)")
+          (eq (arg 2) "sizeof(struct sockaddr_in6)"))
+
 # bundle: Wait for TCP connection to be established
-poll: (and (eq (arg 1) 1)
-           (eq (arg 2) 0))
+#
+# resolv: Wait for DNS responses
+poll
+
+# bundle: configure TCP socket (for gRPC connection)
+#
+# resolv: configure UDP or TCP socket (for DNS queries)
+setsockopt: (or (and (eq (arg 1) SOL_SOCKET)
+                     (eq (arg 2) SO_RCVBUF))
+                (and (eq (arg 1) IPPROTO_TCP)
+                     (eq (arg 2) TCP_FASTOPEN_CONNECT))
+                (and (eq (arg 1) IPPROTO_IPV6)
+                     (eq (arg 2) IPV6_V6ONLY)))
 
-# bundle: configure TCP socket
-setsockopt: (and (eq (arg 1) SOL_SOCKET)
-                 (eq (arg 2) SO_RCVBUF))
+# resolv: check if DNS queries use IPv6
+getsockname
 
 # openssl: RAND_bytes requires getpid
 #
@@ -65,3 +111,9 @@ getpid
 # picking connection IDs.  The OpenSSL implementation calls getrandom
 # internally for periodically reseeding the RNG.
 getrandom
+
+# resolv: Read DNS config
+lseek: (and (or (eq (arg 0) etc_resolv_conf)
+                (eq (arg 0) etc_hosts_fd))
+            (eq (arg 1) 0)
+            (eq (arg 2) "SEEK_SET"))