usockets: close a TLS socket whose send() keeps failing instead of spinning the writable dispatch

packages/bun-usockets/src/context.c

@@ -347,6 +347,8 @@ static void us_internal_init_listen_socket(struct us_listen_socket_t *ls,
     s->flags.adopted = 0;
     s->flags.allow_half_open = (options & LIBUS_SOCKET_ALLOW_HALF_OPEN);
     s->unclassified_send_failures = 0;
+    s->pending_action = US_PENDING_ACTION_NONE;
+    s->pending_code = 0;
     s->next = 0;
     s->prev = 0;
     s->connect_state = NULL;
@@ -491,6 +493,8 @@ static inline void us_internal_init_connect_socket(struct us_socket_t *s,
     s->flags.adopted = 0;
     s->flags.last_write_failed = 0;
     s->unclassified_send_failures = 0;
+    s->pending_action = US_PENDING_ACTION_NONE;
+    s->pending_code = 0;
     s->connect_state = NULL;
     s->connect_next = NULL;
 }

packages/bun-usockets/src/crypto/openssl.c

@@ -526,7 +526,8 @@ static int BIO_s_custom_write(BIO *bio, const char *data, int length) {
    * whatever BoringSSL tries to flush (typically the fatal alert). The bytes
    * are reported as written so the SSL state machine completes its error
    * path instead of retrying. */
-  if (loop_ssl_data->ssl_socket && loop_ssl_data->ssl_socket->ssl_pending_detach) {
+  if (loop_ssl_data->ssl_socket &&
+      loop_ssl_data->ssl_socket->pending_action == US_PENDING_ACTION_DETACH) {
     BIO_clear_retry_flags(bio);
     return length;
   }
@@ -1334,13 +1335,10 @@ void us_internal_ssl_attach(struct us_socket_t *s, SSL_CTX *ctx,
   s->ssl_read_wants_write = 0;
   s->ssl_fatal_error = 0;
   s->ssl_raw_tap = 0;
-  s->ssl_shutdown_after_spill = 0;
-  s->ssl_close_after_spill = 0;
   s->ssl_end_delivered = 0;
   s->ssl_in_use = 0;
-  s->ssl_pending_detach = 0;
-  s->ssl_pending_close_code = 0;
-  s->ssl_is_server = is_client ? 0 : 1;
+  /* pending_action is deliberately not reset: adopt_tls attaches to a live
+   * socket, and a CLOSE_RAW parked by a fatal send must still fire. */
 }
 
 void us_internal_ssl_detach(struct us_socket_t *s) {
@@ -1355,7 +1353,7 @@ void us_internal_ssl_detach(struct us_socket_t *s) {
        * inside it destroyed the socket); freeing now would leave BoringSSL
        * working on freed memory when control returns. The driver frees it
        * when the call unwinds. */
-      s->ssl_pending_detach = 1;
+      us_internal_socket_set_pending(s, US_PENDING_ACTION_DETACH, 0);
       return;
     }
     SSL_free(s_ssl(s));
@@ -1615,18 +1613,17 @@ struct us_socket_t *us_internal_ssl_close(struct us_socket_t *s, int code, void
      * driver's epilogue (the same protocol close_raw and ssl_detach honor),
      * releasing the spill now so the re-issued close cannot itself defer. */
     ssl_release_spill(s->group->loop, s);
-    s->ssl_pending_detach = 1;
-    s->ssl_pending_close_code = (unsigned char) code;
+    us_internal_socket_set_pending(s, US_PENDING_ACTION_DETACH, (unsigned char) code);
     return s;
   }
   /* node's `_handle.close()` (FAST_SHUTDOWN, no reason) must not cut off spilled
    * ciphertext already reported as written: SSL sealed it, so it can only be
-   * delivered, never re-sent. Mirror ssl_shutdown_after_spill; defer at most once. */
+   * delivered, never re-sent. Mirror the deferred shutdown; defer at most once. */
   if (code == LIBUS_SOCKET_CLOSE_CODE_FAST_SHUTDOWN && !reason
-      && !s->ssl_close_after_spill && !s->ssl_fatal_error && !us_socket_is_closed(s)) {
+      && s->pending_action != US_PENDING_ACTION_CLOSE && !s->ssl_fatal_error && !us_socket_is_closed(s)) {
     struct loop_ssl_data *loop_ssl_data = (struct loop_ssl_data *)s->group->loop->data.ssl_data;
     if (loop_ssl_data && !ssl_drain_spill(loop_ssl_data, s)) {
-      s->ssl_close_after_spill = 1;
+      us_internal_socket_set_pending(s, US_PENDING_ACTION_CLOSE, 0);
       return s;
     }
   }
@@ -1696,11 +1693,11 @@ static void ssl_update_handshake(struct us_socket_t *s) {
   s->ssl_in_use = 1;
   int result = SSL_do_handshake(s_ssl(s));
   s->ssl_in_use = ssl_was_in_use;
-  if (!ssl_was_in_use && s->ssl_pending_detach) {
+  if (!ssl_was_in_use && s->pending_action == US_PENDING_ACTION_DETACH) {
     /* A callback run from inside the handshake destroyed this socket; perform
      * the deferred close now and do not touch the SSL again. */
-    s->ssl_pending_detach = 0;
-    us_socket_close(s, s->ssl_pending_close_code, NULL);
+    s->pending_action = US_PENDING_ACTION_NONE;
+    us_socket_close(s, s->pending_code, NULL);
     return;
   }
 
@@ -1824,13 +1821,13 @@ struct us_socket_t *us_internal_ssl_on_writable(struct us_socket_t *s) {
     if (loop_ssl_data && !ssl_drain_spill(loop_ssl_data, s)) {
       return s;
     }
-    if (s->ssl_shutdown_after_spill) {
-      s->ssl_shutdown_after_spill = 0;
+    if (s->pending_action == US_PENDING_ACTION_SHUTDOWN) {
+      s->pending_action = US_PENDING_ACTION_NONE;
       us_internal_ssl_shutdown(s);
       if (ssl_gone(s)) return s;
     }
-    if (s->ssl_close_after_spill) {
-      s->ssl_close_after_spill = 0;
+    if (s->pending_action == US_PENDING_ACTION_CLOSE) {
+      s->pending_action = US_PENDING_ACTION_NONE;
       return us_internal_ssl_close(s, LIBUS_SOCKET_CLOSE_CODE_FAST_SHUTDOWN, NULL);
     }
   }
@@ -1909,11 +1906,11 @@ struct us_socket_t *us_internal_ssl_on_data(struct us_socket_t *s, char *data, i
                              loop_ssl_data->ssl_read_output + LIBUS_RECV_BUFFER_PADDING + read,
                              LIBUS_RECV_BUFFER_LENGTH - read);
     s->ssl_in_use = ssl_was_in_use;
-    if (!ssl_was_in_use && s->ssl_pending_detach) {
+    if (!ssl_was_in_use && s->pending_action == US_PENDING_ACTION_DETACH) {
       /* A callback run from inside this read destroyed the socket; perform
        * the deferred close now and stop processing. */
-      s->ssl_pending_detach = 0;
-      return us_socket_close(s, s->ssl_pending_close_code, NULL);
+      s->pending_action = US_PENDING_ACTION_NONE;
+      return us_socket_close(s, s->pending_code, NULL);
     }
 
     if (just_read <= 0) {
@@ -2182,7 +2179,7 @@ void us_internal_ssl_shutdown(struct us_socket_t *s) {
     struct loop_ssl_data *loop_ssl_data = (struct loop_ssl_data *)s->group->loop->data.ssl_data;
     if (loop_ssl_data && loop_ssl_data->ssl_spill_owner == s) {
       if (!ssl_drain_spill(loop_ssl_data, s)) {
-        s->ssl_shutdown_after_spill = 1;
+        us_internal_socket_set_pending(s, US_PENDING_ACTION_SHUTDOWN, 0);
         return;
       }
     }
@@ -2266,8 +2263,7 @@ void us_socket_sni_resolve(struct us_socket_t *s, struct ssl_ctx_st *ctx, int er
      * the handshake_failure alert BoringSSL queues for select_cert_error and
      * the epilogue closes the socket; the client just sees the connection go
      * away ("disconnected before secure TLS connection was established"). */
-    s->ssl_pending_detach = 1;
-    s->ssl_pending_close_code = 0;
+    us_internal_socket_set_pending(s, US_PENDING_ACTION_DETACH, 0);
   } else {
     pending->state = 2;
     pending->resolved_ctx = ctx; /* may be NULL = default ctx */
@@ -2452,8 +2448,7 @@ static enum ssl_select_cert_result_t us_select_cert_cb(const SSL_CLIENT_HELLO *h
      * deferred-close + BIO-swallow path, same as sni_cb). */
     struct loop_ssl_data *lsd = (struct loop_ssl_data *)BIO_get_data(SSL_get_wbio(ssl));
     if (lsd && lsd->ssl_socket) {
-      lsd->ssl_socket->ssl_pending_detach = 1;
-      lsd->ssl_socket->ssl_pending_close_code = 0;
+      us_internal_socket_set_pending(lsd->ssl_socket, US_PENDING_ACTION_DETACH, 0);
     }
     return ssl_select_cert_error;
   }

packages/bun-usockets/src/internal/internal.h

@@ -259,48 +259,55 @@ struct us_socket_flags {
 
 } __attribute__((packed));
 
+/* The one deferred action a socket carries, run at its trigger point. Higher
+ * values supersede lower (a close outranks a shutdown; a close that must wait
+ * for the SSL call on the stack to unwind outranks all). */
+enum us_socket_pending_action {
+    US_PENDING_ACTION_NONE = 0,
+    /* TLS shutdown once the ciphertext spill drains (writable event). */
+    US_PENDING_ACTION_SHUTDOWN = 1,
+    /* TLS close (FAST_SHUTDOWN) once the ciphertext spill drains (writable). */
+    US_PENDING_ACTION_CLOSE = 2,
+    /* send() failed with fatal errno `pending_code`: close_raw on the next
+     * writable dispatch (loop.c) instead of re-arming a retry that spins. */
+    US_PENDING_ACTION_CLOSE_RAW = 3,
+    /* Close with code `pending_code` once ssl_in_use unwinds; the BIO swallows
+     * writes meanwhile so BoringSSL's error path completes. */
+    US_PENDING_ACTION_DETACH = 4,
+};
+
 struct us_socket_t {
   alignas(LIBUS_EXT_ALIGNMENT) struct us_poll_t p;
   unsigned char timeout;
   unsigned char long_timeout;
   struct us_socket_flags flags;
   /* enum SocketKind. Selects the static dispatch arm in us_dispatch_*. */
   unsigned char kind;
-  /* SSL state. These 6 bits live in the pad-to-pointer gap before `group`, so
-   * they cost nothing on epoll/kqueue (poll=4 + 4×u8 + 1 byte bits = 9, padded
+  /* SSL state. These bits live in the pad-to-pointer gap before `group`, so
+   * they cost nothing on epoll/kqueue (poll=4 + 4×u8 + the bytes below, padded
    * to 16 anyway for the pointer). Per-socket reneg counters and SNI userdata
    * hang off SSL ex_data, allocated on first use only. */
   unsigned char ssl_handshake_state : 2;
   unsigned char ssl_write_wants_read : 1;
   unsigned char ssl_read_wants_write : 1;
   unsigned char ssl_fatal_error : 1;
-  unsigned char ssl_is_server : 1;
   /* If set, us_internal_ssl_on_data() first dispatches the still-encrypted
    * bytes via us_dispatch_ssl_raw_tap() before feeding them to SSL_read.
    * Used by Bun's `socket.upgradeTLS()` so the returned [raw, tls] pair's
    * `raw` half can observe ciphertext (node:net Duplex.ondata semantics). */
   unsigned char ssl_raw_tap : 1;
-  /* A graceful TLS shutdown arrived while batched ciphertext was still
-   * spilled (see ssl_flush_write_batch); the shutdown re-runs once the
-   * spill drains so those records are not cut off by our FIN/close_notify. */
-  unsigned char ssl_shutdown_after_spill : 1;
-  /* Same as ssl_shutdown_after_spill but for us_internal_ssl_close: the
-   * close re-runs from the writable event once the spill drains. */
-  unsigned char ssl_close_after_spill : 1;
   /* The plaintext EOF (peer close_notify or the raw TCP FIN behind it) was
    * already dispatched to the user layer; both EOF paths can fire for one
    * connection, and the end handler must run once. */
   unsigned char ssl_end_delivered : 1;
   /* Set while SSL_do_handshake/SSL_read is on the stack: JS run from inside
    * those calls (ALPN/SNI/keylog callbacks) may destroy the socket, and the
-   * SSL must not be freed under BoringSSL's feet - the detach is deferred to
-   * the driver's epilogue via ssl_pending_detach. */
+   * SSL must not be freed under BoringSSL's feet - the close is parked as
+   * US_PENDING_ACTION_DETACH and performed by the driver's epilogue. */
   unsigned char ssl_in_use : 1;
-  unsigned char ssl_pending_detach : 1;
-  /* The close code passed to the deferred close (e.g. a reset requested from
-   * inside a handshake callback must still RST, not FIN, when it is finally
-   * performed). */
-  unsigned char ssl_pending_close_code;
+  /* enum us_socket_pending_action, and its payload (close code or errno). */
+  unsigned char pending_action : 3;
+  unsigned char pending_code;
   /* Consecutive send() failures with an errno that is neither
    * would-block/transient nor a known peer-gone error (see
    * us_socket_write_check_error). Reset by any send that makes progress.
@@ -324,6 +331,18 @@ struct us_socket_t {
 _Static_assert(sizeof(struct us_socket_flags) == 1, "us_socket_flags grew");
 #endif
 
+/* Latch a pending action. A higher action supersedes; an equal action keeps a
+ * nonzero code already parked (the SNI-abort path re-latches DETACH(0) after
+ * the JS callback may have parked DETACH(RESET)); a lower action is ignored. */
+static inline void us_internal_socket_set_pending(struct us_socket_t *s, unsigned char action, unsigned char code) {
+    if (action > s->pending_action) {
+        s->pending_action = action;
+        s->pending_code = code;
+    } else if (action == s->pending_action && (code || !s->pending_code)) {
+        s->pending_code = code;
+    }
+}
+
 struct us_connecting_socket_t {
     alignas(LIBUS_EXT_ALIGNMENT) struct addrinfo_request *addrinfo_req;
     struct us_socket_group_t *group;

packages/bun-usockets/src/loop.c

@@ -533,6 +533,8 @@ void us_internal_dispatch_ready_poll(struct us_poll_t *p, int error, int eof, in
                         s->flags.adopted = 0;
                         s->flags.last_write_failed = 0;
                         s->unclassified_send_failures = 0;
+                        s->pending_action = US_PENDING_ACTION_NONE;
+                        s->pending_code = 0;
 
                         /* We always use nodelay */
                         bsd_socket_nodelay(client_fd, 1);
@@ -600,6 +602,18 @@ void us_internal_dispatch_ready_poll(struct us_poll_t *p, int error, int eof, in
                     return;
                 }
 
+                /* A send() from one of the us_socket_*write* helpers returned a
+                 * fatal errno (peer-gone, or the unclassified retry ceiling hit).
+                 * Without this, the TLS spill path re-armed the writable poll on
+                 * every failure and spun at 100% for as long as the kernel kept
+                 * the fd writable. Same close shape as the poll-error branch. */
+                if (UNLIKELY(s->pending_action == US_PENDING_ACTION_CLOSE_RAW)) {
+                    int send_errno = s->pending_code;
+                    s->pending_action = US_PENDING_ACTION_NONE;
+                    s = us_internal_socket_close_raw(s, send_errno > 2 ? send_errno : ECONNRESET, NULL);
+                    return;
+                }
+
                 /* If we have no failed write or if we shut down, then stop polling for more writable */
                 if (!s->flags.last_write_failed || us_socket_is_shut_down(s)) {
                     us_poll_change(&s->p, loop, us_poll_events(&s->p) & LIBUS_SOCKET_READABLE);