fix(device): bypass foreign atfork handlers in reattach helper

.github/workflows/build.yml

@@ -137,13 +137,14 @@ jobs:
               "libuvc.reattach.rebinds_after_sigkill",
               "libuvc.reattach.disarmed_interfaces_are_not_rebound",
               "libuvc.reattach.busy_interface_is_retried",
+              "libuvc.reattach.rebinds_despite_a_foreign_atfork_handler",
               "libuvc.reattach.connect_ioctl_is_what_libusb_would_issue",
               "libuvc.reattach.claiming_an_interface_arms_the_guard",
               "libuvc.reattach.claim_failure_after_detach_reattaches",
               "libuvc.reattach.failed_reattach_after_failed_claim_stays_armed",
               "libuvc.reattach.detach_failure_leaves_nothing_armed",
               "libuvc.race.close_races_status_callback"
-            ] | sort) and (.tests | length == 36)
+            ] | sort) and (.tests | length == 37)
           ' "$result_dir/inventory.json" \
             2>&1 | tee "$result_dir/inventory-check.log"
 

CHANGELOG.ceralive.md

@@ -59,12 +59,40 @@ the upstream history, see `changelog.txt`.
 
       cmake .. -DLIBUVC_REATTACH_GUARD=OFF
 
-  Nine regression cases cover it, three of which really do `SIGKILL` a forked
+  Ten regression cases cover it, four of which really do `SIGKILL` a forked
   victim, since a test that calls a cleanup function proves nothing here. CI
   builds and tests the `OFF` variant too.
 
 ### Fixed
 
+- **The reattach helper deadlocked at birth in any host that also registers a
+  `pthread_atfork()` handler.** glibc's `fork()` runs every fork handler
+  registered anywhere in the process, in the child, on every call — it does not
+  matter which library called `fork()` or why. `cerastream` links libuvc next to
+  libsrt, whose child handler (`srt::CUDTUnited::cleanupAtFork`) locks an SRT
+  mutex; a `fork()` carries only the calling thread into the child, so whenever
+  another SRT thread held that mutex at the instant of the guard's fork it was
+  locked in the child with no thread left alive to release it.
+
+  Captured by attaching gdb to a stuck child on an RK3588 board:
+  `pthread_mutex_lock` ← `srt::CUDT::closeInternal` ← `CUDTUnited::cleanupAllSockets`
+  ← `CUDTUnited::cleanupAtFork` ← `fork` ← `uvc_reattach_guard_create`, with
+  `strace` showing an untimed `FUTEX_WAIT_PRIVATE`. `helper_main()` never ran, so
+  nothing was reattached **and** nothing was closed: the child kept the ~99
+  descriptors the fork had copied — the usbfs node, the ALSA capture device, the
+  MPP encoder — and ignored every `SIGTERM`, so the next run found its audio
+  device busy and its camera taken. systemd logged `Found left-over process …
+  in control group while starting unit` on essentially every restart.
+
+  `uvc_reattach_guard_create()` now creates both processes with a raw `clone(2)`
+  system call. `fork(2)` is defined as `clone(2)` with flags of just `SIGCHLD`,
+  and glibc's `__run_fork_handlers()` is reached only from the `fork()` wrapper,
+  so this is the same process creation with none of the handlers. The double-fork
+  topology, the `waitpid()` handshake and everything `helper_main()` does are
+  unchanged. `libuvc.reattach.rebinds_despite_a_foreign_atfork_handler` locks it
+  down by registering exactly such a handler in the victim and holding its mutex
+  hostage on a second thread across the fork.
+
 - **A failed interface claim left the interface with no kernel driver, in a
   process that was alive and well.** `uvc_claim_if()` detaches the kernel driver
   and claims the interface in two separate kernel calls, and rolled back

CMakeLists.txt

@@ -279,6 +279,7 @@ if(BUILD_TESTING)
         rebinds_after_sigkill
         disarmed_interfaces_are_not_rebound
         busy_interface_is_retried
+        rebinds_despite_a_foreign_atfork_handler
         connect_ioctl_is_what_libusb_would_issue
         claiming_an_interface_arms_the_guard
         claim_failure_after_detach_reattaches
@@ -293,6 +294,7 @@ if(BUILD_TESTING)
       libuvc.reattach.rebinds_after_sigkill
       libuvc.reattach.disarmed_interfaces_are_not_rebound
       libuvc.reattach.busy_interface_is_retried
+      libuvc.reattach.rebinds_despite_a_foreign_atfork_handler
       PROPERTIES TIMEOUT 30)
   endif()
 

README.md

@@ -29,7 +29,13 @@ CeraLive changes on top of the base:
    interfaces with `driver = NONE` for good. libuvc now forks a small helper
    when it first claims an interface; the helper's wakeup is the pipe EOF the
    kernel delivers on the arming process's death, whatever killed it, and it
-   re-probes the interfaces that are still armed. Gated by the CMake option
+   re-probes the interfaces that are still armed. That helper is created with a
+   raw `clone(2)`, not `fork()`: glibc's `fork()` runs every `pthread_atfork()`
+   handler registered anywhere in the process, so in a host that also links
+   libsrt the child ran SRT's handler and blocked forever on an SRT mutex a
+   now-vanished thread had held — never reaching the helper, and keeping every
+   descriptor the fork had copied until something `SIGKILL`ed it. Gated by the
+   CMake option
    **`LIBUVC_REATTACH_GUARD` (default `ON`, Linux only)**:
 
        cmake .. -DLIBUVC_REATTACH_GUARD=OFF
@@ -79,10 +85,10 @@ Linux CTest suite. Configure, build, inspect, and run its static build with:
       -DBUILD_TESTING=ON
     cmake --build build/regression --parallel
     ctest --test-dir build/regression --show-only=json-v1 \
-      | jq -e '.tests | length == 36'
+      | jq -e '.tests | length == 37'
     ctest --test-dir build/regression --output-on-failure
 
-The 36 cases are grouped as descriptor (11: `h264`, `h265`,
+The 37 cases are grouped as descriptor (11: `h264`, `h265`,
 `truncated_format`, `truncated_frame`, `degenerate_h26x`,
 `scanner_vc_header_short`, `scanner_vc_oversized`, `scanner_vc_zero`,
 `scanner_vs_header_short`, `scanner_vs_oversized`, `scanner_vs_zero`),
@@ -93,20 +99,24 @@ negotiation (5: `h264`, `h265`, `near_match`, `probe_set_error`,
 `no_status_endpoint_unchanged`, `sparse_interfaces_control_released_last`,
 `high_index_interfaces_released`, `cancel_not_found_still_drains`,
 `undeliverable_status_xfer_quarantines`, `quarantined_handle_stays_armed`),
-reattach (8: `rebinds_after_sigkill`, `disarmed_interfaces_are_not_rebound`,
-`busy_interface_is_retried`, `connect_ioctl_is_what_libusb_would_issue`,
+reattach (9: `rebinds_after_sigkill`, `disarmed_interfaces_are_not_rebound`,
+`busy_interface_is_retried`, `rebinds_despite_a_foreign_atfork_handler`,
+`connect_ioctl_is_what_libusb_would_issue`,
 `claiming_an_interface_arms_the_guard`, `claim_failure_after_detach_reattaches`,
 `failed_reattach_after_failed_claim_stays_armed`,
 `detach_failure_leaves_nothing_armed`), and race
 (1: `close_races_status_callback`).
 
-The nine guard cases exist only when `LIBUVC_REATTACH_GUARD` is `ON`; with
+The ten guard cases exist only when `LIBUVC_REATTACH_GUARD` is `ON`; with
 `-DLIBUVC_REATTACH_GUARD=OFF` the suite is the 27 cases that predate it, and CI
-runs that configuration too so a rollback stays a real rollback. Three of the
+runs that configuration too so a rollback stays a real rollback. Four of the
 reattach cases really do `SIGKILL` a forked victim process — that is the point,
-since the defect is defined by cleanup code never running. The three claim-path
-cases need no kill at all: they cover a claim that fails in a process that stays
-alive, which is a different defect with the same symptom.
+since the defect is defined by cleanup code never running. One of those four
+also registers a `pthread_atfork()` handler in the victim and holds its mutex
+hostage on a second thread, which is what a host linking libsrt does to the
+guard's own fork. The three claim-path cases need no kill at all: they cover a
+claim that fails in a process that stays alive, which is a different defect with
+the same symptom.
 
 CI runs this suite without camera hardware on Ubuntu 22.04
 and Ubuntu 24.04. See

include/libuvc/reattach_guard.h

@@ -17,7 +17,11 @@
  *
  *   - At the first successful interface claim, libuvc forks a tiny helper
  *     (double-forked, so it is reparented to init and the host never sees a
- *     SIGCHLD it did not ask for) and keeps the write end of a pipe.
+ *     SIGCHLD it did not ask for) and keeps the write end of a pipe. The fork is
+ *     issued as a raw clone(2), because glibc's fork() also runs every
+ *     pthread_atfork() handler ANY library in the process registered, and a
+ *     neighbour's handler taking one of its own mutexes wedges the child for
+ *     good -- see the comment on raw_fork() in reattach_guard.c.
  *   - The helper blocks on the read end. Its wakeup is the pipe's EOF, which the
  *     kernel delivers when the last write end closes -- and the kernel closes
  *     every descriptor of a dying process unconditionally, for _exit(), abort(),

src/reattach_guard.c

@@ -18,8 +18,10 @@
 #include <fcntl.h>
 #include <linux/usbdevice_fs.h>
 #include <poll.h>
+#include <signal.h>
 #include <sys/ioctl.h>
 #include <sys/mman.h>
+#include <sys/syscall.h>
 #include <sys/wait.h>
 #include <unistd.h>
 
@@ -183,6 +185,38 @@ static void close_inherited_fds(int keep_fd, int scan_limit) {
   }
 }
 
+/* fork() -- the glibc function, not the kernel operation -- runs every handler
+ * ANY library in the process registered with pthread_atfork(), in the child, on
+ * every call, whoever called fork() and for whatever reason. libuvc is linked
+ * alongside libsrt in cerastream, and SRT's child handler
+ * (srt::CUDTUnited::cleanupAtFork) locks an SRT mutex. A fork carries only the
+ * calling thread into the child, so whenever another SRT thread happened to hold
+ * that mutex at the instant of this call, it was locked in the child with no
+ * thread left alive to release it. Confirmed on an RK3588 board: the child sat
+ * in pthread_mutex_lock() forever, never reached helper_main(), reattached
+ * nothing, and held every descriptor the fork had copied -- the usbfs node and
+ * the ALSA capture device among them -- until something SIGKILLed it.
+ *
+ * The kernel has no such notion: fork(2) is defined as clone(2) with flags of
+ * just SIGCHLD, and glibc's __run_fork_handlers() is reached only from the
+ * fork() wrapper. Issuing that clone directly is therefore the same process
+ * creation with none of the handlers. It also skips glibc's own post-fork
+ * repairs (malloc arenas, the loader lock), which costs nothing here: everything
+ * this child runs is post-fork code in a multi-threaded process and is already
+ * restricted to async-signal-safe calls that touch none of that state.
+ *
+ * A NULL stack is what keeps this a fork rather than a thread -- with CLONE_VM
+ * unset the kernel hands the child a copy-on-write duplicate of the caller's
+ * stack, exactly as for fork(). The three trailing arguments are the tid and TLS
+ * pointers, and their order genuinely differs between x86-64 and aarch64; that
+ * is moot only because none of CLONE_PARENT_SETTID, CLONE_CHILD_SETTID,
+ * CLONE_CHILD_CLEARTID or CLONE_SETTLS is set, so the kernel never reads them.
+ * Passing any of them non-NULL would make this architecture-dependent. */
+static pid_t raw_fork(void) {
+  /* flags, stack, and the three trailing pointers the flags above leave unread. */
+  return (pid_t) syscall(SYS_clone, (long) SIGCHLD, 0L, 0L, 0L, 0L);
+}
+
 static void helper_main(int notify_fd, int scan_limit,
                         const struct uvc_reattach_record *record) {
   char drain[64];
@@ -251,12 +285,12 @@ uvc_reattach_guard_t *uvc_reattach_guard_create(uint8_t busnum, uint8_t devnum,
 
   scan_limit = fd_scan_limit();
 
-  intermediate = fork();
+  intermediate = raw_fork();
   if (intermediate == 0) {
     /* Double fork: the helper is reparented to init, so the host is never
      * handed a SIGCHLD for a process it did not create and never has a zombie
      * it does not know to reap. */
-    if (fork() == 0)
+    if (raw_fork() == 0)
       helper_main(notify[0], scan_limit, record); /* does not return */
     _exit(0);
   }

tests/reattach_guard_assertions.c

@@ -24,6 +24,7 @@
 
 #include <errno.h>
 #include <linux/usbdevice_fs.h>
+#include <pthread.h>
 #include <signal.h>
 #include <stdio.h>
 #include <stdlib.h>
@@ -275,6 +276,133 @@ static int check_busy_interface_is_retried(void) {
   return EXIT_SUCCESS;
 }
 
+/* Stands in for srt::CUDTUnited::cleanupAtFork(). The mutex is deliberately a
+ * plain process-private one: a fork() child gets it in whatever state it was in,
+ * and none of the threads that could unlock it. */
+static pthread_mutex_t hostage_mutex = PTHREAD_MUTEX_INITIALIZER;
+static pthread_mutex_t hostage_taken_mutex = PTHREAD_MUTEX_INITIALIZER;
+static pthread_cond_t hostage_taken = PTHREAD_COND_INITIALIZER;
+static int hostage_is_held;
+
+/* Takes the hostage and never gives it back, so the fork under test provably
+ * happens while another thread holds it -- the timing the board hit by luck. */
+static void *hold_hostage_forever(void *unused) {
+  (void) unused;
+
+  pthread_mutex_lock(&hostage_mutex);
+
+  pthread_mutex_lock(&hostage_taken_mutex);
+  hostage_is_held = 1;
+  pthread_cond_signal(&hostage_taken);
+  pthread_mutex_unlock(&hostage_taken_mutex);
+
+  for (;;)
+    pause();
+  return NULL;
+}
+
+static void child_handler_that_wedges_a_fork(void) {
+  pthread_mutex_lock(&hostage_mutex);
+  pthread_mutex_unlock(&hostage_mutex);
+}
+
+/* Same shape as run_victim_then_sigkill(), with two differences that matter.
+ *
+ * The hostage thread and the handler are installed INSIDE the victim: a handler
+ * registered before this fork would wedge the victim itself, and the claim under
+ * test is about the fork libuvc performs, not this one.
+ *
+ * The wait is bounded. Before the fix the victim never reaches its own
+ * raise(SIGKILL) at all -- it sits in uvc_reattach_guard_create()'s waitpid()
+ * for an intermediate that is wedged in the handler -- so an unbounded wait
+ * would hang the suite where it should fail it. */
+static int run_atfork_hostile_victim_then_sigkill(void) {
+  pid_t victim = fork();
+  pid_t reaped = 0;
+  int status = 0;
+  int waited;
+
+  if (victim == 0) {
+    pthread_t holder;
+    uvc_reattach_guard_t *guard;
+
+    if (pthread_create(&holder, NULL, hold_hostage_forever, NULL) != 0)
+      _exit(4);
+
+    pthread_mutex_lock(&hostage_taken_mutex);
+    while (!hostage_is_held)
+      pthread_cond_wait(&hostage_taken, &hostage_taken_mutex);
+    pthread_mutex_unlock(&hostage_taken_mutex);
+
+    if (pthread_atfork(NULL, NULL, child_handler_that_wedges_a_fork) != 0)
+      _exit(5);
+
+    guard = uvc_reattach_guard_create(TEST_BUSNUM, TEST_DEVNUM, TEST_ID_VENDOR,
+                                      TEST_ID_PRODUCT, TEST_BCD_DEVICE);
+    if (guard == NULL)
+      _exit(2);
+
+    arm_two_interfaces(guard);
+    raise(SIGKILL);
+    _exit(3);
+  }
+
+  if (victim < 0)
+    return -1;
+
+  for (waited = 0; waited < SETTLE_TIMEOUT_MS; waited += SETTLE_POLL_MS) {
+    reaped = waitpid(victim, &status, WNOHANG);
+    if (reaped == victim)
+      break;
+    if (reaped < 0 && errno != EINTR)
+      return -1;
+    sleep_ms(SETTLE_POLL_MS);
+  }
+
+  if (reaped != victim) {
+    /* Wedged. Kill it so the suite does not leave it running; the intermediate
+     * it is waiting on is unkillable by anything short of SIGKILL and holds
+     * every descriptor the fork copied, which is the production symptom. */
+    kill(victim, SIGKILL);
+    while (waitpid(victim, NULL, 0) < 0 && errno == EINTR)
+      continue;
+    return -1;
+  }
+
+  if (!WIFSIGNALED(status) || WTERMSIG(status) != SIGKILL)
+    return -1;
+
+  return 0;
+}
+
+/* THE cross-library case. libuvc's fork is never the only thing a fork does:
+ * glibc runs every pthread_atfork() handler registered anywhere in the process
+ * in the child of EVERY fork(), no matter which library called it or why. In
+ * cerastream that means libsrt's cleanupAtFork(), which takes an SRT mutex --
+ * and since fork() carries only the calling thread into the child, a mutex some
+ * other SRT thread held at that instant is locked in the child with nothing
+ * alive to unlock it. Captured on an RK3588 board as a child parked forever in
+ * pthread_mutex_lock() under srt::CUDTUnited::cleanupAtFork(), directly below
+ * uvc_reattach_guard_create(): helper_main() never ran, so the interfaces were
+ * never handed back AND the ~99 descriptors the fork copied -- the usbfs node,
+ * the ALSA capture device, the MPP encoder -- were held by a process that
+ * ignored every SIGTERM, blocking the next run from opening the same hardware.
+ * The guard therefore has to create its helper with a primitive glibc does not
+ * decorate with somebody else's handlers. */
+static int check_rebinds_despite_a_foreign_atfork_handler(void) {
+  CHECK(install_recorder(0));
+  CHECK(run_atfork_hostile_victim_then_sigkill() == 0);
+
+  wait_for_rebinds(2);
+
+  CHECK(recorder->opened == 1);
+  CHECK(recorder->rebound_count == 2);
+  CHECK(recorder->rebound[0] == 0);
+  CHECK(recorder->rebound[1] == 1);
+  CHECK(recorder->closed == 1);
+  return EXIT_SUCCESS;
+}
+
 static int wrapped_ioctl_result;
 static int wrapped_ioctl_errno;
 static int seen_fd;
@@ -612,6 +740,8 @@ int main(int argc, char **argv) {
     return check_disarmed_interfaces_are_not_rebound();
   if (strcmp(argv[2], "busy_interface_is_retried") == 0)
     return check_busy_interface_is_retried();
+  if (strcmp(argv[2], "rebinds_despite_a_foreign_atfork_handler") == 0)
+    return check_rebinds_despite_a_foreign_atfork_handler();
   if (strcmp(argv[2], "connect_ioctl_is_what_libusb_would_issue") == 0)
     return check_connect_ioctl_is_what_libusb_would_issue();
   if (strcmp(argv[2], "claiming_an_interface_arms_the_guard") == 0)