podman: add opt-in local tfork API

criu/build.sh

@@ -93,12 +93,25 @@ for mod in "${ACTIVE_MODULES[@]}"; do
     fi
 
     if lsmod | awk '{print $1}' | grep -qx "${mod}"; then
-        echo "  -- ${mod}: already loaded, rmmod then insmod"
-        rmmod "${mod}" 2>/dev/null || true
-    else
-        echo "  -- ${mod}: insmod"
+        echo "  -- ${mod}: already loaded, trying reload"
+        if ! rmmod "${mod}"; then
+            echo "build.sh: failed to unload loaded module ${mod}; aborting to avoid stale module" >&2
+            echo "build.sh: stop running tfork/podman containers that may still hold ${mod}, then retry" >&2
+            exit 1
+        fi
+        if lsmod | awk '{print $1}' | grep -qx "${mod}"; then
+            echo "build.sh: module ${mod} is still loaded after rmmod; aborting to avoid stale module" >&2
+            echo "build.sh: stop running tfork/podman containers that may still hold ${mod}, then retry" >&2
+            exit 1
+        fi
+    fi
+
+    echo "  -- ${mod}: insmod"
+    if ! insmod "${ko_path}"; then
+        echo "build.sh: failed to insert rebuilt module ${mod} from ${ko_path}" >&2
+        echo "build.sh: if the old module is still active, stop running tfork/podman containers and rerun this script" >&2
+        exit 1
     fi
-    insmod "${ko_path}"
 done
 
 echo "  -- verify all modules loaded"

criu/criu/action-scripts.c

@@ -32,6 +32,9 @@ static const char *action_names[ACT_MAX] = {
 	[ACT_ORPHAN_PTS_MASTER] = "orphan-pts-master",
 	[ACT_STATUS_READY] = "status-ready",
 	[ACT_QUERY_EXT_FILES] = "query-ext-files",
+	[ACT_POST_TFORK_FREEZE] = "post-tfork-freeze",
+	[ACT_PRE_TFORK_RESTORE] = "pre-tfork-restore",
+	[ACT_TFORK_SOURCE_DETACHED] = "tfork-source-detached",
 };
 
 struct script {

criu/criu/clone-noasan.c

@@ -78,7 +78,10 @@ int clone3_with_pid_noasan(int (*fn)(void *), void *arg, int flags, int exit_sig
 	c_args.flags = flags;
 	c_args.set_tid = ptr_to_u64(&pid);
 	c_args.set_tid_size = 1;
+	pr_info("clone3 set_tid pid=%d flags=0x%x size=1\n", pid, flags);
 	pid = syscall(__NR_clone3, &c_args, sizeof(c_args));
+	if (pid < 0)
+		pr_perror("clone3 set_tid failed flags=0x%x size=1", flags);
 	if (pid == 0)
 		exit(fn(arg));
 	return pid;
@@ -99,6 +102,12 @@ int clone3_with_nested_pid_noasan(int (*fn)(void *), void *arg, int flags, int e
 	BUG_ON(pid->ns_level > MAX_PID_NS_LEVEL || pid->ns_level <= 1);
 	for (i = 0; i < pid->ns_level; i++)
 		tids[i] = pid->ns[i].ns_pid;
+	pr_info("clone3 nested set_tid flags=0x%x size=%d tids=%d/%d/%d/%d\n",
+		flags, pid->ns_level,
+		tids[0],
+		pid->ns_level > 1 ? tids[1] : -1,
+		pid->ns_level > 2 ? tids[2] : -1,
+		pid->ns_level > 3 ? tids[3] : -1);
 
 	if (!(flags & CLONE_PARENT)) {
 		if (exit_signal != SIGCHLD) {
@@ -112,6 +121,13 @@ int clone3_with_nested_pid_noasan(int (*fn)(void *), void *arg, int flags, int e
 	c_args.set_tid = ptr_to_u64(tids);
 	c_args.set_tid_size = pid->ns_level;
 	pid_ret = syscall(__NR_clone3, &c_args, sizeof(c_args));
+	if (pid_ret < 0)
+		pr_perror("clone3 nested set_tid failed flags=0x%x size=%d tids=%d/%d/%d/%d",
+			  flags, pid->ns_level,
+			  tids[0],
+			  pid->ns_level > 1 ? tids[1] : -1,
+			  pid->ns_level > 2 ? tids[2] : -1,
+			  pid->ns_level > 3 ? tids[3] : -1);
 	if (pid_ret == 0)
 		exit(fn(arg));
 	return pid_ret;

criu/criu/cr-dump.c

@@ -885,6 +885,7 @@ static int collect_pstree_ids_predump(void)
 	 * write_img_inventory().
 	 */
 
+	pid_init_dump(crt.i.pid, &crt.i);
 	crt.i.pid->state = TASK_ALIVE;
 	crt.i.pid->real = getpid();
 
@@ -2360,6 +2361,9 @@ int cr_dump_tasks(pid_t pid)
 	if (dump_zombies())
 		goto err;
 
+	if (finalize_nested_pid_ns_ids())
+		goto err;
+
 	if (dump_pstree(root_item))
 		goto err;
 

criu/criu/cr-restore.c

@@ -185,10 +185,77 @@ static int __restore_wait_inprogress_tasks(int participants)
 {
 	int ret;
 	futex_t *np = &task_entries->nr_in_progress;
+	const int tfork_restore_wait_timeout_ms = 10000;
+
+	if (opts.tfork.active) {
+		struct timespec started, now, timeout;
+		int wait_ret = 0;
+
+		/*
+		 * All paths that decrement this barrier use
+		 * futex_dec_and_wake().  Waiting on the observed value avoids
+		 * paying up to one 100ms polling interval at every restore stage.
+		 */
+		if (clock_gettime(CLOCK_MONOTONIC, &started)) {
+			pr_perror("tfork restore wait: clock_gettime");
+			return -errno;
+		}
+		while ((int)futex_get(np) > participants) {
+			int64_t elapsed_ns, remaining_ns;
+			uint32_t observed = futex_get(np);
+
+			if (observed & FUTEX_ABORT_FLAG)
+				break;
+			if (clock_gettime(CLOCK_MONOTONIC, &now)) {
+				pr_perror("tfork restore wait: clock_gettime");
+				return -errno;
+			}
+			elapsed_ns =
+				(int64_t)(now.tv_sec - started.tv_sec) * NSEC_PER_SEC +
+				(now.tv_nsec - started.tv_nsec);
+			remaining_ns =
+				(int64_t)tfork_restore_wait_timeout_ms * 1000000 -
+				elapsed_ns;
+			if (remaining_ns <= 0) {
+				wait_ret = -ETIMEDOUT;
+				break;
+			}
+			timeout.tv_sec = remaining_ns / NSEC_PER_SEC;
+			timeout.tv_nsec = remaining_ns % NSEC_PER_SEC;
+			wait_ret = sys_futex(
+				(uint32_t *)&np->raw.counter, FUTEX_WAIT,
+				observed, &timeout, NULL, 0);
+			if (wait_ret == 0 || wait_ret == -EINTR ||
+			    wait_ret == -EWOULDBLOCK)
+				continue;
+			if (wait_ret == -ETIMEDOUT)
+				break;
+			pr_err("tfork restore futex wait failed: %d\n", wait_ret);
+			set_cr_errno(-wait_ret);
+			return wait_ret;
+		}
+
+		if (wait_ret == -ETIMEDOUT &&
+		    (int)futex_get(np) > participants) {
+			pr_err("tfork restore wait timed out after %dms: participants=%d nr_in_progress=%d start_stage=%d task_cr_err=%d nr_tasks=%d nr_threads=%d nr_helpers=%d\n",
+			       tfork_restore_wait_timeout_ms,
+			       participants, (int)futex_get(np),
+			       (int)futex_get(&task_entries->start),
+			       get_task_cr_err(),
+			       task_entries->nr_tasks, task_entries->nr_threads,
+			       task_entries->nr_helpers);
+			set_cr_errno(ETIMEDOUT);
+			return -ETIMEDOUT;
+		}
+	} else {
+		futex_wait_while_gt(np, participants);
+	}
 
-	futex_wait_while_gt(np, participants);
 	ret = (int)futex_get(np);
 	if (ret < 0) {
+		pr_err("restore wait aborted: participants=%d nr_in_progress=%d start_stage=%d task_cr_err=%d\n",
+		       participants, ret, (int)futex_get(&task_entries->start),
+		       get_task_cr_err());
 		set_cr_errno(get_task_cr_err());
 		return ret;
 	}
@@ -227,6 +294,17 @@ static inline void __restore_switch_stage(int next_stage)
 
 static int restore_switch_stage(int next_stage)
 {
+	if (opts.tfork.active)
+		pr_warn("tfork: restore_switch_stage %d participants=%d nr_tasks=%d nr_threads=%d nr_helpers=%d\n",
+			next_stage, stage_participants(next_stage),
+			task_entries->nr_tasks, task_entries->nr_threads,
+			task_entries->nr_helpers);
+	else
+		pr_info("restore_switch_stage %d participants=%d nr_tasks=%d nr_threads=%d nr_helpers=%d\n",
+			next_stage, stage_participants(next_stage),
+			task_entries->nr_tasks, task_entries->nr_threads,
+			task_entries->nr_helpers);
+
 	__restore_switch_stage(next_stage);
 	return restore_wait_inprogress_tasks();
 }
@@ -1395,6 +1473,16 @@ static inline int fork_with_pid(struct pstree_item *item)
 
 	ca.item = item;
 	ca.clone_flags = rsti(item)->clone_flags;
+	if (opts.tfork.active && item != root_item &&
+	    !(ca.clone_flags & CLONE_NEWPID) &&
+	    item->pid->ns_level > 1 &&
+	    item->pid->ns[0].ns_pid == INIT_PID) {
+		pr_info("tfork: repairing missing CLONE_NEWPID for pidns init uid=%d local=%d parent_local=%d level=%d\n",
+			uid(item), pid,
+			item->parent ? localpid(item->parent) : -1,
+			item->pid->ns_level);
+		ca.clone_flags |= CLONE_NEWPID;
+	}
 
 	BUG_ON(ca.clone_flags & CLONE_VM);
 
@@ -1434,27 +1522,72 @@ static inline int fork_with_pid(struct pstree_item *item)
 		strip |= CLONE_NEWUSER;
 
 	if (kdat.has_clone3_set_tid) {
-		if (item->pid->ns_level == 1)
+		if (opts.tfork.active && (ca.clone_flags & CLONE_NEWPID)) {
+			pr_info("tfork: restore pidns init uid=%d local pid %d with fresh parent pid, dumped chain level=%d\n",
+				uid(item), pid, item->pid->ns_level);
 			ret = clone3_with_pid_noasan(restore_task_with_children, &ca,
 						     ca.clone_flags & ~strip, SIGCHLD, pid);
-		else
+		} else if (item->pid->ns_level == 1)
+			ret = clone3_with_pid_noasan(restore_task_with_children, &ca,
+						     ca.clone_flags & ~strip, SIGCHLD, pid);
+		else {
+			struct pid tfork_pid = {};
+			struct pid *restore_pid = item->pid;
+
+			if (opts.tfork.active && (root_ns_mask & CLONE_NEWPID) &&
+			    root_item && root_item->pid->ns_level > 1 &&
+			    item->pid->ns_level > 1) {
+				/*
+				 * Copy only scalar pid identity. struct pid also
+				 * embeds rb_node links owned by the dumped pid trees;
+				 * copying those nodes into a temporary stack object
+				 * corrupts the tree metadata if it ever gets reused.
+				 */
+				tfork_pid.item = item->pid->item;
+				tfork_pid.real = item->pid->real;
+				tfork_pid.local = item->pid->local;
+				tfork_pid.uid = item->pid->uid;
+				tfork_pid.state = item->pid->state;
+				tfork_pid.stop_signo = item->pid->stop_signo;
+				tfork_pid.ns_level = item->pid->ns_level - 1;
+				tfork_pid.leaf_ns_id = item->pid->leaf_ns_id;
+				memcpy(tfork_pid.ns, item->pid->ns, sizeof(tfork_pid.ns));
+				restore_pid = &tfork_pid;
+				pr_info("tfork: restore pid uid=%d local=%d with rebased pid chain level %d -> %d\n",
+					uid(item), pid, item->pid->ns_level,
+					restore_pid->ns_level);
+			}
 			ret = clone3_with_nested_pid_noasan(restore_task_with_children, &ca,
 							    ca.clone_flags & ~strip,
-							    SIGCHLD, item->pid);
+							    SIGCHLD, restore_pid);
+		}
 	} else {
 		BUG_ON(item->pid->ns_level >= 1);
 		close_pid_proc();
 		ret = clone_noasan(restore_task_with_children,
 				   (ca.clone_flags & ~strip) | SIGCHLD, &ca);
 	}
 	if (ret < 0) {
+		pr_err("fork_with_pid failed item uid=%d local=%d real=%d parent_local=%d flags=0x%lx stripped_flags=0x%lx ns_level=%d root_ns_mask=0x%lx tfork=%d\n",
+		       uid(item), pid, realpid(item),
+		       item->parent ? localpid(item->parent) : -1,
+		       ca.clone_flags, ca.clone_flags & ~strip,
+		       item->pid->ns_level, root_ns_mask,
+		       opts.tfork.active ? 1 : 0);
+		if (item->pid->ns_level > 0)
+			pr_err("fork_with_pid pid chain uid=%d ns=%d/%d/%d/%d\n",
+			       uid(item),
+			       item->pid->ns[0].ns_pid,
+			       item->pid->ns_level > 1 ? item->pid->ns[1].ns_pid : -1,
+			       item->pid->ns_level > 2 ? item->pid->ns[2].ns_pid : -1,
+			       item->pid->ns_level > 3 ? item->pid->ns[3].ns_pid : -1);
 		pr_perror("Can't fork for %d", pid);
 		if (errno == EEXIST)
 			set_cr_errno(EEXIST);
 		goto err_unlock;
 	}
 
-	if (item == root_item) {
+	if (opts.tfork.active || item == root_item) {
 		item->pid->real = ret;
 		pr_debug("PID: real %d virt %d\n", item->pid->real, localpid(item));
 	}
@@ -4045,6 +4178,7 @@ static int sigreturn_restore(struct task_restore_args *task_args, unsigned long
 	task_args->vdso_rt_size = vdso_rt_size;
 	task_args->can_map_vdso = kdat.can_map_vdso;
 	task_args->has_clone3_set_tid = kdat.has_clone3_set_tid;
+	task_args->tfork_active = opts.tfork.active;
 
 	new_sp = restorer_stack(task_args->t->mz);
 

criu/criu/cr-tfork.c

@@ -271,7 +271,7 @@ int tfork_read_cropt(void)
 		return 0;
 
 	snap_path = opts.tfork.snap_root;
-	if (!snap_path && opts.tfork.copies > 1 && opts.tfork.snap_roots &&
+	if (!snap_path && opts.tfork.copies >= 1 && opts.tfork.snap_roots &&
 	    opts.tfork.copy_idx < opts.tfork.snap_roots_n)
 		snap_path = opts.tfork.snap_roots[opts.tfork.copy_idx];
 
@@ -615,6 +615,8 @@ static int cr_tfork_finish(int ret)
 
 	pstree_switch_state(root_item, TASK_ALIVE);
 	timing_stop(TIME_FROZEN);
+	if (run_scripts(ACT_TFORK_SOURCE_DETACHED))
+		pr_warn("tfork: source-detached notification failed\n");
 
 	seccomp_free_entries();
 	free_file_locks();
@@ -966,8 +968,9 @@ int cr_tfork_tasks(pid_t pid)
 		opts.tfork.pidfd_map[opts.tfork.pidfd_map_nr].pidfd = pidfd;
 		opts.tfork.pidfd_map[opts.tfork.pidfd_map_nr].memfd = -1;
 		opts.tfork.pidfd_map_nr++;
-		pr_info("tfork: pidfd %d for pid %d (vpid %d uid %d)\n",
-			pidfd, item->pid->real, localpid(item), uid(item));
+		pr_info("tfork: pidfd %d for pid %d (vpid %d uid %d nsid %d level %d)\n",
+			pidfd, item->pid->real, localpid(item), uid(item),
+			item->pid->leaf_ns_id, item->pid->ns_level);
 	}
 
 	ret = run_scripts(ACT_PRE_TFORK_RESTORE);
@@ -1078,7 +1081,7 @@ int cr_tfork_tasks(pid_t pid)
 			list_for_each_entry(cgo_iter, &opts.new_cgroup_roots, node)
 				rpc_n_cg_root++;
 
-			rpc_max = 32 + 2 * (rpc_n_ifd + rpc_n_ext + rpc_n_cg_root
+			rpc_max = 33 + 2 * (rpc_n_ifd + rpc_n_ext + rpc_n_cg_root
 					    + opts.tfork.snap_mount_n)
 				+ rpc_n_copy_args + 2;
 			rpc_argv = calloc(rpc_max, sizeof(*rpc_argv));
@@ -1098,6 +1101,7 @@ int cr_tfork_tasks(pid_t pid)
 			rpc_argv[rpc_n++] = "-o";
 			rpc_argv[rpc_n++] = restore_log_arg;
 			rpc_argv[rpc_n++] = "-v2";
+			rpc_argv[rpc_n++] = "--keep-pid-hierarchy";
 
 			if (opts.root) {
 				rpc_argv[rpc_n++] = "--root";
@@ -1171,7 +1175,7 @@ int cr_tfork_tasks(pid_t pid)
 				rpc_argv[rpc_n++] = "--tfork-snap-mounts";
 				rpc_argv[rpc_n++] = snap_mounts_csv;
 			}
-			if (opts.tfork.copies > 1) {
+			if (opts.tfork.copies >= 1) {
 				snprintf(copies_arg, sizeof(copies_arg),
 					 "%d", opts.tfork.copies);
 				rpc_argv[rpc_n++] = "--tfork-copies";
@@ -1267,7 +1271,7 @@ int cr_tfork_tasks(pid_t pid)
 		buf[off] = '\0';
 		end = buf + off;
 
-		argv_max = 8 + 2;
+		argv_max = 8 + 3;
 		for (p = buf; p < end; p++)
 			if (*p == '\0')
 				argv_max++;
@@ -1312,6 +1316,11 @@ int cr_tfork_tasks(pid_t pid)
 			argv_new[argc_new++] = "--pidfile";
 			argv_new[argc_new++] = pidfile_arg;
 		}
+		if ((size_t)argc_new + 1 >= argv_max) {
+			pr_err("tfork restore argv overflow: used=%d max=%zu\n", argc_new, argv_max);
+			exit(1);
+		}
+		argv_new[argc_new++] = "--keep-pid-hierarchy";
 		argv_new[argc_new] = NULL;
 
 		execv("/proc/self/exe", argv_new);