fix(tfork): make live-clone cleanup transactional

criu/build.sh

@@ -93,12 +93,25 @@ for mod in "${ACTIVE_MODULES[@]}"; do
     fi
 
     if lsmod | awk '{print $1}' | grep -qx "${mod}"; then
-        echo "  -- ${mod}: already loaded, rmmod then insmod"
-        rmmod "${mod}" 2>/dev/null || true
-    else
-        echo "  -- ${mod}: insmod"
+        echo "  -- ${mod}: already loaded, trying reload"
+        if ! rmmod "${mod}"; then
+            echo "build.sh: failed to unload loaded module ${mod}; aborting to avoid stale module" >&2
+            echo "build.sh: stop running tfork/podman containers that may still hold ${mod}, then retry" >&2
+            exit 1
+        fi
+        if lsmod | awk '{print $1}' | grep -qx "${mod}"; then
+            echo "build.sh: module ${mod} is still loaded after rmmod; aborting to avoid stale module" >&2
+            echo "build.sh: stop running tfork/podman containers that may still hold ${mod}, then retry" >&2
+            exit 1
+        fi
+    fi
+
+    echo "  -- ${mod}: insmod"
+    if ! insmod "${ko_path}"; then
+        echo "build.sh: failed to insert rebuilt module ${mod} from ${ko_path}" >&2
+        echo "build.sh: if the old module is still active, stop running tfork/podman containers and rerun this script" >&2
+        exit 1
     fi
-    insmod "${ko_path}"
 done
 
 echo "  -- verify all modules loaded"

criu/criu/clone-noasan.c

@@ -78,7 +78,10 @@ int clone3_with_pid_noasan(int (*fn)(void *), void *arg, int flags, int exit_sig
 	c_args.flags = flags;
 	c_args.set_tid = ptr_to_u64(&pid);
 	c_args.set_tid_size = 1;
+	pr_info("clone3 set_tid pid=%d flags=0x%x size=1\n", pid, flags);
 	pid = syscall(__NR_clone3, &c_args, sizeof(c_args));
+	if (pid < 0)
+		pr_perror("clone3 set_tid failed flags=0x%x size=1", flags);
 	if (pid == 0)
 		exit(fn(arg));
 	return pid;
@@ -99,6 +102,12 @@ int clone3_with_nested_pid_noasan(int (*fn)(void *), void *arg, int flags, int e
 	BUG_ON(pid->ns_level > MAX_PID_NS_LEVEL || pid->ns_level <= 1);
 	for (i = 0; i < pid->ns_level; i++)
 		tids[i] = pid->ns[i].ns_pid;
+	pr_info("clone3 nested set_tid flags=0x%x size=%d tids=%d/%d/%d/%d\n",
+		flags, pid->ns_level,
+		tids[0],
+		pid->ns_level > 1 ? tids[1] : -1,
+		pid->ns_level > 2 ? tids[2] : -1,
+		pid->ns_level > 3 ? tids[3] : -1);
 
 	if (!(flags & CLONE_PARENT)) {
 		if (exit_signal != SIGCHLD) {
@@ -112,6 +121,13 @@ int clone3_with_nested_pid_noasan(int (*fn)(void *), void *arg, int flags, int e
 	c_args.set_tid = ptr_to_u64(tids);
 	c_args.set_tid_size = pid->ns_level;
 	pid_ret = syscall(__NR_clone3, &c_args, sizeof(c_args));
+	if (pid_ret < 0)
+		pr_perror("clone3 nested set_tid failed flags=0x%x size=%d tids=%d/%d/%d/%d",
+			  flags, pid->ns_level,
+			  tids[0],
+			  pid->ns_level > 1 ? tids[1] : -1,
+			  pid->ns_level > 2 ? tids[2] : -1,
+			  pid->ns_level > 3 ? tids[3] : -1);
 	if (pid_ret == 0)
 		exit(fn(arg));
 	return pid_ret;

criu/criu/cr-dump.c

@@ -885,6 +885,7 @@ static int collect_pstree_ids_predump(void)
 	 * write_img_inventory().
 	 */
 
+	pid_init_dump(crt.i.pid, &crt.i);
 	crt.i.pid->state = TASK_ALIVE;
 	crt.i.pid->real = getpid();
 
@@ -2360,6 +2361,9 @@ int cr_dump_tasks(pid_t pid)
 	if (dump_zombies())
 		goto err;
 
+	if (finalize_nested_pid_ns_ids())
+		goto err;
+
 	if (dump_pstree(root_item))
 		goto err;
 

criu/criu/cr-restore.c

@@ -185,10 +185,39 @@ static int __restore_wait_inprogress_tasks(int participants)
 {
 	int ret;
 	futex_t *np = &task_entries->nr_in_progress;
+	const int tfork_restore_wait_timeout_ms = 10000;
+	const int tfork_restore_wait_poll_us = 100000;
+
+	if (opts.tfork.active) {
+		int waited;
+
+		for (waited = 0; waited < tfork_restore_wait_timeout_ms;
+		     waited += tfork_restore_wait_poll_us / 1000) {
+			if ((int)futex_get(np) <= participants)
+				break;
+			usleep(tfork_restore_wait_poll_us);
+		}
+
+		if ((int)futex_get(np) > participants) {
+			pr_err("tfork restore wait timed out after %dms: participants=%d nr_in_progress=%d start_stage=%d task_cr_err=%d nr_tasks=%d nr_threads=%d nr_helpers=%d\n",
+			       tfork_restore_wait_timeout_ms,
+			       participants, (int)futex_get(np),
+			       (int)futex_get(&task_entries->start),
+			       get_task_cr_err(),
+			       task_entries->nr_tasks, task_entries->nr_threads,
+			       task_entries->nr_helpers);
+			set_cr_errno(ETIMEDOUT);
+			return -ETIMEDOUT;
+		}
+	} else {
+		futex_wait_while_gt(np, participants);
+	}
 
-	futex_wait_while_gt(np, participants);
 	ret = (int)futex_get(np);
 	if (ret < 0) {
+		pr_err("restore wait aborted: participants=%d nr_in_progress=%d start_stage=%d task_cr_err=%d\n",
+		       participants, ret, (int)futex_get(&task_entries->start),
+		       get_task_cr_err());
 		set_cr_errno(get_task_cr_err());
 		return ret;
 	}
@@ -227,6 +256,17 @@ static inline void __restore_switch_stage(int next_stage)
 
 static int restore_switch_stage(int next_stage)
 {
+	if (opts.tfork.active)
+		pr_warn("tfork: restore_switch_stage %d participants=%d nr_tasks=%d nr_threads=%d nr_helpers=%d\n",
+			next_stage, stage_participants(next_stage),
+			task_entries->nr_tasks, task_entries->nr_threads,
+			task_entries->nr_helpers);
+	else
+		pr_info("restore_switch_stage %d participants=%d nr_tasks=%d nr_threads=%d nr_helpers=%d\n",
+			next_stage, stage_participants(next_stage),
+			task_entries->nr_tasks, task_entries->nr_threads,
+			task_entries->nr_helpers);
+
 	__restore_switch_stage(next_stage);
 	return restore_wait_inprogress_tasks();
 }
@@ -1395,6 +1435,16 @@ static inline int fork_with_pid(struct pstree_item *item)
 
 	ca.item = item;
 	ca.clone_flags = rsti(item)->clone_flags;
+	if (opts.tfork.active && item != root_item &&
+	    !(ca.clone_flags & CLONE_NEWPID) &&
+	    item->pid->ns_level > 1 &&
+	    item->pid->ns[0].ns_pid == INIT_PID) {
+		pr_info("tfork: repairing missing CLONE_NEWPID for pidns init uid=%d local=%d parent_local=%d level=%d\n",
+			uid(item), pid,
+			item->parent ? localpid(item->parent) : -1,
+			item->pid->ns_level);
+		ca.clone_flags |= CLONE_NEWPID;
+	}
 
 	BUG_ON(ca.clone_flags & CLONE_VM);
 
@@ -1434,27 +1484,72 @@ static inline int fork_with_pid(struct pstree_item *item)
 		strip |= CLONE_NEWUSER;
 
 	if (kdat.has_clone3_set_tid) {
-		if (item->pid->ns_level == 1)
+		if (opts.tfork.active && (ca.clone_flags & CLONE_NEWPID)) {
+			pr_info("tfork: restore pidns init uid=%d local pid %d with fresh parent pid, dumped chain level=%d\n",
+				uid(item), pid, item->pid->ns_level);
 			ret = clone3_with_pid_noasan(restore_task_with_children, &ca,
 						     ca.clone_flags & ~strip, SIGCHLD, pid);
-		else
+		} else if (item->pid->ns_level == 1)
+			ret = clone3_with_pid_noasan(restore_task_with_children, &ca,
+						     ca.clone_flags & ~strip, SIGCHLD, pid);
+		else {
+			struct pid tfork_pid = {};
+			struct pid *restore_pid = item->pid;
+
+			if (opts.tfork.active && (root_ns_mask & CLONE_NEWPID) &&
+			    root_item && root_item->pid->ns_level > 1 &&
+			    item->pid->ns_level > 1) {
+				/*
+				 * Copy only scalar pid identity. struct pid also
+				 * embeds rb_node links owned by the dumped pid trees;
+				 * copying those nodes into a temporary stack object
+				 * corrupts the tree metadata if it ever gets reused.
+				 */
+				tfork_pid.item = item->pid->item;
+				tfork_pid.real = item->pid->real;
+				tfork_pid.local = item->pid->local;
+				tfork_pid.uid = item->pid->uid;
+				tfork_pid.state = item->pid->state;
+				tfork_pid.stop_signo = item->pid->stop_signo;
+				tfork_pid.ns_level = item->pid->ns_level - 1;
+				tfork_pid.leaf_ns_id = item->pid->leaf_ns_id;
+				memcpy(tfork_pid.ns, item->pid->ns, sizeof(tfork_pid.ns));
+				restore_pid = &tfork_pid;
+				pr_info("tfork: restore pid uid=%d local=%d with rebased pid chain level %d -> %d\n",
+					uid(item), pid, item->pid->ns_level,
+					restore_pid->ns_level);
+			}
 			ret = clone3_with_nested_pid_noasan(restore_task_with_children, &ca,
 							    ca.clone_flags & ~strip,
-							    SIGCHLD, item->pid);
+							    SIGCHLD, restore_pid);
+		}
 	} else {
 		BUG_ON(item->pid->ns_level >= 1);
 		close_pid_proc();
 		ret = clone_noasan(restore_task_with_children,
 				   (ca.clone_flags & ~strip) | SIGCHLD, &ca);
 	}
 	if (ret < 0) {
+		pr_err("fork_with_pid failed item uid=%d local=%d real=%d parent_local=%d flags=0x%lx stripped_flags=0x%lx ns_level=%d root_ns_mask=0x%lx tfork=%d\n",
+		       uid(item), pid, realpid(item),
+		       item->parent ? localpid(item->parent) : -1,
+		       ca.clone_flags, ca.clone_flags & ~strip,
+		       item->pid->ns_level, root_ns_mask,
+		       opts.tfork.active ? 1 : 0);
+		if (item->pid->ns_level > 0)
+			pr_err("fork_with_pid pid chain uid=%d ns=%d/%d/%d/%d\n",
+			       uid(item),
+			       item->pid->ns[0].ns_pid,
+			       item->pid->ns_level > 1 ? item->pid->ns[1].ns_pid : -1,
+			       item->pid->ns_level > 2 ? item->pid->ns[2].ns_pid : -1,
+			       item->pid->ns_level > 3 ? item->pid->ns[3].ns_pid : -1);
 		pr_perror("Can't fork for %d", pid);
 		if (errno == EEXIST)
 			set_cr_errno(EEXIST);
 		goto err_unlock;
 	}
 
-	if (item == root_item) {
+	if (opts.tfork.active || item == root_item) {
 		item->pid->real = ret;
 		pr_debug("PID: real %d virt %d\n", item->pid->real, localpid(item));
 	}
@@ -4045,6 +4140,7 @@ static int sigreturn_restore(struct task_restore_args *task_args, unsigned long
 	task_args->vdso_rt_size = vdso_rt_size;
 	task_args->can_map_vdso = kdat.can_map_vdso;
 	task_args->has_clone3_set_tid = kdat.has_clone3_set_tid;
+	task_args->tfork_active = opts.tfork.active;
 
 	new_sp = restorer_stack(task_args->t->mz);
 

criu/criu/cr-tfork.c

@@ -271,7 +271,7 @@ int tfork_read_cropt(void)
 		return 0;
 
 	snap_path = opts.tfork.snap_root;
-	if (!snap_path && opts.tfork.copies > 1 && opts.tfork.snap_roots &&
+	if (!snap_path && opts.tfork.copies >= 1 && opts.tfork.snap_roots &&
 	    opts.tfork.copy_idx < opts.tfork.snap_roots_n)
 		snap_path = opts.tfork.snap_roots[opts.tfork.copy_idx];
 
@@ -966,8 +966,9 @@ int cr_tfork_tasks(pid_t pid)
 		opts.tfork.pidfd_map[opts.tfork.pidfd_map_nr].pidfd = pidfd;
 		opts.tfork.pidfd_map[opts.tfork.pidfd_map_nr].memfd = -1;
 		opts.tfork.pidfd_map_nr++;
-		pr_info("tfork: pidfd %d for pid %d (vpid %d uid %d)\n",
-			pidfd, item->pid->real, localpid(item), uid(item));
+		pr_info("tfork: pidfd %d for pid %d (vpid %d uid %d nsid %d level %d)\n",
+			pidfd, item->pid->real, localpid(item), uid(item),
+			item->pid->leaf_ns_id, item->pid->ns_level);
 	}
 
 	ret = run_scripts(ACT_PRE_TFORK_RESTORE);
@@ -1078,7 +1079,7 @@ int cr_tfork_tasks(pid_t pid)
 			list_for_each_entry(cgo_iter, &opts.new_cgroup_roots, node)
 				rpc_n_cg_root++;
 
-			rpc_max = 32 + 2 * (rpc_n_ifd + rpc_n_ext + rpc_n_cg_root
+			rpc_max = 33 + 2 * (rpc_n_ifd + rpc_n_ext + rpc_n_cg_root
 					    + opts.tfork.snap_mount_n)
 				+ rpc_n_copy_args + 2;
 			rpc_argv = calloc(rpc_max, sizeof(*rpc_argv));
@@ -1098,6 +1099,7 @@ int cr_tfork_tasks(pid_t pid)
 			rpc_argv[rpc_n++] = "-o";
 			rpc_argv[rpc_n++] = restore_log_arg;
 			rpc_argv[rpc_n++] = "-v2";
+			rpc_argv[rpc_n++] = "--keep-pid-hierarchy";
 
 			if (opts.root) {
 				rpc_argv[rpc_n++] = "--root";
@@ -1171,7 +1173,7 @@ int cr_tfork_tasks(pid_t pid)
 				rpc_argv[rpc_n++] = "--tfork-snap-mounts";
 				rpc_argv[rpc_n++] = snap_mounts_csv;
 			}
-			if (opts.tfork.copies > 1) {
+			if (opts.tfork.copies >= 1) {
 				snprintf(copies_arg, sizeof(copies_arg),
 					 "%d", opts.tfork.copies);
 				rpc_argv[rpc_n++] = "--tfork-copies";
@@ -1267,7 +1269,7 @@ int cr_tfork_tasks(pid_t pid)
 		buf[off] = '\0';
 		end = buf + off;
 
-		argv_max = 8 + 2;
+		argv_max = 8 + 3;
 		for (p = buf; p < end; p++)
 			if (*p == '\0')
 				argv_max++;
@@ -1312,6 +1314,11 @@ int cr_tfork_tasks(pid_t pid)
 			argv_new[argc_new++] = "--pidfile";
 			argv_new[argc_new++] = pidfile_arg;
 		}
+		if ((size_t)argc_new + 1 >= argv_max) {
+			pr_err("tfork restore argv overflow: used=%d max=%zu\n", argc_new, argv_max);
+			exit(1);
+		}
+		argv_new[argc_new++] = "--keep-pid-hierarchy";
 		argv_new[argc_new] = NULL;
 
 		execv("/proc/self/exe", argv_new);

criu/criu/crtools.c

@@ -332,25 +332,30 @@ int main(int argc, char *argv[], char *envp[])
 		if (opts.tree_id)
 			pr_warn("Using -t with criu restore is obsoleted\n");
 
-		if (opts.tfork.copies > 1) {
+		if (!opts.tfork.active && opts.tfork.copies >= 1) {
+			pr_err("--tfork-copies requires --tfork-restore "
+			       "(use 'criu tfork --tfork-copies=N', not "
+			       "'criu restore --tfork-copies=N')\n");
+			return 1;
+		}
+
+		if (opts.tfork.active && opts.tfork.copies >= 1) {
 			int n = opts.tfork.copies, i;
 			pid_t *children;
 			int (*ready_pipes)[2];
 			int failed = 0;
 			const char *base_log = opts.output;
 
-			const int ns_flags =
-				CLONE_NEWPID | CLONE_NEWNS;
-
-			if (!opts.tfork.active) {
-				pr_err("--tfork-copies>1 requires --tfork-restore "
-				       "(use 'criu tfork --tfork-copies=N', not "
-				       "'criu restore --tfork-copies=N')\n");
-				return 1;
-			}
+			/*
+			 * The n-copy helper must not create/occupy PID 1 in a new
+			 * PID namespace. CRIU restores the real root task as PID 1
+			 * from the image; if the helper has already consumed it,
+			 * restore fails with EEXIST ("Can't fork for 1").
+			 */
+			const int ns_flags = CLONE_NEWNS;
 
 			if (!opts.restore_detach) {
-				pr_err("--tfork-copies>1 requires --restore-detached\n");
+				pr_err("--tfork-copies requires --restore-detached\n");
 				return 1;
 			}
 
@@ -445,6 +450,13 @@ int main(int argc, char *argv[], char *envp[])
 					if (opts.tfork.snap_roots_n > 0)
 						opts.root = opts.tfork.snap_roots[i];
 
+					/*
+					 * The n-copy child only creates the per-copy mount namespace. PID
+					 * namespaces must be recreated by CRIU from the image, otherwise the
+					 * helper would occupy PID 1 before the restored root task.
+					 */
+					opts.keep_pid_hierarchy = 0;
+
 					if (tfork_load_ncopy_fabric(i)) {
 						pr_err("tfork-ncopy: copy %d fabric load failed\n",
 						       i);

criu/criu/image.c

@@ -374,6 +374,7 @@ int prepare_inventory(InventoryEntry *he)
 	he->has_lsmtype = true;
 	he->lsmtype = host_lsm_type();
 
+	pid_init_dump(crt.i.pid, &crt.i);
 	crt.i.pid->state = TASK_ALIVE;
 	crt.i.pid->real = getpid();
 	if (get_task_ids(&crt.i))

criu/criu/include/namespaces.h

@@ -187,6 +187,7 @@ extern int restore_mnt_ns(int rst, int *cwd_fd);
 
 extern int dump_task_ns_ids(struct pstree_item *);
 extern int predump_task_ns_ids(struct pstree_item *);
+extern int finalize_nested_pid_ns_ids(void);
 extern int rst_add_ns_id(unsigned int id, struct pstree_item *, struct ns_desc *nd);
 extern struct ns_id *lookup_ns_by_id(unsigned int id, struct ns_desc *nd);