diff --git a/criu/build.sh b/criu/build.sh index cb1d9966c..cdb1cb472 100755 --- a/criu/build.sh +++ b/criu/build.sh @@ -93,12 +93,25 @@ for mod in "${ACTIVE_MODULES[@]}"; do fi if lsmod | awk '{print $1}' | grep -qx "${mod}"; then - echo " -- ${mod}: already loaded, rmmod then insmod" - rmmod "${mod}" 2>/dev/null || true - else - echo " -- ${mod}: insmod" + echo " -- ${mod}: already loaded, trying reload" + if ! rmmod "${mod}"; then + echo "build.sh: failed to unload loaded module ${mod}; aborting to avoid stale module" >&2 + echo "build.sh: stop running tfork/podman containers that may still hold ${mod}, then retry" >&2 + exit 1 + fi + if lsmod | awk '{print $1}' | grep -qx "${mod}"; then + echo "build.sh: module ${mod} is still loaded after rmmod; aborting to avoid stale module" >&2 + echo "build.sh: stop running tfork/podman containers that may still hold ${mod}, then retry" >&2 + exit 1 + fi + fi + + echo " -- ${mod}: insmod" + if ! insmod "${ko_path}"; then + echo "build.sh: failed to insert rebuilt module ${mod} from ${ko_path}" >&2 + echo "build.sh: if the old module is still active, stop running tfork/podman containers and rerun this script" >&2 + exit 1 fi - insmod "${ko_path}" done echo " -- verify all modules loaded" diff --git a/criu/criu/clone-noasan.c b/criu/criu/clone-noasan.c index 4ba7f6f31..368f6aa1e 100644 --- a/criu/criu/clone-noasan.c +++ b/criu/criu/clone-noasan.c @@ -78,7 +78,10 @@ int clone3_with_pid_noasan(int (*fn)(void *), void *arg, int flags, int exit_sig c_args.flags = flags; c_args.set_tid = ptr_to_u64(&pid); c_args.set_tid_size = 1; + pr_info("clone3 set_tid pid=%d flags=0x%x size=1\n", pid, flags); pid = syscall(__NR_clone3, &c_args, sizeof(c_args)); + if (pid < 0) + pr_perror("clone3 set_tid failed flags=0x%x size=1", flags); if (pid == 0) exit(fn(arg)); return pid; @@ -99,6 +102,12 @@ int clone3_with_nested_pid_noasan(int (*fn)(void *), void *arg, int flags, int e BUG_ON(pid->ns_level > MAX_PID_NS_LEVEL || pid->ns_level <= 1); for (i = 0; i < pid->ns_level; i++) tids[i] = pid->ns[i].ns_pid; + pr_info("clone3 nested set_tid flags=0x%x size=%d tids=%d/%d/%d/%d\n", + flags, pid->ns_level, + tids[0], + pid->ns_level > 1 ? tids[1] : -1, + pid->ns_level > 2 ? tids[2] : -1, + pid->ns_level > 3 ? tids[3] : -1); if (!(flags & CLONE_PARENT)) { if (exit_signal != SIGCHLD) { @@ -112,6 +121,13 @@ int clone3_with_nested_pid_noasan(int (*fn)(void *), void *arg, int flags, int e c_args.set_tid = ptr_to_u64(tids); c_args.set_tid_size = pid->ns_level; pid_ret = syscall(__NR_clone3, &c_args, sizeof(c_args)); + if (pid_ret < 0) + pr_perror("clone3 nested set_tid failed flags=0x%x size=%d tids=%d/%d/%d/%d", + flags, pid->ns_level, + tids[0], + pid->ns_level > 1 ? tids[1] : -1, + pid->ns_level > 2 ? tids[2] : -1, + pid->ns_level > 3 ? tids[3] : -1); if (pid_ret == 0) exit(fn(arg)); return pid_ret; diff --git a/criu/criu/cr-dump.c b/criu/criu/cr-dump.c index d196521fc..5179e5e53 100644 --- a/criu/criu/cr-dump.c +++ b/criu/criu/cr-dump.c @@ -885,6 +885,7 @@ static int collect_pstree_ids_predump(void) * write_img_inventory(). */ + pid_init_dump(crt.i.pid, &crt.i); crt.i.pid->state = TASK_ALIVE; crt.i.pid->real = getpid(); @@ -2360,6 +2361,9 @@ int cr_dump_tasks(pid_t pid) if (dump_zombies()) goto err; + if (finalize_nested_pid_ns_ids()) + goto err; + if (dump_pstree(root_item)) goto err; diff --git a/criu/criu/cr-restore.c b/criu/criu/cr-restore.c index d85279a82..bd4fa8b0a 100644 --- a/criu/criu/cr-restore.c +++ b/criu/criu/cr-restore.c @@ -185,10 +185,39 @@ static int __restore_wait_inprogress_tasks(int participants) { int ret; futex_t *np = &task_entries->nr_in_progress; + const int tfork_restore_wait_timeout_ms = 10000; + const int tfork_restore_wait_poll_us = 100000; + + if (opts.tfork.active) { + int waited; + + for (waited = 0; waited < tfork_restore_wait_timeout_ms; + waited += tfork_restore_wait_poll_us / 1000) { + if ((int)futex_get(np) <= participants) + break; + usleep(tfork_restore_wait_poll_us); + } + + if ((int)futex_get(np) > participants) { + pr_err("tfork restore wait timed out after %dms: participants=%d nr_in_progress=%d start_stage=%d task_cr_err=%d nr_tasks=%d nr_threads=%d nr_helpers=%d\n", + tfork_restore_wait_timeout_ms, + participants, (int)futex_get(np), + (int)futex_get(&task_entries->start), + get_task_cr_err(), + task_entries->nr_tasks, task_entries->nr_threads, + task_entries->nr_helpers); + set_cr_errno(ETIMEDOUT); + return -ETIMEDOUT; + } + } else { + futex_wait_while_gt(np, participants); + } - futex_wait_while_gt(np, participants); ret = (int)futex_get(np); if (ret < 0) { + pr_err("restore wait aborted: participants=%d nr_in_progress=%d start_stage=%d task_cr_err=%d\n", + participants, ret, (int)futex_get(&task_entries->start), + get_task_cr_err()); set_cr_errno(get_task_cr_err()); return ret; } @@ -227,6 +256,17 @@ static inline void __restore_switch_stage(int next_stage) static int restore_switch_stage(int next_stage) { + if (opts.tfork.active) + pr_warn("tfork: restore_switch_stage %d participants=%d nr_tasks=%d nr_threads=%d nr_helpers=%d\n", + next_stage, stage_participants(next_stage), + task_entries->nr_tasks, task_entries->nr_threads, + task_entries->nr_helpers); + else + pr_info("restore_switch_stage %d participants=%d nr_tasks=%d nr_threads=%d nr_helpers=%d\n", + next_stage, stage_participants(next_stage), + task_entries->nr_tasks, task_entries->nr_threads, + task_entries->nr_helpers); + __restore_switch_stage(next_stage); return restore_wait_inprogress_tasks(); } @@ -1395,6 +1435,16 @@ static inline int fork_with_pid(struct pstree_item *item) ca.item = item; ca.clone_flags = rsti(item)->clone_flags; + if (opts.tfork.active && item != root_item && + !(ca.clone_flags & CLONE_NEWPID) && + item->pid->ns_level > 1 && + item->pid->ns[0].ns_pid == INIT_PID) { + pr_info("tfork: repairing missing CLONE_NEWPID for pidns init uid=%d local=%d parent_local=%d level=%d\n", + uid(item), pid, + item->parent ? localpid(item->parent) : -1, + item->pid->ns_level); + ca.clone_flags |= CLONE_NEWPID; + } BUG_ON(ca.clone_flags & CLONE_VM); @@ -1434,13 +1484,45 @@ static inline int fork_with_pid(struct pstree_item *item) strip |= CLONE_NEWUSER; if (kdat.has_clone3_set_tid) { - if (item->pid->ns_level == 1) + if (opts.tfork.active && (ca.clone_flags & CLONE_NEWPID)) { + pr_info("tfork: restore pidns init uid=%d local pid %d with fresh parent pid, dumped chain level=%d\n", + uid(item), pid, item->pid->ns_level); ret = clone3_with_pid_noasan(restore_task_with_children, &ca, ca.clone_flags & ~strip, SIGCHLD, pid); - else + } else if (item->pid->ns_level == 1) + ret = clone3_with_pid_noasan(restore_task_with_children, &ca, + ca.clone_flags & ~strip, SIGCHLD, pid); + else { + struct pid tfork_pid = {}; + struct pid *restore_pid = item->pid; + + if (opts.tfork.active && (root_ns_mask & CLONE_NEWPID) && + root_item && root_item->pid->ns_level > 1 && + item->pid->ns_level > 1) { + /* + * Copy only scalar pid identity. struct pid also + * embeds rb_node links owned by the dumped pid trees; + * copying those nodes into a temporary stack object + * corrupts the tree metadata if it ever gets reused. + */ + tfork_pid.item = item->pid->item; + tfork_pid.real = item->pid->real; + tfork_pid.local = item->pid->local; + tfork_pid.uid = item->pid->uid; + tfork_pid.state = item->pid->state; + tfork_pid.stop_signo = item->pid->stop_signo; + tfork_pid.ns_level = item->pid->ns_level - 1; + tfork_pid.leaf_ns_id = item->pid->leaf_ns_id; + memcpy(tfork_pid.ns, item->pid->ns, sizeof(tfork_pid.ns)); + restore_pid = &tfork_pid; + pr_info("tfork: restore pid uid=%d local=%d with rebased pid chain level %d -> %d\n", + uid(item), pid, item->pid->ns_level, + restore_pid->ns_level); + } ret = clone3_with_nested_pid_noasan(restore_task_with_children, &ca, ca.clone_flags & ~strip, - SIGCHLD, item->pid); + SIGCHLD, restore_pid); + } } else { BUG_ON(item->pid->ns_level >= 1); close_pid_proc(); @@ -1448,13 +1530,26 @@ static inline int fork_with_pid(struct pstree_item *item) (ca.clone_flags & ~strip) | SIGCHLD, &ca); } if (ret < 0) { + pr_err("fork_with_pid failed item uid=%d local=%d real=%d parent_local=%d flags=0x%lx stripped_flags=0x%lx ns_level=%d root_ns_mask=0x%lx tfork=%d\n", + uid(item), pid, realpid(item), + item->parent ? localpid(item->parent) : -1, + ca.clone_flags, ca.clone_flags & ~strip, + item->pid->ns_level, root_ns_mask, + opts.tfork.active ? 1 : 0); + if (item->pid->ns_level > 0) + pr_err("fork_with_pid pid chain uid=%d ns=%d/%d/%d/%d\n", + uid(item), + item->pid->ns[0].ns_pid, + item->pid->ns_level > 1 ? item->pid->ns[1].ns_pid : -1, + item->pid->ns_level > 2 ? item->pid->ns[2].ns_pid : -1, + item->pid->ns_level > 3 ? item->pid->ns[3].ns_pid : -1); pr_perror("Can't fork for %d", pid); if (errno == EEXIST) set_cr_errno(EEXIST); goto err_unlock; } - if (item == root_item) { + if (opts.tfork.active || item == root_item) { item->pid->real = ret; pr_debug("PID: real %d virt %d\n", item->pid->real, localpid(item)); } @@ -4045,6 +4140,7 @@ static int sigreturn_restore(struct task_restore_args *task_args, unsigned long task_args->vdso_rt_size = vdso_rt_size; task_args->can_map_vdso = kdat.can_map_vdso; task_args->has_clone3_set_tid = kdat.has_clone3_set_tid; + task_args->tfork_active = opts.tfork.active; new_sp = restorer_stack(task_args->t->mz); diff --git a/criu/criu/cr-tfork.c b/criu/criu/cr-tfork.c index ea64d6627..f99a6d2da 100644 --- a/criu/criu/cr-tfork.c +++ b/criu/criu/cr-tfork.c @@ -271,7 +271,7 @@ int tfork_read_cropt(void) return 0; snap_path = opts.tfork.snap_root; - if (!snap_path && opts.tfork.copies > 1 && opts.tfork.snap_roots && + if (!snap_path && opts.tfork.copies >= 1 && opts.tfork.snap_roots && opts.tfork.copy_idx < opts.tfork.snap_roots_n) snap_path = opts.tfork.snap_roots[opts.tfork.copy_idx]; @@ -966,8 +966,9 @@ int cr_tfork_tasks(pid_t pid) opts.tfork.pidfd_map[opts.tfork.pidfd_map_nr].pidfd = pidfd; opts.tfork.pidfd_map[opts.tfork.pidfd_map_nr].memfd = -1; opts.tfork.pidfd_map_nr++; - pr_info("tfork: pidfd %d for pid %d (vpid %d uid %d)\n", - pidfd, item->pid->real, localpid(item), uid(item)); + pr_info("tfork: pidfd %d for pid %d (vpid %d uid %d nsid %d level %d)\n", + pidfd, item->pid->real, localpid(item), uid(item), + item->pid->leaf_ns_id, item->pid->ns_level); } ret = run_scripts(ACT_PRE_TFORK_RESTORE); @@ -1078,7 +1079,7 @@ int cr_tfork_tasks(pid_t pid) list_for_each_entry(cgo_iter, &opts.new_cgroup_roots, node) rpc_n_cg_root++; - rpc_max = 32 + 2 * (rpc_n_ifd + rpc_n_ext + rpc_n_cg_root + rpc_max = 33 + 2 * (rpc_n_ifd + rpc_n_ext + rpc_n_cg_root + opts.tfork.snap_mount_n) + rpc_n_copy_args + 2; rpc_argv = calloc(rpc_max, sizeof(*rpc_argv)); @@ -1098,6 +1099,7 @@ int cr_tfork_tasks(pid_t pid) rpc_argv[rpc_n++] = "-o"; rpc_argv[rpc_n++] = restore_log_arg; rpc_argv[rpc_n++] = "-v2"; + rpc_argv[rpc_n++] = "--keep-pid-hierarchy"; if (opts.root) { rpc_argv[rpc_n++] = "--root"; @@ -1171,7 +1173,7 @@ int cr_tfork_tasks(pid_t pid) rpc_argv[rpc_n++] = "--tfork-snap-mounts"; rpc_argv[rpc_n++] = snap_mounts_csv; } - if (opts.tfork.copies > 1) { + if (opts.tfork.copies >= 1) { snprintf(copies_arg, sizeof(copies_arg), "%d", opts.tfork.copies); rpc_argv[rpc_n++] = "--tfork-copies"; @@ -1267,7 +1269,7 @@ int cr_tfork_tasks(pid_t pid) buf[off] = '\0'; end = buf + off; - argv_max = 8 + 2; + argv_max = 8 + 3; for (p = buf; p < end; p++) if (*p == '\0') argv_max++; @@ -1312,6 +1314,11 @@ int cr_tfork_tasks(pid_t pid) argv_new[argc_new++] = "--pidfile"; argv_new[argc_new++] = pidfile_arg; } + if ((size_t)argc_new + 1 >= argv_max) { + pr_err("tfork restore argv overflow: used=%d max=%zu\n", argc_new, argv_max); + exit(1); + } + argv_new[argc_new++] = "--keep-pid-hierarchy"; argv_new[argc_new] = NULL; execv("/proc/self/exe", argv_new); diff --git a/criu/criu/crtools.c b/criu/criu/crtools.c index df40ec14f..639ad06f5 100644 --- a/criu/criu/crtools.c +++ b/criu/criu/crtools.c @@ -332,25 +332,30 @@ int main(int argc, char *argv[], char *envp[]) if (opts.tree_id) pr_warn("Using -t with criu restore is obsoleted\n"); - if (opts.tfork.copies > 1) { + if (!opts.tfork.active && opts.tfork.copies >= 1) { + pr_err("--tfork-copies requires --tfork-restore " + "(use 'criu tfork --tfork-copies=N', not " + "'criu restore --tfork-copies=N')\n"); + return 1; + } + + if (opts.tfork.active && opts.tfork.copies >= 1) { int n = opts.tfork.copies, i; pid_t *children; int (*ready_pipes)[2]; int failed = 0; const char *base_log = opts.output; - const int ns_flags = - CLONE_NEWPID | CLONE_NEWNS; - - if (!opts.tfork.active) { - pr_err("--tfork-copies>1 requires --tfork-restore " - "(use 'criu tfork --tfork-copies=N', not " - "'criu restore --tfork-copies=N')\n"); - return 1; - } + /* + * The n-copy helper must not create/occupy PID 1 in a new + * PID namespace. CRIU restores the real root task as PID 1 + * from the image; if the helper has already consumed it, + * restore fails with EEXIST ("Can't fork for 1"). + */ + const int ns_flags = CLONE_NEWNS; if (!opts.restore_detach) { - pr_err("--tfork-copies>1 requires --restore-detached\n"); + pr_err("--tfork-copies requires --restore-detached\n"); return 1; } @@ -445,6 +450,13 @@ int main(int argc, char *argv[], char *envp[]) if (opts.tfork.snap_roots_n > 0) opts.root = opts.tfork.snap_roots[i]; + /* + * The n-copy child only creates the per-copy mount namespace. PID + * namespaces must be recreated by CRIU from the image, otherwise the + * helper would occupy PID 1 before the restored root task. + */ + opts.keep_pid_hierarchy = 0; + if (tfork_load_ncopy_fabric(i)) { pr_err("tfork-ncopy: copy %d fabric load failed\n", i); diff --git a/criu/criu/image.c b/criu/criu/image.c index 2783b5797..1a93344d5 100644 --- a/criu/criu/image.c +++ b/criu/criu/image.c @@ -374,6 +374,7 @@ int prepare_inventory(InventoryEntry *he) he->has_lsmtype = true; he->lsmtype = host_lsm_type(); + pid_init_dump(crt.i.pid, &crt.i); crt.i.pid->state = TASK_ALIVE; crt.i.pid->real = getpid(); if (get_task_ids(&crt.i)) diff --git a/criu/criu/include/namespaces.h b/criu/criu/include/namespaces.h index e442e0a39..7a5aa6437 100644 --- a/criu/criu/include/namespaces.h +++ b/criu/criu/include/namespaces.h @@ -187,6 +187,7 @@ extern int restore_mnt_ns(int rst, int *cwd_fd); extern int dump_task_ns_ids(struct pstree_item *); extern int predump_task_ns_ids(struct pstree_item *); +extern int finalize_nested_pid_ns_ids(void); extern int rst_add_ns_id(unsigned int id, struct pstree_item *, struct ns_desc *nd); extern struct ns_id *lookup_ns_by_id(unsigned int id, struct ns_desc *nd); diff --git a/criu/criu/include/pstree.h b/criu/criu/include/pstree.h index f9bdeffd1..51617015a 100644 --- a/criu/criu/include/pstree.h +++ b/criu/criu/include/pstree.h @@ -20,6 +20,23 @@ extern atomic_t pid_uid_generator; #define HELPER_UID_BASE (0x40000000) +static inline void pid_init_dump(struct pid *pid, struct pstree_item *item) +{ + *pid = (struct pid){ + .item = item, + .real = -1, + .local = -1, + .uid = atomic_inc_return(&pid_uid_generator), + .state = TASK_UNDEF, + .stop_signo = -1, + .ns_level = -1, + .leaf_ns_id = ALL_PID_NS_ID, + }; + rb_init_node(&pid->leaf_ns_node); + rb_init_node(&pid->root_ns_node); + rb_init_node(&pid->uid_node); +} + struct pstree_item { struct pstree_item *parent; struct list_head children; /* list of my children */ diff --git a/criu/criu/include/restorer.h b/criu/criu/include/restorer.h index 73e27caa3..40bb132d1 100644 --- a/criu/criu/include/restorer.h +++ b/criu/criu/include/restorer.h @@ -242,6 +242,7 @@ struct task_restore_args { int child_subreaper; int membarrier_registration_mask; bool has_clone3_set_tid; + bool tfork_active; /* * info about rseq from libc used to diff --git a/criu/criu/namespaces.c b/criu/criu/namespaces.c index df224db68..11909cea2 100644 --- a/criu/criu/namespaces.c +++ b/criu/criu/namespaces.c @@ -592,6 +592,93 @@ static unsigned int get_ns_id(int pid, struct ns_desc *nd, protobuf_c_boolean *s return __get_ns_id(pid, nd, supported, NULL); } +static unsigned int add_nested_pid_leaf_ns_id(struct pstree_item *item) +{ + struct ns_id *nsid; + + nsid = xzalloc(sizeof(*nsid)); + if (!nsid) + return 0; + + nsid->type = NS_OTHER; + nsid->kid = 0; + nsid->ns_populated = true; + nsid_add(nsid, &pid_ns_desc, ns_next_id++, localpid(item)); + + pr_info("Add nested pid leaf ns %d for task %d(%d), level %d\n", + nsid->id, localpid(item), realpid(item), item->pid->ns_level); + return nsid->id; +} + +static unsigned int ensure_task_leaf_pid_ns_id(struct pstree_item *item); + +static unsigned int task_leaf_pid_ns_id(struct pstree_item *item, unsigned int proc_pid_ns_id) +{ + struct pstree_item *parent = item->parent; + unsigned int selected; + + /* + * os4agent stores localpid as the innermost NSpid (pid->ns[0]). + * Keep pstree_entry.nsid at the same namespace level. Otherwise a + * nested pid namespace init such as bwrap can become (nsid=N, + * localpid=1) and collide with the container init in the same nsid. + */ + if (parent && ensure_task_leaf_pid_ns_id(parent) == 0) + return 0; + + if (parent && parent->pid->leaf_ns_id != ALL_PID_NS_ID) { + if (item->pid->ns_level == parent->pid->ns_level) { + selected = parent->pid->leaf_ns_id; + goto out; + } + if (item->pid->ns_level > parent->pid->ns_level && + proc_pid_ns_id != parent->pid->leaf_ns_id) { + selected = proc_pid_ns_id; + goto out; + } + if (item->pid->ns_level > parent->pid->ns_level) { + selected = add_nested_pid_leaf_ns_id(item); + goto out; + } + } + + selected = proc_pid_ns_id; + +out: + pr_info("pid leaf ns task=%d(%d) uid=%d level=%d parent_level=%d proc_nsid=%u parent_nsid=%d selected=%u\n", + localpid(item), realpid(item), uid(item), item->pid->ns_level, + parent ? parent->pid->ns_level : -1, proc_pid_ns_id, + parent ? parent->pid->leaf_ns_id : -1, selected); + return selected; +} + +static unsigned int ensure_task_leaf_pid_ns_id(struct pstree_item *item) +{ + struct pstree_item *parent = item->parent; + unsigned int proc_pid_ns_id; + + if (parent && ensure_task_leaf_pid_ns_id(parent) == 0) + return 0; + + if (item->pid->leaf_ns_id != ALL_PID_NS_ID) { + if (!parent) + return item->pid->leaf_ns_id; + if (item->pid->ns_level == parent->pid->ns_level && + item->pid->leaf_ns_id == parent->pid->leaf_ns_id) + return item->pid->leaf_ns_id; + if (item->pid->ns_level > parent->pid->ns_level && + item->pid->leaf_ns_id != parent->pid->leaf_ns_id) + return item->pid->leaf_ns_id; + } + + proc_pid_ns_id = get_ns_id(item->pid->real, &pid_ns_desc, NULL); + if (!proc_pid_ns_id) + return 0; + + item->pid->leaf_ns_id = task_leaf_pid_ns_id(item, proc_pid_ns_id); + return item->pid->leaf_ns_id; +} + int dump_one_ns_file(int lfd, u32 id, const struct fd_parms *p) { struct cr_img *img; @@ -771,15 +858,37 @@ int dump_task_ns_ids(struct pstree_item *item) int i; int pid = item->pid->real; TaskKobjIdsEntry *ids = item->ids; + struct pstree_item *parent = item->parent; + unsigned int proc_pid_ns_id; ids->has_pid_ns_id = true; - ids->pid_ns_id = get_ns_id(pid, &pid_ns_desc, NULL); + proc_pid_ns_id = get_ns_id(pid, &pid_ns_desc, NULL); + if (!proc_pid_ns_id) { + pr_err("Can't make pidns id\n"); + return -1; + } + + if (parent && ensure_task_leaf_pid_ns_id(parent) == 0) + return -1; + + ids->pid_ns_id = proc_pid_ns_id; + if (parent && item->pid->ns_level == parent->pid->ns_level) + ids->pid_ns_id = parent->pid->leaf_ns_id; + else if (parent && item->pid->ns_level > parent->pid->ns_level && + ids->pid_ns_id == parent->pid->leaf_ns_id) + ids->pid_ns_id = add_nested_pid_leaf_ns_id(item); + if (!ids->pid_ns_id) { pr_err("Can't make pidns id\n"); return -1; } item->pid->leaf_ns_id = ids->pid_ns_id; + pr_info("dump pid ns task=%d(%d) uid=%d level=%d parent_level=%d proc_nsid=%u parent_nsid=%d selected=%u\n", + localpid(item), realpid(item), uid(item), item->pid->ns_level, + parent ? parent->pid->ns_level : -1, proc_pid_ns_id, + parent ? parent->pid->leaf_ns_id : -1, ids->pid_ns_id); + for (i = 0; i < item->nr_threads; i++) item->threads[i].leaf_ns_id = ids->pid_ns_id; @@ -865,6 +974,40 @@ int dump_task_ns_ids(struct pstree_item *item) return 0; } +int finalize_nested_pid_ns_ids(void) +{ + struct pstree_item *item; + + for_each_pstree_item(item) { + struct pstree_item *parent = item->parent; + unsigned int nsid; + int i; + + if (!parent) + continue; + if (item->pid->ns_level <= parent->pid->ns_level) + continue; + if (item->pid->leaf_ns_id != parent->pid->leaf_ns_id) + continue; + + nsid = add_nested_pid_leaf_ns_id(item); + if (!nsid) + return -1; + + item->pid->leaf_ns_id = nsid; + for (i = 0; i < item->nr_threads; i++) + item->threads[i].leaf_ns_id = nsid; + if (item->ids && item->ids->has_pid_ns_id) + item->ids->pid_ns_id = nsid; + + pr_info("finalize nested pid ns task=%d(%d) uid=%d level=%d parent_nsid=%d selected=%u\n", + localpid(item), realpid(item), uid(item), item->pid->ns_level, + parent->pid->leaf_ns_id, nsid); + } + + return 0; +} + static UsernsEntry userns_entry = USERNS_ENTRY__INIT; #define INVALID_ID (~0U) diff --git a/criu/criu/pie/restorer.c b/criu/criu/pie/restorer.c index 9aaefe502..a63dbdd62 100644 --- a/criu/criu/pie/restorer.c +++ b/criu/criu/pie/restorer.c @@ -97,6 +97,7 @@ static pid_t *helpers; static int n_helpers; static pid_t *zombies; static int n_zombies; +static bool tfork_active_local; static enum faults fi_strategy; bool fault_injected(enum faults f) { @@ -162,7 +163,8 @@ static void sigchld_handler(int signal, siginfo_t *siginfo, void *data) else r = "disappeared with"; - pr_info("Task %d %s %d\n", siginfo->si_pid, r, siginfo->si_status); + pr_err("SIGCHLD during restore: task %d %s %d\n", + siginfo->si_pid, r, siginfo->si_status); futex_abort_and_wake(&task_entries_local->nr_in_progress); /* sa_restorer may be unmaped, so we can't go back to userspace*/ @@ -807,12 +809,24 @@ __visible long __export_restore_thread(struct thread_restore_args *args) } pr_info("%ld: Restored\n", sys_gettid()); - restore_finish_stage(task_entries_local, CR_STATE_RESTORE); + if (args->ta->tfork_active) + pr_debug("tfork: thread restore stage complete pid=%d tid=%ld comm=%s ns_level=%d\n", + args->pid, sys_gettid(), args->comm, args->ns_level); + ret = restore_finish_stage(task_entries_local, CR_STATE_RESTORE); + if (args->ta->tfork_active) + pr_debug("tfork: thread restore barrier returned pid=%d tid=%ld comm=%s stage=%d\n", + args->pid, sys_gettid(), args->comm, ret); if (restore_signals(args->siginfo, args->siginfo_n, false)){ goto core_restore_end; } - restore_finish_stage(task_entries_local, CR_STATE_RESTORE_SIGCHLD); + if (args->ta->tfork_active) + pr_debug("tfork: thread sigchld stage complete pid=%d tid=%ld comm=%s ns_level=%d\n", + args->pid, sys_gettid(), args->comm, args->ns_level); + ret = restore_finish_stage(task_entries_local, CR_STATE_RESTORE_SIGCHLD); + if (args->ta->tfork_active) + pr_debug("tfork: thread sigchld barrier returned pid=%d tid=%ld comm=%s stage=%d\n", + args->pid, sys_gettid(), args->comm, ret); /* * Make sure it's before creds, since it's privileged @@ -1476,6 +1490,11 @@ static int wait_zombies(struct task_restore_args *task_args) ret = sys_waitid(P_PID, task_args->zombies[i], NULL, WNOWAIT | WEXITED, NULL); if (ret == -ECHILD) { + if (task_args->tfork_active) { + pr_warn("tfork: zombie pid %d is not reparented to task %ld; skipping wait to avoid restore barrier deadlock\n", + task_args->zombies[i], sys_getpid()); + continue; + } /* A process isn't reparented to this task yet. * Let's wait when someone complete this stage * and try again. @@ -1752,6 +1771,7 @@ __visible long __export_restore_task(struct task_restore_args *args) fi_strategy = args->fault_strategy; task_entries_local = args->task_entries; + tfork_active_local = args->tfork_active; helpers = args->helpers; n_helpers = args->helpers_n; zombies = args->zombies; @@ -2465,6 +2485,18 @@ __visible long __export_restore_task(struct task_restore_args *args) c_args.set_tid = ptr_to_u64(thread_args[i].tid_in_ns); c_args.flags = clone_flags; c_args.set_tid_size = thread_args[i].ns_level; + if (args->tfork_active && thread_args[i].ns_level > 0) { + /* + * Preserve the TID visible in the clone's innermost PID namespace. + * Outer namespace TIDs are allocated by the kernel so concurrent + * copy helpers cannot collide with each other on the host. + */ + pr_debug("tfork: restore thread pid=%d with innermost tid=%d, set_tid_size %d -> 1\n", + thread_args[i].pid, + thread_args[i].tid_in_ns[0], + thread_args[i].ns_level); + c_args.set_tid_size = 1; + } /* The kernel does stack + stack_size. */ c_args.stack = new_sp - RESTORE_STACK_SIZE; c_args.stack_size = RESTORE_STACK_SIZE; @@ -2495,7 +2527,14 @@ __visible long __export_restore_task(struct task_restore_args *args) args->clone_restore_fn); } if (ret != thread_args[i].pid) { - pr_err("Unable to create a thread: %ld\n", ret); + pr_err("Unable to create a thread: %ld expected=%d ns_level=%d tids=%d/%d/%d/%d tfork=%d\n", + ret, thread_args[i].pid, + thread_args[i].ns_level, + thread_args[i].tid_in_ns[0], + thread_args[i].ns_level > 1 ? thread_args[i].tid_in_ns[1] : -1, + thread_args[i].ns_level > 2 ? thread_args[i].tid_in_ns[2] : -1, + thread_args[i].ns_level > 3 ? thread_args[i].tid_in_ns[3] : -1, + args->tfork_active ? 1 : 0); sys_close(fd); mutex_unlock(&task_entries_local->last_pid_mutex); goto core_restore_end; @@ -2524,13 +2563,33 @@ __visible long __export_restore_task(struct task_restore_args *args) if (restore_membarrier_registrations(args->membarrier_registration_mask) < 0) goto core_restore_end; pr_info("%ld: Restored\n", sys_getpid()); - - restore_finish_stage(task_entries_local, CR_STATE_RESTORE); - + if (args->tfork_active) + pr_debug("tfork: leader restore stage complete pid=%d tid=%ld comm=%s threads=%d ns_level=%d\n", + args->t->pid, sys_getpid(), args->comm, args->nr_threads, + args->t->ns_level); + + ret = restore_finish_stage(task_entries_local, CR_STATE_RESTORE); + if (args->tfork_active) + pr_debug("tfork: leader restore barrier returned pid=%d tid=%ld comm=%s stage=%ld helpers=%u zombies=%u inotify=%u\n", + args->t->pid, sys_getpid(), args->comm, ret, + args->helpers_n, args->zombies_n, args->inotify_fds_n); + + if (args->tfork_active) + pr_debug("tfork: leader wait_helpers start pid=%d tid=%ld comm=%s helpers=%u\n", + args->t->pid, sys_getpid(), args->comm, args->helpers_n); if (wait_helpers(args) < 0) goto core_restore_end; + if (args->tfork_active) + pr_debug("tfork: leader wait_helpers done pid=%d tid=%ld comm=%s\n", + args->t->pid, sys_getpid(), args->comm); + if (args->tfork_active) + pr_debug("tfork: leader wait_zombies start pid=%d tid=%ld comm=%s zombies=%u\n", + args->t->pid, sys_getpid(), args->comm, args->zombies_n); if (wait_zombies(args) < 0) goto core_restore_end; + if (args->tfork_active) + pr_debug("tfork: leader wait_zombies done pid=%d tid=%ld comm=%s\n", + args->t->pid, sys_getpid(), args->comm); ksigfillset(&to_block); ret = sys_sigprocmask(SIG_SETMASK, &to_block, NULL, sizeof(k_rtsigset_t)); @@ -2539,9 +2598,18 @@ __visible long __export_restore_task(struct task_restore_args *args) goto core_restore_end; } + if (args->tfork_active) + pr_debug("tfork: leader cleanup_inotify start pid=%d tid=%ld comm=%s inotify=%u\n", + args->t->pid, sys_getpid(), args->comm, args->inotify_fds_n); if (cleanup_current_inotify_events(args)) goto core_restore_end; + if (args->tfork_active) + pr_debug("tfork: leader cleanup_inotify done pid=%d tid=%ld comm=%s\n", + args->t->pid, sys_getpid(), args->comm); + if (args->tfork_active) + pr_debug("tfork: leader restore sigaction start pid=%d tid=%ld comm=%s\n", + args->t->pid, sys_getpid(), args->comm); if (!args->compatible_mode) { ret = sys_sigaction(SIGCHLD, &args->sigchld_act, NULL, sizeof(k_rtsigset_t)); } else { @@ -2558,16 +2626,38 @@ __visible long __export_restore_task(struct task_restore_args *args) pr_err("Failed to restore SIGCHLD: %ld\n", ret); goto core_restore_end; } + if (args->tfork_active) + pr_debug("tfork: leader restore sigaction done pid=%d tid=%ld comm=%s\n", + args->t->pid, sys_getpid(), args->comm); + if (args->tfork_active) + pr_debug("tfork: leader restore shared signals start pid=%d tid=%ld comm=%s\n", + args->t->pid, sys_getpid(), args->comm); ret = restore_signals(args->siginfo, args->siginfo_n, true); if (ret) goto core_restore_end; + if (args->tfork_active) + pr_debug("tfork: leader restore shared signals done pid=%d tid=%ld comm=%s\n", + args->t->pid, sys_getpid(), args->comm); + if (args->tfork_active) + pr_debug("tfork: leader restore private signals start pid=%d tid=%ld comm=%s\n", + args->t->pid, sys_getpid(), args->comm); ret = restore_signals(args->t->siginfo, args->t->siginfo_n, false); if (ret) goto core_restore_end; - - restore_finish_stage(task_entries_local, CR_STATE_RESTORE_SIGCHLD); + if (args->tfork_active) + pr_debug("tfork: leader restore private signals done pid=%d tid=%ld comm=%s\n", + args->t->pid, sys_getpid(), args->comm); + + if (args->tfork_active) + pr_debug("tfork: leader sigchld stage complete pid=%d tid=%ld comm=%s threads=%d ns_level=%d\n", + args->t->pid, sys_getpid(), args->comm, args->nr_threads, + args->t->ns_level); + ret = restore_finish_stage(task_entries_local, CR_STATE_RESTORE_SIGCHLD); + if (args->tfork_active) + pr_debug("tfork: leader sigchld barrier returned pid=%d tid=%ld comm=%s stage=%ld\n", + args->t->pid, sys_getpid(), args->comm, ret); rst_tcp_socks_all(args); diff --git a/criu/criu/pstree.c b/criu/criu/pstree.c index c448c9516..29f1e5d61 100644 --- a/criu/criu/pstree.c +++ b/criu/criu/pstree.c @@ -272,26 +272,26 @@ struct pstree_item *__alloc_pstree_item(bool rst) INIT_LIST_HEAD(&item->children); INIT_LIST_HEAD(&item->sibling); - item->pid->ns_level = -1; - item->pid->leaf_ns_id = ALL_PID_NS_ID; - item->pid->real = -1; - item->pid->local = -1; - if (!rst) - item->pid->uid = atomic_inc_return(&pid_uid_generator); - else + pid_init_dump(item->pid, item); + else { + item->pid->ns_level = -1; + item->pid->leaf_ns_id = ALL_PID_NS_ID; + item->pid->real = -1; + item->pid->local = -1; item->pid->uid = -1; - item->pid->state = TASK_UNDEF; - item->pid->stop_signo = -1; + item->pid->state = TASK_UNDEF; + item->pid->stop_signo = -1; + item->pid->item = item; + rb_init_node(&item->pid->leaf_ns_node); + rb_init_node(&item->pid->root_ns_node); + rb_init_node(&item->pid->uid_node); + } item->born_sid = -1; item->tfork_pidfd = -1; item->tfork_memfd = -1; item->tfork_pagemap_fd = -1; - item->pid->item = item; futex_init(&item->task_st); - rb_init_node(&item->pid->leaf_ns_node); - rb_init_node(&item->pid->root_ns_node); - rb_init_node(&item->pid->uid_node); return item; } @@ -436,7 +436,10 @@ int dump_pstree(struct pstree_item *root_item) pstree_entry__init(e); tree_entries[nr_items++] = e; - pr_info("Process: %d(%d)\n", localpid(item), realpid(item)); + pr_info("Process: %d(%d) uid=%d nsid=%d level=%d parent=%d parent_nsid=%d\n", + localpid(item), realpid(item), uid(item), item->pid->leaf_ns_id, + item->pid->ns_level, item->parent ? realpid(item->parent) : 0, + item->parent ? item->parent->pid->leaf_ns_id : -1); e->realpid = realpid(item); e->ppid = item->parent ? realpid(item->parent) : 0; @@ -729,11 +732,37 @@ static int __pstree_insert_pid(struct pid *pid_node, struct rb_node *root_parent rb_link_and_balance(&uid_root_rb, &pid_node->uid_node, parent, link); } - return 0; + return 0; err: - rb_erase(&pid_node->root_ns_node, &pid_root_rb[ALL_PID_NS_ID]); - return -1; + rb_erase(&pid_node->root_ns_node, &pid_root_rb[ALL_PID_NS_ID]); + return -1; +} + +static void pstree_remove_pid_if_linked(struct pid *pid_node) +{ + struct pid *found; + bool valid_leaf_ns = pid_node->leaf_ns_id >= 0 && + (unsigned int)pid_node->leaf_ns_id < pid_namespace_count; + + if (pid_node->uid > 0) { + found = __lookup_pid_uid(&uid_root_rb, pid_node->uid, NULL, NULL); + if (found == pid_node) + rb_erase(&pid_node->uid_node, &uid_root_rb); + } + + if (pid_node->leaf_ns_id != ALL_PID_NS_ID && valid_leaf_ns) { + found = __lookup_pid_leaf(&pid_root_rb[pid_node->leaf_ns_id], + pid_node->local, NULL, NULL); + if (found == pid_node) + rb_erase(&pid_node->leaf_ns_node, + &pid_root_rb[pid_node->leaf_ns_id]); + } + + found = __lookup_pid_root(&pid_root_rb[ALL_PID_NS_ID], + pid_node->real, NULL, NULL); + if (found == pid_node) + rb_erase(&pid_node->root_ns_node, &pid_root_rb[ALL_PID_NS_ID]); } int pstree_insert_pid(struct pid *pid_node) @@ -741,13 +770,14 @@ int pstree_insert_pid(struct pid *pid_node) return __pstree_insert_pid(pid_node, NULL, NULL); } -static struct pstree_item *get_or_create_pstree_item(pid_t real, pid_t local, int pidns_id) +static struct pstree_item *get_or_create_pstree_item(pid_t real, pid_t local, int pidns_id, bool *created) { struct pid *found; struct pstree_item *item; - struct rb_node **root_link, *root_parent; - found = __lookup_pid_root(&pid_root_rb[ALL_PID_NS_ID], real, &root_parent, &root_link); + *created = false; + + found = __lookup_pid_root(&pid_root_rb[ALL_PID_NS_ID], real, NULL, NULL); if (found) { if (pidns_id != ALL_PID_NS_ID) { BUG_ON(found->leaf_ns_id != pidns_id || found->local != local); @@ -762,11 +792,7 @@ static struct pstree_item *get_or_create_pstree_item(pid_t real, pid_t local, in item->pid->real = real; item->pid->local = local; item->pid->leaf_ns_id = pidns_id; - - if (__pstree_insert_pid(item->pid, root_parent, root_link) < 0) { - xfree(item); - return NULL; - } + *created = true; return item; } @@ -876,13 +902,24 @@ static int read_pstree_ids(struct pstree_item *pi) */ static int read_one_pstree_item(PstreeEntry *e) { - struct pstree_item *pi; - int ret = -1, i, j; + struct pstree_item *pi = NULL; + int ret = -1, i, j, next_inserted_thread = 1; + bool linked = false, pid_inserted = false, threads_allocated = false; + bool created_item = false; - pi = get_or_create_pstree_item(e->realpid, e->localpid, e->nsid); + pi = get_or_create_pstree_item(e->realpid, e->localpid, e->nsid, &created_item); if (!pi) goto err; + /* + * get_or_create_pstree_item() can only reuse an item that is still + * TASK_UNDEF. Completed items are rejected here, so the unwind below + * cannot tear down a previously parsed pstree item. + */ BUG_ON(pi->pid->state != TASK_UNDEF); + if (!created_item && (pi->threads || pi->nr_threads)) { + pr_err("Refusing to reuse partially populated pstree item for %d\n", e->realpid); + goto err; + } /* * Populate the ns-chain on pi from the thread-leader entry before @@ -915,6 +952,27 @@ static int read_one_pstree_item(PstreeEntry *e) } pi->pid->state = TASK_ALIVE; pi->pid->uid = e->uid; + pi->nr_threads = e->n_threads; + pi->threads = xmalloc(e->n_threads * sizeof(struct pid)); + if (!pi->threads) + goto err; + threads_allocated = true; + + /* note: we don't fail if we have empty ids */ + if (read_pstree_ids(pi) < 0) + goto err; + + if (pi->ids && pi->ids->has_pid_ns_id) { + if (pi->ids->pid_ns_id != pi->pid->leaf_ns_id) { + pr_warn("PID namespace id mismatch for uid %d: pstree=%d ids=%d, keeping pstree\n", + uid(pi), pi->pid->leaf_ns_id, pi->ids->pid_ns_id); + pi->ids->pid_ns_id = pi->pid->leaf_ns_id; + } + } + + if (__pstree_insert_pid(pi->pid, NULL, NULL) < 0) + goto err; + pid_inserted = true; if (e->ppid == 0) { if (root_item) { @@ -938,13 +996,9 @@ static int read_one_pstree_item(PstreeEntry *e) parent = pid->item; pi->parent = parent; list_add(&pi->sibling, &parent->children); + linked = true; } - pi->nr_threads = e->n_threads; - pi->threads = xmalloc(e->n_threads * sizeof(struct pid)); - if (!pi->threads) - goto err; - for (i = 0; i < e->n_threads; i++) { int insert_status; pi->threads[i].uid = e->threads[i]->uid; @@ -960,7 +1014,7 @@ static int read_one_pstree_item(PstreeEntry *e) pi->threads[i].state = TASK_THREAD; pi->threads[i].item = NULL; if (i == 0) { - + /* The leader is indexed through pi->pid, not this mirror. */ pi->pid->ns_level = pi->threads[0].ns_level; pi->pid->local = pi->threads[0].ns[0].ns_pid; memcpy(pi->pid->ns, pi->threads[0].ns, e->threads[0]->n_ns * sizeof(struct pid_ns)); @@ -972,17 +1026,42 @@ static int read_one_pstree_item(PstreeEntry *e) pr_err("Unexpected task %d in a tree %d\n", e->threads[i]->ns[0]->nspid, i); goto err; } + next_inserted_thread = i + 1; } task_entries->nr_threads += e->n_threads; task_entries->nr_tasks++; - /* note: we don't fail if we have empty ids */ - if (read_pstree_ids(pi) < 0) - goto err; - ret = 1; err: + if (ret < 0 && pi) { + /* + * threads[0] is the leader mirrored by pi->pid. Only + * non-leader threads are inserted independently, and + * next_inserted_thread always points one past the last + * successfully inserted non-leader slot. + */ + for (i = 1; i < next_inserted_thread; i++) + pstree_remove_pid_if_linked(&pi->threads[i]); + if (root_item == pi) + root_item = NULL; + if (linked) + list_del_init(&pi->sibling); + if (pid_inserted) + pstree_remove_pid_if_linked(pi->pid); + pi->pid->state = TASK_UNDEF; + if (threads_allocated) { + xfree(pi->threads); + pi->threads = NULL; + pi->nr_threads = 0; + } + /* + * Restore pstree items come from the shared linear arena. The + * item may no longer be the last allocation after read_pstree_ids(), + * so it cannot be released individually. Restore teardown reclaims + * the arena after this parse failure. + */ + } return ret; } @@ -1262,8 +1341,12 @@ static int new_pid_ns_truncate_pid_hierarchy(pid_t *pid_max) unsigned int ns_level_to_truncate; clone_flags = get_clone_mask(root_item->ids, root_ids); - if (!(clone_flags & CLONE_NEWPID)) + if (!(clone_flags & CLONE_NEWPID) && + !(opts.tfork.active && root_item->pid->ns_level > 1)) return 0; + if (!(clone_flags & CLONE_NEWPID)) + pr_info("pidns: forcing tfork pid hierarchy truncation for root level=%d\n", + root_item->pid->ns_level); if (root_item->pid->ns_level <= 1) { pr_err("only 1 level of pid namespace, but CLONE_NEWPID is set, " @@ -1272,6 +1355,8 @@ static int new_pid_ns_truncate_pid_hierarchy(pid_t *pid_max) } ns_level_to_truncate = root_item->pid->ns_level - 1; + pr_info("pidns: truncating %u outer pid namespace level(s) for new root pid namespace\n", + ns_level_to_truncate); for (node = rb_first(&pid_root_rb[ALL_PID_NS_ID]); node; ) { next = rb_next(node); @@ -1279,9 +1364,15 @@ static int new_pid_ns_truncate_pid_hierarchy(pid_t *pid_max) pid_node = rb_entry(node, struct pid, root_ns_node); rb_erase(node, &pid_root_rb[ALL_PID_NS_ID]); + pr_info("pidns: truncate before uid=%d real=%d local=%d level=%d\n", + pid_node->uid, pid_node->real, pid_node->local, + pid_node->ns_level); pid_node->ns_level -= ns_level_to_truncate; BUG_ON(pid_node->ns_level <= 0); pid_node->real = pid_node->ns[pid_node->ns_level - 1].ns_pid; + pr_info("pidns: truncate after uid=%d real=%d local=%d level=%d\n", + pid_node->uid, pid_node->real, pid_node->local, + pid_node->ns_level); found = __lookup_pid_root(&new_real_rbtree, pid_node->real, &parent, &link); if (found) { diff --git a/criu/criu/seize.c b/criu/criu/seize.c index f5cde74e9..db8d95b92 100644 --- a/criu/criu/seize.c +++ b/criu/criu/seize.c @@ -379,7 +379,7 @@ static int seize_cgroup_tree(char *root_path, enum freezer_state state) */ static int freezer_wait_processes(void) { - int i; + int i, collected = 0; processes_to_wait_pids = xmalloc(sizeof(pid_t) * processes_to_wait); if (processes_to_wait_pids == NULL) @@ -388,23 +388,42 @@ static int freezer_wait_processes(void) for (i = 0; i < processes_to_wait; i++) { int status; pid_t pid; + int waited_ms = 0; /* * Here we are going to skip tasks which are already traced. * Ptraced tasks looks like children for us, so if * a task isn't ptraced yet, waitpid() will return a error. */ - pid = waitpid(-1, &status, 0); - if (pid < 0) { - pr_perror("Unable to wait processes"); - xfree(processes_to_wait_pids); - processes_to_wait_pids = NULL; - return -1; + while (1) { + pid = waitpid(-1, &status, opts.tfork.active ? WNOHANG : 0); + if (pid > 0) + break; + if (opts.tfork.active && pid < 0 && errno == EINTR && waited_ms < 500) { + usleep(10 * 1000); + waited_ms += 10; + continue; + } + if (!opts.tfork.active || (pid < 0 && errno != ECHILD && errno != EINTR)) { + pr_perror("Unable to wait processes"); + xfree(processes_to_wait_pids); + processes_to_wait_pids = NULL; + return -1; + } + if (pid < 0 || waited_ms >= 500) { + pr_warn("tfork: collected %d/%d unexpected freezer processes; continuing\n", + collected, processes_to_wait); + processes_to_wait = collected; + return 0; + } + usleep(10 * 1000); + waited_ms += 10; } pr_warn("Unexpected process %d in the freezer cgroup (status 0x%x)\n", pid, status); - processes_to_wait_pids[i] = pid; + processes_to_wait_pids[collected++] = pid; } + processes_to_wait = collected; return 0; } diff --git a/criu/criu/unittest/mock.c b/criu/criu/unittest/mock.c index b2d507278..b9601fffa 100644 --- a/criu/criu/unittest/mock.c +++ b/criu/criu/unittest/mock.c @@ -97,6 +97,10 @@ int close_service_fd(int type) return 0; } +void invalidate_proc_self_fd(void) +{ +} + void compel_log_init(int log_fn, unsigned int level) { } diff --git a/criu/criu/unittest/unit.c b/criu/criu/unittest/unit.c index 54769e6f2..cf9d79b24 100644 --- a/criu/criu/unittest/unit.c +++ b/criu/criu/unittest/unit.c @@ -3,19 +3,41 @@ #include #include "log.h" +#include "pstree.h" #include "util.h" #include "criu-log.h" int parse_statement(int i, char *line, char **configuration); +atomic_t pid_uid_generator = ATOMIC_INIT(0); + int main(int argc, char *argv[], char *envp[]) { char **configuration; + struct pid first_pid = {}; + struct pid second_pid = {}; + struct pstree_item first_item = { .pid = &first_pid }; + struct pstree_item second_item = { .pid = &second_pid }; int i; configuration = malloc(10 * sizeof(char *)); log_init(NULL); + pid_init_dump(&first_pid, &first_item); + pid_init_dump(&second_pid, &second_item); + assert(first_pid.uid > 0); + assert(second_pid.uid > first_pid.uid); + assert(first_pid.item == &first_item); + assert(first_pid.real == -1); + assert(first_pid.local == -1); + assert(first_pid.state == TASK_UNDEF); + assert(first_pid.stop_signo == -1); + assert(first_pid.ns_level == -1); + assert(first_pid.leaf_ns_id == ALL_PID_NS_ID); + assert(RB_EMPTY_NODE(&first_pid.leaf_ns_node)); + assert(RB_EMPTY_NODE(&first_pid.root_ns_node)); + assert(RB_EMPTY_NODE(&first_pid.uid_node)); + i = parse_statement(0, "", configuration); assert(i == 0); diff --git a/criu/lib/pycriu/images/images.py b/criu/lib/pycriu/images/images.py index 9db506e1e..3d2e02777 100644 --- a/criu/lib/pycriu/images/images.py +++ b/criu/lib/pycriu/images/images.py @@ -43,6 +43,8 @@ import os import array +from google.protobuf.message import DecodeError + from . import magic from . import pb from . import pb2dict @@ -190,6 +192,98 @@ def count(self, f): return entries +class pstree_handler: + """Read both legacy per-task and current file-level PSTREE images.""" + + @staticmethod + def _read_payload(f): + header = f.read(4) + if not header: + return None + if len(header) != 4: + raise ValueError("truncated PSTREE entry header") + + size, = struct.unpack('i', header) + if size < 0: + raise ValueError("negative PSTREE entry size") + + payload = f.read(size) + if len(payload) != size: + raise ValueError("truncated PSTREE entry payload") + return payload + + @staticmethod + def _parse(payload, message_type): + message = message_type() + try: + message.ParseFromString(payload) + except DecodeError: + return None + return message if message.IsInitialized() else None + + def load(self, f, pretty=False, no_payload=False): + # PSTREE has no out-of-band EXTRA data, so no_payload has no effect. + payload = self._read_payload(f) + if payload is None: + return [] + + # Format detection intentionally relies on pstree_entry being proto2 + # with required fields. A file-level payload may parse as that message, + # but it cannot be initialized because its wire fields have other types. + legacy = self._parse(payload, pb.pstree_entry) + if legacy is not None: + entries = [legacy] + while True: + payload = self._read_payload(f) + if payload is None: + break + entry = self._parse(payload, pb.pstree_entry) + if entry is None: + raise ValueError("invalid legacy PSTREE entry") + entries.append(entry) + return [pb2dict.pb2dict(entry, pretty) for entry in entries] + + entry = self._parse(payload, pb.pstree_file_entry) + if entry is None: + raise ValueError("invalid PSTREE entry") + if self._read_payload(f) is not None: + raise ValueError("file-level PSTREE image has multiple entries") + + return [pb2dict.pb2dict(entry, pretty)] + + def loads(self, data, pretty=False): + return self.load(io.BytesIO(data), pretty) + + def dump(self, entries, f): + if not entries: + return + + file_level = ('tree' in entries[0] or + 'ns_max_pids' in entries[0]) + if file_level and len(entries) != 1: + raise ValueError("file-level PSTREE image requires one entry") + + message_type = (pb.pstree_file_entry if file_level + else pb.pstree_entry) + for entry in entries: + message = message_type() + pb2dict.dict2pb(entry, message) + payload = message.SerializeToString() + f.write(struct.pack('i', len(payload))) + f.write(payload) + + def dumps(self, entries): + f = io.BytesIO() + self.dump(entries, f) + return f.getvalue() + + def count(self, f): + entries = 0 + while self._read_payload(f) is not None: + entries += 1 + return entries + + # Special handler for pagemap.img class pagemap_handler: """ @@ -502,7 +596,7 @@ def skip(self, f, pbuff): tcp_stream_extra_handler()), 'STATS': entry_handler(pb.stats_entry), 'PAGEMAP': pagemap_handler(), # Special one - 'PSTREE': entry_handler(pb.pstree_entry), + 'PSTREE': pstree_handler(), 'REG_FILES': entry_handler(pb.reg_file_entry), 'NS_FILES': entry_handler(pb.ns_file_entry), 'EVENTFD_FILE': entry_handler(pb.eventfd_file_entry), diff --git a/criu/test/others/pycriu/Makefile b/criu/test/others/pycriu/Makefile index b6e3b4814..369ac5667 100644 --- a/criu/test/others/pycriu/Makefile +++ b/criu/test/others/pycriu/Makefile @@ -13,7 +13,8 @@ CRIU_SOCKET := $(BUILD_DIR)/$(SOCKET_NAME) STATUS_FIFO := $(BUILD_DIR)/startup.status STATUS_FD := 200 -run: start +run: pstree-compat + $(MAKE) --no-print-directory start cleanup() { $(MAKE) --no-print-directory stop || true; } trap cleanup EXIT INT TERM "$(PYTHON)" test_check.py @@ -21,6 +22,9 @@ run: start "$(PYTHON)" test_check_images_dir.py "$(PYTHON)" test_check_work_dir_fd.py +pstree-compat: + "$(PYTHON)" test_pstree_compat.py + start: mkdir -p "$(BUILD_DIR)" if [ -s "$(PIDFILE)" ] && kill -0 "$$(cat "$(PIDFILE)")" 2>/dev/null; then @@ -60,4 +64,4 @@ clean: fi rm -rf "$(BUILD_DIR)" -.PHONY: start stop clean run \ No newline at end of file +.PHONY: start stop clean run pstree-compat diff --git a/criu/test/others/pycriu/test_pstree_compat.py b/criu/test/others/pycriu/test_pstree_compat.py new file mode 100644 index 000000000..5cda76e13 --- /dev/null +++ b/criu/test/others/pycriu/test_pstree_compat.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 +import io +import os +import struct +import sys + +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +LIB_DIR = os.path.normpath(os.path.join(SCRIPT_DIR, "../../../lib")) +if LIB_DIR not in sys.path: + sys.path.insert(0, LIB_DIR) + +from pycriu.images import images, pb # noqa: E402 + + +def encode(*messages): + output = io.BytesIO() + for message in messages: + payload = message.SerializeToString() + output.write(struct.pack('i', len(payload))) + output.write(payload) + return output.getvalue() + + +def task(message, realpid, localpid, uid): + message.realpid = realpid + message.ppid = 0 + message.pgid = localpid + message.sid = localpid + message.nsid = 7 + message.localpid = localpid + message.uid = uid + + +def main(): + handler = images.handlers['PSTREE'] + + old_first = pb.pstree_entry() + task(old_first, 1001, 1, 11) + old_second = pb.pstree_entry() + task(old_second, 1002, 2, 12) + old_blob = encode(old_first, old_second) + old_entries = handler.loads(old_blob) + assert [entry['realpid'] for entry in old_entries] == [1001, 1002] + assert handler.loads(handler.dumps(old_entries)) == old_entries + assert handler.count(io.BytesIO(old_blob)) == 2 + + current = pb.pstree_file_entry() + ns_max = current.ns_max_pids.add() + ns_max.ns_id = 7 + ns_max.pid_max = 2 + task(current.tree.add(), 1001, 1, 11) + task(current.tree.add(), 1002, 2, 12) + current_blob = encode(current) + current_entries = handler.loads(current_blob) + assert len(current_entries) == 1 + assert [entry['realpid'] for entry in current_entries[0]['tree']] == [1001, 1002] + assert handler.loads(handler.dumps(current_entries)) == current_entries + assert handler.count(io.BytesIO(current_blob)) == 1 + + print("PASS") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/crun/src/libcrun/criu.c b/crun/src/libcrun/criu.c index 8bf068d03..20c7bdd80 100644 --- a/crun/src/libcrun/criu.c +++ b/crun/src/libcrun/criu.c @@ -46,6 +46,8 @@ # define DESCRIPTORS_FILENAME "descriptors.json" # define CRIU_RUNC_CONFIG_FILE "/etc/criu/runc.conf" # define CRIU_CRUN_CONFIG_FILE "/etc/criu/crun.conf" +# define CRIU_LOG_TAIL_LINES 80 +# define CRIU_LOG_LINE_SIZE 1024 # define CRIU_EXT_NETNS "extRootNetNS" # define CRIU_EXT_PIDNS "extRootPidNS" @@ -543,8 +545,11 @@ static void show_criu_log (const char *work_path, const char *log) { cleanup_free char *log_path = NULL; + cleanup_free char *tail = NULL; libcrun_error_t *tmp_err = NULL; - char line[1024]; + char line[CRIU_LOG_LINE_SIZE]; + size_t tail_index = 0; + size_t tail_count = 0; FILE *f; if (UNLIKELY (append_paths (&log_path, tmp_err, work_path, log, NULL)) < 0) @@ -563,12 +568,43 @@ show_criu_log (const char *work_path, const char *log) /* Log with error verbosity as this is the default. */ libcrun_error (0, "--- excerpt from CRIU log `%s`", log_path); + tail = calloc (CRIU_LOG_TAIL_LINES, CRIU_LOG_LINE_SIZE); + if (tail == NULL) + { + fclose (f); + return; + } + while (fgets (line, sizeof (line), f) != NULL) - if (strstr (line, "Error ") != NULL) - { - line[strcspn (line, "\n")] = '\0'; - libcrun_error (0, "%s", line); - } + { + char *slot = tail + tail_index * CRIU_LOG_LINE_SIZE; + strncpy (slot, line, CRIU_LOG_LINE_SIZE - 1); + slot[CRIU_LOG_LINE_SIZE - 1] = '\0'; + tail_index = (tail_index + 1) % CRIU_LOG_TAIL_LINES; + if (tail_count < CRIU_LOG_TAIL_LINES) + tail_count++; + + if (strstr (line, "Error ") != NULL + || strstr (line, "failed") != NULL || strstr (line, "FAILED") != NULL + || strstr (line, "Unable") != NULL || strstr (line, "Can't") != NULL + || strstr (line, "No such") != NULL) + { + line[strcspn (line, "\n")] = '\0'; + libcrun_error (0, "%s", line); + } + } + + if (tail_count > 0) + { + size_t start = (tail_count == CRIU_LOG_TAIL_LINES) ? tail_index : 0; + libcrun_error (0, "--- last %zu CRIU log lines", tail_count); + for (size_t i = 0; i < tail_count; i++) + { + char *entry = tail + ((start + i) % CRIU_LOG_TAIL_LINES) * CRIU_LOG_LINE_SIZE; + entry[strcspn (entry, "\n")] = '\0'; + libcrun_error (0, "%s", entry); + } + } fclose (f); libcrun_error (0, "--- end of excerpt"); @@ -1351,14 +1387,33 @@ libcrun_container_restore_linux_criu (libcrun_container_status_t *status, libcru } # define CRIU_TFORK_LOG_FILE "tfork.log" +# define CRIU_TFORK_RESTORE_LOG_FILE "tfork-restore.log" +# define CRIU_TFORK_MAX_COPY_LOGS 16 + +static void +show_criu_tfork_restore_copy_logs (const char *image_path, int copy_count) +{ + if (copy_count < 0) + copy_count = 0; + if (copy_count > CRIU_TFORK_MAX_COPY_LOGS) + copy_count = CRIU_TFORK_MAX_COPY_LOGS; + + for (int i = 0; i < copy_count; i++) + { + char log[64]; + snprintf (log, sizeof (log), "%s.copy%d", CRIU_TFORK_RESTORE_LOG_FILE, i); + show_criu_log (image_path, log); + } +} static int -read_source_state_pid (const char *path, pid_t *pid_out, libcrun_error_t *err) +read_source_state_pid_cgroup (const char *path, pid_t *pid_out, char **cgroup_path_out, libcrun_error_t *err) { cleanup_free char *buffer = NULL; char err_buffer[256]; yajl_val tree, tmp; const char *pid_path[] = { "pid", NULL }; + const char *cgroup_path[] = { "cgroup-path", NULL }; int ret; ret = read_all_file (path, &buffer, NULL, err); @@ -1377,6 +1432,15 @@ read_source_state_pid (const char *path, pid_t *pid_out, libcrun_error_t *err) } *pid_out = (pid_t) strtoull (YAJL_GET_NUMBER (tmp), NULL, 10); + + tmp = yajl_tree_get (tree, cgroup_path, yajl_t_string); + if (UNLIKELY (tmp == NULL)) + { + yajl_tree_free (tree); + return crun_make_error (err, 0, "`cgroup-path` missing in source state.json `%s`", path); + } + + *cgroup_path_out = xstrdup (YAJL_GET_STRING (tmp)); yajl_tree_free (tree); return 0; } @@ -1439,10 +1503,13 @@ libcrun_container_tfork_linux_criu (libcrun_container_t *container, libcrun_chec { runtime_spec_schema_config_schema *def = container->container_def; cleanup_wrapper struct libcriu_wrapper_s *wrapper = NULL; + cleanup_free char *freezer_path = NULL; cleanup_free char *rootfs_path = NULL; + cleanup_free char *source_cgroup_path = NULL; cleanup_close int image_fd = -1; cleanup_close int work_fd = -1; pid_t source_pid = 0; + int cgroup_mode; int ret; ret = load_wrapper (&wrapper, err); @@ -1461,7 +1528,7 @@ libcrun_container_tfork_linux_criu (libcrun_container_t *container, libcrun_chec return crun_make_error (err, 0, "--tfork-snap-root, --tfork-snap-roots, or --tfork-copy=::--tfork-snap-root=PATH is required"); - if (cr_options->tfork_copies > 1 && cr_options->tfork_snap_roots_n > 0 + if (cr_options->tfork_copies >= 1 && cr_options->tfork_snap_roots_n > 0 && (size_t) cr_options->tfork_copies != cr_options->tfork_snap_roots_n) return crun_make_error (err, 0, "--tfork-copies=%d but --tfork-snap-roots has %zu entries", @@ -1470,7 +1537,7 @@ libcrun_container_tfork_linux_criu (libcrun_container_t *container, libcrun_chec if (UNLIKELY (cr_options->image_path == NULL)) return crun_make_error (err, 0, "--image-path is required"); - ret = read_source_state_pid (cr_options->source_state, &source_pid, err); + ret = read_source_state_pid_cgroup (cr_options->source_state, &source_pid, &source_cgroup_path, err); if (UNLIKELY (ret < 0)) return ret; if (UNLIKELY (source_pid <= 0)) @@ -1514,8 +1581,24 @@ libcrun_container_tfork_linux_criu (libcrun_container_t *container, libcrun_chec libcriu_wrapper->criu_set_pid (source_pid); libcriu_wrapper->criu_set_leave_running (true); + libcriu_wrapper->criu_set_ext_unix_sk (cr_options->ext_unix_sk); libcriu_wrapper->criu_set_file_locks (true); + cgroup_mode = libcrun_get_cgroup_mode (err); + if (UNLIKELY (cgroup_mode < 0)) + return cgroup_mode; + + if (cgroup_mode == CGROUP_MODE_UNIFIED) + ret = append_paths (&freezer_path, err, CGROUP_ROOT, source_cgroup_path, NULL); + else + ret = append_paths (&freezer_path, err, CGROUP_ROOT "/freezer", source_cgroup_path, NULL); + if (UNLIKELY (ret < 0)) + return ret; + + ret = libcriu_wrapper->criu_set_freeze_cgroup (freezer_path); + if (UNLIKELY (ret < 0)) + return crun_make_error (err, -ret, "CRIU: failed setting tfork freezer %d", ret); + if (def->root != NULL && def->root->path != NULL) { ret = append_paths (&rootfs_path, err, container->context ? container->context->bundle : ".", @@ -1564,7 +1647,7 @@ libcrun_container_tfork_linux_criu (libcrun_container_t *container, libcrun_chec } } - if (cr_options->tfork_copies > 1) + if (cr_options->tfork_copies >= 1) libcriu_wrapper->criu_set_tfork_copies (cr_options->tfork_copies); if (cr_options->tfork_memdump_async) @@ -1781,6 +1864,8 @@ libcrun_container_tfork_linux_criu (libcrun_container_t *container, libcrun_chec if (UNLIKELY (ret != 0)) { show_criu_log (cr_options->work_path, CRIU_TFORK_LOG_FILE); + show_criu_log (cr_options->image_path, CRIU_TFORK_RESTORE_LOG_FILE); + show_criu_tfork_restore_copy_logs (cr_options->image_path, cr_options->tfork_copies); return crun_make_error (err, 0, "criu_tfork failed: %d", ret); } @@ -1791,7 +1876,7 @@ libcrun_container_tfork_linux_criu (libcrun_container_t *container, libcrun_chec pid_t clone_pid; const char *pidfile_name = "tfork.pid"; - if (cr_options->tfork_copies > 1) + if (cr_options->tfork_copies >= 1) pidfile_name = "tfork.pid.copy0"; ret = append_paths (&pidfile_path, err, cr_options->image_path, pidfile_name, NULL); @@ -1806,7 +1891,7 @@ libcrun_container_tfork_linux_criu (libcrun_container_t *container, libcrun_chec if (UNLIKELY (clone_pid <= 0)) return crun_make_error (err, 0, "invalid clone PID %d in `%s`", (int) clone_pid, pidfile_path); - if (cr_options->tfork_copies > 1) + if (cr_options->tfork_copies >= 1) { char children_path[64]; cleanup_free char *children_buf = NULL; diff --git a/crun/src/tfork.c b/crun/src/tfork.c index 31db969ef..3a64b2bb9 100644 --- a/crun/src/tfork.c +++ b/crun/src/tfork.c @@ -83,7 +83,7 @@ static struct argp_option options[] { "parent-path", OPTION_PARENT_PATH, "DIR", 0, "previous criu images dir, for incremental memdump chains", 0 }, { "tfork-memdump", OPTION_TFORK_MEMDUMP, 0, 0, "dump pages-*.img to image-path during tfork", 0 }, { "tfork-memdump-async", OPTION_TFORK_MEMDUMP_ASYNC, 0, 0, "async page dump (implies --tfork-memdump)", 0 }, - { "tfork-copies", OPTION_TFORK_COPIES, "N", 0, "produce N parallel clones (default 1)", 0 }, + { "tfork-copies", OPTION_TFORK_COPIES, "N", 0, "produce N clones through the n-copy helper (omitted: legacy direct single-copy)", 0 }, { "track-mem", OPTION_TRACK_MEM, 0, 0, "arm soft-dirty for chained incremental dumps", 0 }, { "manage-cgroups-mode", OPTION_MANAGE_CGROUPS_MODE, "MODE", 0, "cgroups mode: 'soft' (default), 'ignore', 'full' and 'strict'", 0 }, @@ -359,7 +359,14 @@ int crun_command_tfork (struct crun_global_arguments *global_args, int argc, char **argv, libcrun_error_t *err) { cr_options.manage_cgroups_mode = -1; - cr_options.tfork_copies = 1; + cr_options.tfork_copies = 0; + /* + * External Unix stream sockets can make Codex/tmux stacks dumpable, but they + * may hide unsupported socket topology. Keep the default fail-loud and expose + * this as an explicit escape hatch for agent integrations. + */ + if (getenv ("CRUN_TFORK_EXT_UNIX_SK") != NULL) + cr_options.ext_unix_sk = true; cr_options.leave_running = true; return crun_run_create_internal (global_args, argc, argv, container_tfork, get_options, &crun_context, &run_argp, diff --git a/linux-pagecache-cow/build_kernel.sh b/linux-pagecache-cow/build_kernel.sh index c36ce0a59..94bbaeefd 100755 --- a/linux-pagecache-cow/build_kernel.sh +++ b/linux-pagecache-cow/build_kernel.sh @@ -9,6 +9,7 @@ version="7.0.1" # Append a suffix LocalVersion="-pgcachecow" num_cores=$(($(nproc --all) - 2)) +num_cores=$(( num_cores > 1 ? num_cores : 1 )) ## Functions delete_old_kernel_contents () { diff --git a/podman/hack/tfork-transaction-smoke.sh b/podman/hack/tfork-transaction-smoke.sh new file mode 100755 index 000000000..8008fb68f --- /dev/null +++ b/podman/hack/tfork-transaction-smoke.sh @@ -0,0 +1,102 @@ +#!/usr/bin/env bash +set -euo pipefail + +PODMAN=${PODMAN:-podman} +PODMAN_GLOBAL_ARGS=${PODMAN_GLOBAL_ARGS:-} +IMAGE=${IMAGE:-docker.io/library/alpine:3.19} +PREFIX=${PREFIX:-tfork-txn-$RANDOM} +TFORK_RUNTIME_ROOT=${TFORK_RUNTIME_ROOT:-/run/libpod/tfork} + +source_name=${PREFIX}-source +clone_name=${PREFIX}-clone +read -r -a podman_global_args <<<"$PODMAN_GLOBAL_ARGS" + +podman_cmd() { + "$PODMAN" "${podman_global_args[@]}" "$@" +} + +count_dirs() { + local path=$1 + if [[ ! -d $path ]]; then + echo 0 + return + fi + find "$path" -mindepth 1 -maxdepth 1 -type d -print | wc -l +} + +cleanup() { + podman_cmd kill -s KILL "$clone_name" >/dev/null 2>&1 || true + podman_cmd rm -f -t 0 "$clone_name" >/dev/null 2>&1 || true + podman_cmd kill -s KILL "$source_name" >/dev/null 2>&1 || true + podman_cmd rm -f -t 0 "$source_name" >/dev/null 2>&1 || true +} +trap cleanup EXIT + +runtime_before=$(count_dirs "$TFORK_RUNTIME_ROOT") +bundle_root=$(podman_cmd info --format '{{.Store.GraphRoot}}')/tfork-bundles +bundles_before=$(count_dirs "$bundle_root") + +podman_cmd run -d --name "$source_name" \ + --log-driver k8s-file \ + --security-opt seccomp=unconfined \ + --security-opt apparmor=unconfined \ + "$IMAGE" sh -c 'echo source-before-fork >/tmp/sentinel; exec tail -f /dev/null' >/dev/null +cgroups_before=$(find /sys/fs/cgroup -type d -name 'libpod-*' -print 2>/dev/null | wc -l) + +for stage in \ + before_freeze \ + after_freeze \ + after_filesystem \ + before_restore \ + after_restore \ + after_thaw \ + after_register_0 \ + before_commit +do + if env PODMAN_TFORK_FAULT_INJECT="$stage" \ + "$PODMAN" "${podman_global_args[@]}" container clone --live --tfork-overlay-btrfs \ + "$source_name" "$clone_name" >/tmp/tfork-fault.stdout 2>/tmp/tfork-fault.stderr + then + echo "fault stage $stage unexpectedly succeeded" >&2 + exit 1 + fi + if ! grep -Fq "injected tfork fault at $stage" /tmp/tfork-fault.stderr; then + echo "fault stage $stage was not reached; clone failed for another reason" >&2 + sed -n '1,120p' /tmp/tfork-fault.stderr >&2 + exit 1 + fi + podman_cmd exec "$source_name" sh -c \ + 'test "$(cat /tmp/sentinel)" = source-before-fork' + if podman_cmd container exists "$clone_name"; then + echo "fault stage $stage published clone $clone_name" >&2 + exit 1 + fi + if [[ $(count_dirs "$bundle_root") != "$bundles_before" ]]; then + echo "fault stage $stage leaked a graphroot bundle" >&2 + exit 1 + fi + if [[ $(count_dirs "$TFORK_RUNTIME_ROOT") != "$runtime_before" ]]; then + echo "fault stage $stage leaked a runtime publication" >&2 + exit 1 + fi + if [[ $(find /sys/fs/cgroup -type d -name 'libpod-*' -print 2>/dev/null | wc -l) != "$cgroups_before" ]]; then + echo "fault stage $stage leaked a cgroup" >&2 + exit 1 + fi + echo "PASS fault=$stage" +done + +podman_cmd container clone --live --tfork-overlay-btrfs \ + "$source_name" "$clone_name" >/dev/null +[[ $(podman_cmd exec "$clone_name" cat /tmp/sentinel) == source-before-fork ]] +podman_cmd exec "$clone_name" sh -c 'echo child-only >/tmp/sentinel' +[[ $(podman_cmd exec "$source_name" cat /tmp/sentinel) == source-before-fork ]] + +podman_cmd kill -s KILL "$clone_name" >/dev/null +podman_cmd rm -f -t 0 "$clone_name" >/dev/null +if [[ $(count_dirs "$TFORK_RUNTIME_ROOT") != "$runtime_before" ]]; then + echo "successful clone cleanup leaked a runtime publication" >&2 + exit 1 +fi + +echo "PASS normal-fork" diff --git a/podman/libpod/container_config.go b/podman/libpod/container_config.go index dc5e02efe..f13a8a4f8 100644 --- a/podman/libpod/container_config.go +++ b/podman/libpod/container_config.go @@ -472,6 +472,7 @@ type ContainerMiscConfig struct { TforkPersistent string `json:"tforkPersistent,omitempty"` TforkSourceID string `json:"tforkSourceID,omitempty"` TforkParentClone string `json:"tforkParentClone,omitempty"` + TforkInitPIDStartTime uint64 `json:"tforkInitPIDStartTime,omitempty"` TforkDumpdHolderPid int `json:"tforkDumpdHolderPid,omitempty"` TforkDumpdHolderStartTime uint64 `json:"tforkDumpdHolderStartTime,omitempty"` } diff --git a/podman/libpod/container_internal_linux.go b/podman/libpod/container_internal_linux.go index 29159584d..d87256fc7 100644 --- a/podman/libpod/container_internal_linux.go +++ b/podman/libpod/container_internal_linux.go @@ -528,6 +528,22 @@ func (c *Container) CleanupExternalCloneStorage() { return } + // Direct n-copy tfork uses a small runtime-only publication directory for + // attach/log plumbing. It is outside the graphroot bundle and therefore + // was not covered by the storage teardown below. + runtimeBundle := filepath.Clean(c.config.ExternalBundlePath) + runtimeRoot := filepath.Clean("/run/libpod/tfork") + if runtimeBundle != "" && runtimeBundle != "." && + strings.HasPrefix(runtimeBundle, runtimeRoot+string(os.PathSeparator)) { + if err := os.RemoveAll(runtimeBundle); err != nil && !errors.Is(err, fs.ErrNotExist) { + logrus.Debugf("tfork: clone %s rm runtime bundle %s: %v", c.ID(), runtimeBundle, err) + } + parent := filepath.Dir(runtimeBundle) + if entries, err := os.ReadDir(parent); err == nil && len(entries) == 0 { + _ = os.Remove(parent) + } + } + if hpid := c.config.TforkDumpdHolderPid; hpid > 0 { expectedStart := c.config.TforkDumpdHolderStartTime curStart, stErr := ReadProcStartTime(hpid) diff --git a/podman/libpod/oci_conmon_common.go b/podman/libpod/oci_conmon_common.go index 0b521a180..f289e328e 100644 --- a/podman/libpod/oci_conmon_common.go +++ b/podman/libpod/oci_conmon_common.go @@ -828,6 +828,30 @@ func (r *ConmonOCIRuntime) CheckpointContainer(ctr *Container, options Container func (r *ConmonOCIRuntime) CheckConmonRunning(ctr *Container) (bool, error) { if ctr.state.ConmonPID == 0 { + // Direct tfork clones deliberately use a lightweight exit watcher + // instead of conmon. Their init PID, not a missing conmon PID, is the + // authoritative liveness signal. + if ctr.config.ExternalSetup && ctr.state.PID > 0 { + if expected := ctr.config.TforkInitPIDStartTime; expected != 0 { + current, err := ReadProcStartTime(ctr.state.PID) + if errors.Is(err, os.ErrNotExist) || errors.Is(err, unix.ESRCH) { + return false, nil + } + if err != nil { + return false, fmt.Errorf("reading external clone pid %d start time: %w", ctr.state.PID, err) + } + if current != expected { + return false, nil + } + } + if err := unix.Kill(ctr.state.PID, 0); err != nil { + if errors.Is(err, unix.ESRCH) { + return false, nil + } + return false, fmt.Errorf("pinging external clone pid %d: %w", ctr.state.PID, err) + } + return true, nil + } // If the container is running or paused, assume Conmon is // running. We didn't record Conmon PID on some old versions, so // that is likely what's going on... diff --git a/podman/libpod/oci_conmon_tfork_test.go b/podman/libpod/oci_conmon_tfork_test.go new file mode 100644 index 000000000..63d080f60 --- /dev/null +++ b/podman/libpod/oci_conmon_tfork_test.go @@ -0,0 +1,44 @@ +//go:build !remote && linux + +package libpod + +import ( + "os" + "testing" +) + +func TestCheckConmonRunningExternalClonePIDIdentity(t *testing.T) { + pid := os.Getpid() + startTime, err := ReadProcStartTime(pid) + if err != nil { + t.Fatalf("reading test process start time: %v", err) + } + + ctr := &Container{ + config: &ContainerConfig{ + ContainerMiscConfig: ContainerMiscConfig{ + ExternalSetup: true, + TforkInitPIDStartTime: startTime, + }, + }, + state: &ContainerState{PID: pid}, + } + runtime := &ConmonOCIRuntime{} + + alive, err := runtime.CheckConmonRunning(ctr) + if err != nil { + t.Fatalf("checking matching process identity: %v", err) + } + if !alive { + t.Fatal("matching process identity reported dead") + } + + ctr.config.TforkInitPIDStartTime++ + alive, err = runtime.CheckConmonRunning(ctr) + if err != nil { + t.Fatalf("checking recycled process identity: %v", err) + } + if alive { + t.Fatal("mismatched process identity reported alive") + } +} diff --git a/podman/libpod/runtime_ctr.go b/podman/libpod/runtime_ctr.go index fafaac15a..0cb13dbfd 100644 --- a/podman/libpod/runtime_ctr.go +++ b/podman/libpod/runtime_ctr.go @@ -150,6 +150,17 @@ func (r *Runtime) RegisterExternalContainer(ctx context.Context, rSpec *spec.Spe } ctr.valid = true + createdPaths := make([]string, 0, 2) + defer func() { + if retErr == nil { + return + } + for i := len(createdPaths) - 1; i >= 0; i-- { + if err := os.RemoveAll(createdPaths[i]); err != nil { + logrus.Errorf("Removing path for failed external container registration %s: %v", createdPaths[i], err) + } + } + }() if ctr.config.StaticDir == "" { sd := filepath.Join(r.config.Engine.StaticDir, "containers", ctr.ID(), "userdata") @@ -157,11 +168,13 @@ func (r *Runtime) RegisterExternalContainer(ctx context.Context, rSpec *spec.Spe return nil, fmt.Errorf("creating static dir: %w", err) } ctr.config.StaticDir = sd + createdPaths = append(createdPaths, filepath.Dir(sd)) } rd := filepath.Join(r.storageConfig.RunRoot, "containers", ctr.ID(), "userdata") if err := os.MkdirAll(rd, 0o700); err != nil { return nil, fmt.Errorf("creating run dir: %w", err) } + createdPaths = append(createdPaths, filepath.Dir(rd)) ctr.state.RunDir = rd if ctr.config.ConmonPidFile == "" { diff --git a/podman/pkg/domain/infra/abi/container_tfork.go b/podman/pkg/domain/infra/abi/container_tfork.go index 8aa6926d5..ed6617f6c 100644 --- a/podman/pkg/domain/infra/abi/container_tfork.go +++ b/podman/pkg/domain/infra/abi/container_tfork.go @@ -4,11 +4,13 @@ import ( "bufio" "context" "encoding/json" + "errors" "fmt" "net" "os" "os/exec" "path/filepath" + "strconv" "strings" "syscall" "time" @@ -16,17 +18,38 @@ import ( "github.com/containers/podman/v5/libpod" "github.com/containers/podman/v5/libpod/define" - "strconv" "github.com/containers/podman/v5/pkg/domain/entities" "github.com/containers/podman/v5/utils" spec "github.com/opencontainers/runtime-spec/specs-go" "github.com/sirupsen/logrus" "go.podman.io/common/libnetwork/types" + commonconfig "go.podman.io/common/pkg/config" "go.podman.io/storage/pkg/stringid" "golang.org/x/sys/unix" ) +const ( + tforkSourceFreezeTimeout = 10 * time.Second + tforkSourceThawTimeout = 10 * time.Second + tforkCloneReadyTimeout = 60 * time.Second + tforkCgroupPollInterval = 50 * time.Millisecond + tforkClonePollInterval = 200 * time.Millisecond +) + +func tforkCloneReadyTimeoutFromEnv() time.Duration { + value := strings.TrimSpace(os.Getenv("PODMAN_TFORK_CLONE_READY_TIMEOUT_SECS")) + if value == "" { + return tforkCloneReadyTimeout + } + seconds, err := strconv.Atoi(value) + if err != nil || seconds <= 0 { + logrus.Warnf("tfork: ignoring invalid PODMAN_TFORK_CLONE_READY_TIMEOUT_SECS=%q", value) + return tforkCloneReadyTimeout + } + return time.Duration(seconds) * time.Second +} + func (ic *ContainerEngine) containerCloneLive(ctx context.Context, opts entities.ContainerCloneOptions) (rep *entities.ContainerCreateReport, retErr error) { src, err := ic.Libpod.LookupContainer(opts.ID) if err != nil { @@ -40,11 +63,22 @@ func (ic *ContainerEngine) containerCloneLive(ctx context.Context, opts entities if state != define.ContainerStateRunning { return nil, fmt.Errorf("source %q is not running (state=%s); tfork requires a live source", src.ID(), state.String()) } + if manager := src.CgroupManager(); manager != commonconfig.CgroupfsCgroupsManager { + return nil, fmt.Errorf("tfork currently requires the cgroupfs cgroup manager (source uses %q); retry Podman with --cgroup-manager=cgroupfs", manager) + } copies := opts.Copies if copies <= 0 { copies = 1 } + requestedCopies := copies + useSingleCopyConmon := requestedCopies == 1 && os.Getenv("PODMAN_TFORK_SINGLE_COPY_CONMON") == "1" + // PODMAN_TFORK_SINGLE_COPY_DIRECT is a debugging escape hatch that skips + // the n-copy restore helper for single-copy experiments. Production paths + // keep the n-copy helper even for copies=1 so attach/status handling is + // consistent with multi-copy forks. + useSingleCopyDirect := requestedCopies == 1 && os.Getenv("PODMAN_TFORK_SINGLE_COPY_DIRECT") == "1" + useNcopyRestore := copies > 1 || (requestedCopies == 1 && !useSingleCopyConmon && !useSingleCopyDirect) var srcRootfs string if cfg := src.Config(); cfg != nil && cfg.ExternalSetup && cfg.Rootfs != "" { @@ -66,9 +100,27 @@ func (ic *ContainerEngine) containerCloneLive(ctx context.Context, opts entities if err := os.MkdirAll(bundleDir, 0o700); err != nil { return nil, fmt.Errorf("mkdir bundle: %w", err) } + txn := newTforkCloneTransaction(ctx, ic.Libpod, src, bundleDir, copies) + defer func() { + if retErr != nil { + txn.rollback(retErr) + } + }() snapRO := filepath.Join(bundleDir, "snap-ro") + if err := tforkInjectFault("before_freeze"); err != nil { + return nil, err + } + thawSource, err := tforkFreezeSourceCgroup(src, tforkSourceFreezeTimeout) + if err != nil { + return nil, fmt.Errorf("freeze source cgroup before tfork snapshot: %w", err) + } + txn.setSourceRestore(thawSource) + if err := tforkInjectFault("after_freeze"); err != nil { + return nil, err + } + if out, err := exec.Command("sync").CombinedOutput(); err != nil { return nil, fmt.Errorf("sync: %s: %w", out, err) } @@ -157,6 +209,10 @@ func (ic *ContainerEngine) containerCloneLive(ctx context.Context, opts entities } } } + txn.setCloneIDs(cloneIDs) + if err := tforkInjectFault("after_filesystem"); err != nil { + return nil, err + } if recursive && parentUpperFrozen != "" { if err := os.RemoveAll(parentUpperFrozen); err != nil { @@ -260,6 +316,7 @@ func (ic *ContainerEngine) containerCloneLive(ctx context.Context, opts entities if err := os.MkdirAll(imgDir, 0o700); err != nil { return nil, fmt.Errorf("mkdir img: %w", err) } + txn.setImageDir(imgDir) cloneCgroupPaths := make([]string, copies) if cfg := src.Config(); cfg != nil { @@ -272,6 +329,7 @@ func (ic *ContainerEngine) containerCloneLive(ctx context.Context, opts entities cloneCgroupPaths[i] = cgRel } } + txn.setCgroupPaths(cloneCgroupPaths) srcStatePath := fmt.Sprintf("/run/crun/%s/status", src.ID()) crunArgs := []string{ @@ -280,7 +338,7 @@ func (ic *ContainerEngine) containerCloneLive(ctx context.Context, opts entities "--source-state", srcStatePath, "--image-path", imgDir, } - if copies == 1 { + if !useNcopyRestore { crunArgs = append(crunArgs, "--tfork-snap-root", cloneRootfsList[0]) crunArgs = append(crunArgs, "--tfork-snap-mount", "/") if cloneCgroupPaths[0] != "" { @@ -384,6 +442,9 @@ func (ic *ContainerEngine) containerCloneLive(ctx context.Context, opts entities crunArgs = append(crunArgs, "--parent-path", parentImgDir) logrus.Infof("tfork: --with-previous chains off clone %s (imgDir=%s)", parentCloneID, parentImgDir) } + if err := tforkInjectFault("before_restore"); err != nil { + return nil, err + } cloneLogPaths := make([]string, copies) cloneConmonPids := make([]int, copies) @@ -400,7 +461,9 @@ func (ic *ContainerEngine) containerCloneLive(ctx context.Context, opts entities } }() hasTTY := len(ttySrcFds) > 0 - useConmon := copies == 1 + // Keep the legacy single-copy conmon bootstrap opt-in because it can fail + // before crun starts and report only `conmon reported pid=-1`. + useConmon := copies == 1 && useSingleCopyConmon if useConmon { conmonInheritFds := inheritFds @@ -433,6 +496,7 @@ func (ic *ContainerEngine) containerCloneLive(ctx context.Context, opts entities } cloneLogPaths[0] = logPath cloneConmonPids[0] = conmonPid + txn.trackPID(conmonPid, "conmon") logrus.Infof("tfork: clone %s up via conmon pid=%d; log=%s", cloneIDs[0], conmonPid, logPath) } else { directArgs := append([]string{}, crunArgs...) @@ -443,8 +507,8 @@ func (ic *ContainerEngine) containerCloneLive(ctx context.Context, opts entities var perCopyExtraFiles []*os.File var perCopyReadEnds []*os.File var perCopyArgs []string - skipTtySrcFds := copies > 1 && hasTTY - if copies > 1 { + skipTtySrcFds := useNcopyRestore && hasTTY + if useNcopyRestore { ifdsForPerCopy, stdioKeys := splitStdioInheritFds(inheritFds) inheritFds = ifdsForPerCopy extraFDBase := 3 @@ -497,6 +561,8 @@ func (ic *ContainerEngine) containerCloneLive(ctx context.Context, opts entities logF.Close() return nil, fmt.Errorf("start crun tfork: %w", err) } + txn.trackPID(crunCmd.Process.Pid, "crun-tfork") + crunDone := make(chan error, 1) var crunAborted bool defer func() { if !crunAborted && retErr != nil { @@ -507,7 +573,7 @@ func (ic *ContainerEngine) containerCloneLive(ctx context.Context, opts entities _ = f.Close() } go func() { - _ = crunCmd.Wait() + crunDone <- crunCmd.Wait() logF.Close() }() perCopyAttachSocks := make([]*os.File, copies) @@ -516,6 +582,7 @@ func (ic *ContainerEngine) containerCloneLive(ctx context.Context, opts entities shortBatch = shortBatch[:12] } runtimeAttachBase := filepath.Join("/run/libpod/tfork", shortBatch) + txn.setRuntimeBatchDir(runtimeAttachBase) for i := 0; i < copies; i++ { perCopyBundle := filepath.Join(runtimeAttachBase, fmt.Sprintf("%d", i)) if err := os.MkdirAll(perCopyBundle, 0o700); err != nil { @@ -538,8 +605,11 @@ func (ic *ContainerEngine) containerCloneLive(ctx context.Context, opts entities } else { cloneLogPath = filepath.Join(bundleDir, fmt.Sprintf("clone.%d.log", i)) } - if err := spawnTforkStdioHelper(readEnd, perCopyAttachSocks[i], cloneLogPath, hasTTY); err != nil { + helperPID, err := spawnTforkStdioHelper(readEnd, perCopyAttachSocks[i], cloneLogPath, hasTTY) + if err != nil { logrus.Warnf("tfork: per-copy %d stdio-helper spawn: %v", i, err) + } else { + txn.trackPID(helperPID, fmt.Sprintf("stdio-helper-%d", i)) } if perCopyAttachSocks[i] != nil { _ = perCopyAttachSocks[i].Close() @@ -549,17 +619,27 @@ func (ic *ContainerEngine) containerCloneLive(ctx context.Context, opts entities } pidFileFor := func(i int) string { - if copies == 1 { + if !useNcopyRestore { return filepath.Join(imgDir, "tfork.pid") } return filepath.Join(imgDir, fmt.Sprintf("tfork.pid.copy%d", i)) } statePath := fmt.Sprintf("/run/crun/%s/status", cloneIDs[0]) - needState := copies == 1 - deadline := time.Now().Add(60 * time.Second) + needState := !useNcopyRestore + cloneReadyTimeout := tforkCloneReadyTimeoutFromEnv() + deadline := time.Now().Add(cloneReadyTimeout) readyCopies := 0 stateReady := !needState + crunExited := false + var crunErr error for readyCopies < copies || !stateReady { + if !crunExited { + select { + case crunErr = <-crunDone: + crunExited = true + default: + } + } readyCopies = 0 for i := 0; i < copies; i++ { if _, err := os.Stat(pidFileFor(i)); err == nil { @@ -574,7 +654,7 @@ func (ic *ContainerEngine) containerCloneLive(ctx context.Context, opts entities if readyCopies >= copies && stateReady { break } - if crunCmd.ProcessState != nil && crunCmd.ProcessState.Exited() && readyCopies < copies { + if crunExited && readyCopies < copies { tforkAbortCrunCmd(crunCmd, src, bundleDir, copies) crunAborted = true return nil, fmt.Errorf("crun tfork exited before %d clones came up (got %d); see %s", @@ -586,24 +666,69 @@ func (ic *ContainerEngine) containerCloneLive(ctx context.Context, opts entities return nil, fmt.Errorf("timeout waiting for %d tfork.pid* files in %s (got %d, stateReady=%v); see %s", copies, imgDir, readyCopies, stateReady, logPath) } - time.Sleep(200 * time.Millisecond) + time.Sleep(tforkClonePollInterval) + } + if !crunExited { + select { + case crunErr = <-crunDone: + crunExited = true + case <-time.After(cloneReadyTimeout): + tforkAbortCrunCmd(crunCmd, src, bundleDir, copies) + crunAborted = true + return nil, fmt.Errorf("timeout waiting %s for crun tfork to finish after %d clones came up; see %s", + cloneReadyTimeout, copies, logPath) + } + } + if crunErr != nil { + tforkAbortCrunCmd(crunCmd, src, bundleDir, copies) + crunAborted = true + return nil, fmt.Errorf("crun tfork failed after %d clones came up: %w; see %s", + copies, crunErr, logPath) } logrus.Infof("tfork: batch %s up (N=%d); crun-tfork.log at %s", batchID, copies, logPath) } + if err := tforkInjectFault("after_restore"); err != nil { + return nil, err + } + + if err := txn.restoreSourceOnce(); err != nil { + logrus.Warnf("tfork: clones are up, but thawing source cgroup after restore failed: %v", err) + return nil, fmt.Errorf("thaw source cgroup after tfork restore: %w", err) + } + if err := tforkInjectFault("after_thaw"); err != nil { + return nil, err + } + srcPID, err := src.PID() + if err != nil { + return nil, fmt.Errorf("read source PID after restore: %w", err) + } + if err := tforkPIDRunning(srcPID); err != nil { + return nil, fmt.Errorf("source is not usable after tfork restore: %w", err) + } srcCfg := src.Config() if srcCfg == nil { return nil, fmt.Errorf("source %q: could not read libpod config", src.ID()) } + visibleCloneIDs := make([]string, 0, requestedCopies) for i, cloneID := range cloneIDs { - clonePID, err := readTforkClonePID(cloneID, imgDir, i, copies) + clonePID, err := readTforkClonePID(cloneID, imgDir, i, copies, useNcopyRestore) if err != nil { return nil, fmt.Errorf("read clone %d PID: %w", i, err) } + txn.trackPID(clonePID, fmt.Sprintf("clone-init-%d", i)) + if err := tforkPIDRunning(clonePID); err != nil { + return nil, fmt.Errorf("clone %d is not running before publication: %w", i, err) + } + clonePIDStartTime, err := libpod.ReadProcStartTime(clonePID) + if err != nil { + return nil, fmt.Errorf("read clone %d PID start time: %w", i, err) + } cloneCfg, err := buildCloneContainerConfig(srcCfg, cloneID, cloneNames[i], cloneRootfsList[i], cloneSpecs[i]) if err != nil { return nil, fmt.Errorf("build clone %d config: %w", i, err) } + cloneCfg.TforkInitPIDStartTime = clonePIDStartTime if len(srcCfg.PortMappings) > 0 { clonePorts, err := buildClonePortMappings(srcCfg.PortMappings) if err != nil { @@ -633,16 +758,17 @@ func (ic *ContainerEngine) containerCloneLive(ctx context.Context, opts entities if err != nil { return nil, fmt.Errorf("register clone %d (%s) in libpod state: %w", i, cloneID, err) } + txn.addRegistered(ctr) if opts.TforkOverlayBtrfs { tforkFreezeUpperForRollback(bundleDir, i, copies) } if cloneConmonPids[i] > 0 { if err := ctr.SetConmonPID(cloneConmonPids[i]); err != nil { - logrus.Warnf("tfork: clone %s SetConmonPID(%d): %v", cloneID, cloneConmonPids[i], err) + return nil, fmt.Errorf("clone %s SetConmonPID(%d): %w", cloneID, cloneConmonPids[i], err) } } if err := ic.Libpod.SetupExternalCloneNetwork(src, ctr, clonePID); err != nil { - logrus.Warnf("tfork: clone %s network setup failed (clone has no network): %v", cloneID, err) + return nil, fmt.Errorf("clone %s network setup: %w", cloneID, err) } if cmd := exec.Command("nsenter", "-t", strconv.Itoa(clonePID), "-n", "--", @@ -653,17 +779,204 @@ func (ic *ContainerEngine) containerCloneLive(ctx context.Context, opts entities } } if err := ctr.MoveExternalCloneToOwnCgroup(clonePID); err != nil { - logrus.Warnf("tfork: clone %s F8 cgroup migration failed: %v (clone shares source's cgroup)", cloneID, err) + return nil, fmt.Errorf("clone %s cgroup migration: %w", cloneID, err) } if cloneConmonPids[i] == 0 { - if err := spawnTforkExitWatcher(ctx, ic.Libpod, ctr, clonePID); err != nil { - logrus.Warnf("tfork: clone %s exit-watcher spawn: %v (podman will fall back to F4.2 /proc liveness check)", cloneID, err) + watcherPID, err := spawnTforkExitWatcher(ctx, ic.Libpod, ctr, clonePID) + if err != nil { + return nil, fmt.Errorf("tfork: clone %s exit-watcher spawn: %w", cloneID, err) } + txn.trackPID(watcherPID, fmt.Sprintf("exit-watcher-%d", i)) + } + if err := tforkPIDRunning(clonePID); err != nil { + return nil, fmt.Errorf("clone %d died during publication: %w", i, err) + } + if err := tforkInjectFault(fmt.Sprintf("after_register_%d", i)); err != nil { + return nil, err } logrus.Infof("tfork: clone %s (%s) registered in libpod state, pid=%d", cloneID, cloneNames[i], clonePID) + visibleCloneIDs = append(visibleCloneIDs, cloneID) + } + + if err := tforkInjectFault("before_commit"); err != nil { + return nil, err + } + txn.commit() + return &entities.ContainerCreateReport{Id: strings.Join(visibleCloneIDs, "\n")}, nil +} + +type tforkCgroupFreezer struct { + root string + statePath string + eventsPath string + version string +} + +func tforkFreezeSourceCgroup(src *libpod.Container, timeout time.Duration) (func() error, error) { + if src == nil { + return nil, fmt.Errorf("source container is nil") + } + cgPath, err := src.CgroupPath() + if err != nil { + return nil, fmt.Errorf("read source cgroup path: %w", err) + } + cgPath = strings.TrimPrefix(filepath.Clean(cgPath), string(os.PathSeparator)) + if cgPath == "" || cgPath == "." { + return nil, fmt.Errorf("source cgroup path is empty") + } + freezer, err := tforkSourceCgroupFreezer(cgPath) + if err != nil { + return nil, err + } + + originalFrozen, err := freezer.frozen() + if err != nil { + return nil, err + } + if err := freezer.freeze(); err != nil { + return nil, err } + if err := freezer.waitFrozen(true, timeout); err != nil { + if !originalFrozen { + _ = freezer.thaw() + } + return nil, err + } + logrus.Infof("tfork: froze source cgroup for snapshot consistency: %s (%s)", freezer.root, freezer.version) + + thaw := func() error { + if originalFrozen { + return freezer.freeze() + } + if err := freezer.thaw(); err != nil { + return err + } + if err := freezer.waitFrozen(false, tforkSourceThawTimeout); err != nil { + return err + } + logrus.Infof("tfork: thawed source cgroup after clone restore: %s (%s)", freezer.root, freezer.version) + return nil + } + return thaw, nil +} - return &entities.ContainerCreateReport{Id: strings.Join(cloneIDs, "\n")}, nil +func tforkSourceCgroupFreezer(cgPath string) (*tforkCgroupFreezer, error) { + v2Root := filepath.Join("/sys/fs/cgroup", cgPath) + v2FreezePath := filepath.Join(v2Root, "cgroup.freeze") + v2EventsPath := filepath.Join(v2Root, "cgroup.events") + if _, err := os.Stat(v2FreezePath); err == nil { + return &tforkCgroupFreezer{ + root: v2Root, + statePath: v2FreezePath, + eventsPath: v2EventsPath, + version: "cgroup v2", + }, nil + } else if !os.IsNotExist(err) { + return nil, fmt.Errorf("stat %s: %w", v2FreezePath, err) + } + + v1Root := filepath.Join("/sys/fs/cgroup/freezer", cgPath) + v1StatePath := filepath.Join(v1Root, "freezer.state") + if _, err := os.Stat(v1StatePath); err == nil { + return &tforkCgroupFreezer{ + root: v1Root, + statePath: v1StatePath, + version: "cgroup v1 freezer", + }, nil + } else if !os.IsNotExist(err) { + return nil, fmt.Errorf("stat %s: %w", v1StatePath, err) + } + + return nil, fmt.Errorf("source cgroup freezer not found for %q; expected cgroup v2 %s or cgroup v1 %s", + cgPath, v2FreezePath, v1StatePath) +} + +func (f *tforkCgroupFreezer) frozen() (bool, error) { + data, err := os.ReadFile(f.statePath) + if err != nil { + return false, fmt.Errorf("read %s: %w", f.statePath, err) + } + frozen, ok := f.parseFrozen(data) + if !ok { + return false, fmt.Errorf("could not parse frozen state from %s", f.statePath) + } + return frozen, nil +} + +func (f *tforkCgroupFreezer) freeze() error { + value := []byte("1") + if f.version == "cgroup v1 freezer" { + value = []byte("FROZEN") + } + if err := os.WriteFile(f.statePath, value, 0o644); err != nil { + return fmt.Errorf("write %s=%s: %w", f.statePath, value, err) + } + return nil +} + +func (f *tforkCgroupFreezer) thaw() error { + value := []byte("0") + if f.version == "cgroup v1 freezer" { + value = []byte("THAWED") + } + if err := os.WriteFile(f.statePath, value, 0o644); err != nil { + return fmt.Errorf("write %s=%s: %w", f.statePath, value, err) + } + return nil +} + +func (f *tforkCgroupFreezer) waitFrozen(wantFrozen bool, timeout time.Duration) error { + deadline := time.Now().Add(timeout) + for { + path := f.statePath + if f.eventsPath != "" { + path = f.eventsPath + } + data, err := os.ReadFile(path) + if err != nil { + return fmt.Errorf("read %s: %w", path, err) + } + frozen, ok := f.parseFrozen(data) + if ok && frozen == wantFrozen { + return nil + } + if time.Now().After(deadline) { + return fmt.Errorf("timeout waiting %s for %s to report frozen=%t", timeout, path, wantFrozen) + } + time.Sleep(tforkCgroupPollInterval) + } +} + +func (f *tforkCgroupFreezer) parseFrozen(data []byte) (bool, bool) { + if f.version == "cgroup v1 freezer" { + switch strings.TrimSpace(string(data)) { + case "FROZEN": + return true, true + case "THAWED": + return false, true + } + return false, false + } + + switch strings.TrimSpace(string(data)) { + case "0": + return false, true + case "1": + return true, true + } + + for _, line := range strings.Split(string(data), "\n") { + fields := strings.Fields(line) + if len(fields) == 2 && fields[0] == "frozen" { + switch fields[1] { + case "0": + return false, true + case "1": + return true, true + } + } + } + return false, false } func spawnTforkDumpdHolder() (int, uint64, error) { @@ -689,23 +1002,40 @@ func spawnTforkDumpdHolder() (int, uint64, error) { func tforkAbortCrunCmd(crunCmd *exec.Cmd, src *libpod.Container, bundleDir string, copies int) { if crunCmd != nil && crunCmd.Process != nil { + protected := make(map[int]bool) + if src != nil { + if srcPID, err := src.PID(); err == nil && srcPID > 0 { + protected[srcPID] = true + for _, pid := range tforkCollectDescendants(srcPID) { + protected[pid] = true + } + } + } toKill := tforkCollectDescendants(crunCmd.Process.Pid) - toKill = append(toKill, crunCmd.Process.Pid) - for _, pid := range toKill { - _ = syscall.Kill(pid, syscall.SIGKILL) + // Give CRIU's service and restore helpers a chance to unwind ptrace, + // parasite, namespace, and cgyard state before killing their parent. + // Never signal a source-tree PID even if transient reparenting makes it + // appear below the runtime. + for i := len(toKill) - 1; i >= 0; i-- { + if !protected[toKill[i]] { + _ = syscall.Kill(toKill[i], syscall.SIGTERM) + } } - done := make(chan struct{}) - go func() { - _, _ = crunCmd.Process.Wait() - close(done) - }() - select { - case <-done: - case <-time.After(3 * time.Second): - logrus.Warnf("tfork: crun-tfork didn't exit in 3s after SIGKILL — source may still be ptraced") + if !tforkWaitPIDGone(crunCmd.Process.Pid, time.Second) { + for i := len(toKill) - 1; i >= 0; i-- { + if !protected[toKill[i]] { + _ = syscall.Kill(toKill[i], syscall.SIGKILL) + } + } + _ = syscall.Kill(crunCmd.Process.Pid, syscall.SIGKILL) + if !tforkWaitPIDGone(crunCmd.Process.Pid, 3*time.Second) { + logrus.Warnf("tfork: crun-tfork didn't exit after TERM/KILL escalation — source may still be ptraced") + } } for _, pid := range tforkCollectDescendants(crunCmd.Process.Pid) { - _ = syscall.Kill(pid, syscall.SIGKILL) + if !protected[pid] { + _ = syscall.Kill(pid, syscall.SIGKILL) + } } } if src != nil { @@ -725,6 +1055,19 @@ func tforkAbortCrunCmd(crunCmd *exec.Cmd, src *libpod.Container, bundleDir strin } } +func tforkWaitPIDGone(pid int, timeout time.Duration) bool { + deadline := time.Now().Add(timeout) + for { + if err := syscall.Kill(pid, 0); errors.Is(err, syscall.ESRCH) { + return true + } + if time.Now().After(deadline) { + return false + } + time.Sleep(20 * time.Millisecond) + } +} + func tforkCollectDescendants(root int) []int { entries, err := os.ReadDir("/proc") if err != nil { @@ -863,9 +1206,9 @@ while True: return nil } -func spawnTforkStdioHelper(readEnd *os.File, attachSock *os.File, logPath string, tty bool) error { +func spawnTforkStdioHelper(readEnd *os.File, attachSock *os.File, logPath string, tty bool) (int, error) { if err := os.MkdirAll(filepath.Dir(logPath), 0o755); err != nil { - return fmt.Errorf("mkdir %s: %w", filepath.Dir(logPath), err) + return 0, fmt.Errorf("mkdir %s: %w", filepath.Dir(logPath), err) } pyCode := `import os, sys, asyncio, datetime, socket, io, traceback LOG_PATH = os.environ["LOG_PATH"] @@ -985,13 +1328,15 @@ except Exception: env = append(env, fmt.Sprintf("ATTACH_FD=%d", attachFD)) cmd.Env = env if err := cmd.Start(); err != nil { - return fmt.Errorf("start stdio-helper: %w", err) + return 0, fmt.Errorf("start stdio-helper: %w", err) } + pid := 0 if cmd.Process != nil { + pid = cmd.Process.Pid _ = cmd.Process.Release() } logrus.Debugf("tfork: spawned stdio-helper for read-fd → %s (attach=%v, tty=%v)", logPath, attachSock != nil, tty) - return nil + return pid, nil } func boolToInt(b bool) int { @@ -1025,10 +1370,10 @@ func allocPerCopyAttachSocket(path string) (*os.File, error) { return f, nil } -func spawnTforkExitWatcher(ctx context.Context, rt *libpod.Runtime, ctr *libpod.Container, clonePID int) error { +func spawnTforkExitWatcher(ctx context.Context, rt *libpod.Runtime, ctr *libpod.Container, clonePID int) (int, error) { exitDir := "/run/libpod/exits" if err := os.MkdirAll(exitDir, 0o755); err != nil { - return fmt.Errorf("mkdir %s: %w", exitDir, err) + return 0, fmt.Errorf("mkdir %s: %w", exitDir, err) } exitFile := filepath.Join(exitDir, ctr.ID()) script := fmt.Sprintf(`while [ -d /proc/%d ]; do sleep 0.2; done; tmp=%s.tmp; printf 137 > "$tmp" && mv "$tmp" %s`, clonePID, exitFile, exitFile) @@ -1037,16 +1382,18 @@ func spawnTforkExitWatcher(ctx context.Context, rt *libpod.Runtime, ctr *libpod. cmd.Stdout = nil cmd.Stderr = nil if err := cmd.Start(); err != nil { - return fmt.Errorf("start exit-watcher: %w", err) + return 0, fmt.Errorf("start exit-watcher: %w", err) } + pid := 0 if cmd.Process != nil { + pid = cmd.Process.Pid _ = cmd.Process.Release() } logrus.Debugf("tfork: spawned exit-watcher for clone %s (PID %d) → %s", ctr.ID(), clonePID, exitFile) - return nil + return pid, nil } -func readTforkClonePID(cloneID string, imgDir string, copyIdx, copies int) (int, error) { +func readTforkClonePID(cloneID string, imgDir string, copyIdx, copies int, useNcopyRestore bool) (int, error) { statePath := fmt.Sprintf("/run/crun/%s/status", cloneID) if data, err := os.ReadFile(statePath); err == nil { var st struct { @@ -1057,7 +1404,7 @@ func readTforkClonePID(cloneID string, imgDir string, copyIdx, copies int) (int, } } var pidFile string - if copies == 1 { + if !useNcopyRestore { pidFile = filepath.Join(imgDir, "tfork.pid") } else { pidFile = filepath.Join(imgDir, fmt.Sprintf("tfork.pid.copy%d", copyIdx)) @@ -1070,7 +1417,7 @@ func readTforkClonePID(cloneID string, imgDir string, copyIdx, copies int) (int, if err != nil { return 0, err } - if copies == 1 { + if !useNcopyRestore { return rcPID, nil } initPID, err := readFirstChildPID(rcPID) diff --git a/podman/pkg/domain/infra/abi/container_tfork_transaction.go b/podman/pkg/domain/infra/abi/container_tfork_transaction.go new file mode 100644 index 000000000..c8314ec8c --- /dev/null +++ b/podman/pkg/domain/infra/abi/container_tfork_transaction.go @@ -0,0 +1,291 @@ +//go:build !remote + +package abi + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "strconv" + "strings" + "syscall" + "time" + + "github.com/containers/podman/v5/libpod" + "github.com/sirupsen/logrus" + "golang.org/x/sys/unix" +) + +const tforkRollbackWait = 3 * time.Second + +type tforkTrackedPID struct { + pid int + startTime uint64 + role string +} + +// tforkCloneTransaction owns every host-side object created before a clone is +// published. Ownership is transferred to libpod only by commit. Until then, +// every return path converges on rollback. +type tforkCloneTransaction struct { + ctx context.Context + rt *libpod.Runtime + src *libpod.Container + + bundleDir string + runtimeBatchDir string + imgDir string + copies int + cloneIDs []string + cgroupPaths []string + + restoreSource func() error + sourceRestored bool + registered []*libpod.Container + pids []tforkTrackedPID + committed bool + rolledBack bool +} + +func newTforkCloneTransaction(ctx context.Context, rt *libpod.Runtime, src *libpod.Container, bundleDir string, copies int) *tforkCloneTransaction { + return &tforkCloneTransaction{ + ctx: ctx, + rt: rt, + src: src, + bundleDir: bundleDir, + copies: copies, + } +} + +func (t *tforkCloneTransaction) setCloneIDs(ids []string) { + t.cloneIDs = append([]string(nil), ids...) +} + +func (t *tforkCloneTransaction) setCgroupPaths(paths []string) { + t.cgroupPaths = append([]string(nil), paths...) +} + +func (t *tforkCloneTransaction) setRuntimeBatchDir(path string) { + t.runtimeBatchDir = path +} + +func (t *tforkCloneTransaction) setImageDir(path string) { + t.imgDir = path +} + +func (t *tforkCloneTransaction) setSourceRestore(restore func() error) { + t.restoreSource = restore +} + +func (t *tforkCloneTransaction) restoreSourceOnce() error { + if t.sourceRestored || t.restoreSource == nil { + return nil + } + if err := t.restoreSource(); err != nil { + return err + } + t.sourceRestored = true + return nil +} + +func (t *tforkCloneTransaction) trackPID(pid int, role string) { + if pid <= 0 { + return + } + for _, tracked := range t.pids { + if tracked.pid == pid { + return + } + } + startTime, err := libpod.ReadProcStartTime(pid) + if err != nil { + logrus.Debugf("tfork: transaction cannot record %s pid=%d start time: %v", role, pid, err) + return + } + t.pids = append(t.pids, tforkTrackedPID{pid: pid, startTime: startTime, role: role}) +} + +func (t *tforkCloneTransaction) addRegistered(ctr *libpod.Container) { + if ctr != nil { + t.registered = append(t.registered, ctr) + } +} + +func (t *tforkCloneTransaction) commit() { + t.committed = true +} + +func (t *tforkCloneTransaction) rollback(cause error) { + if t == nil || t.committed || t.rolledBack { + return + } + t.rolledBack = true + logrus.Warnf("tfork: rolling back unpublished clone transaction: %v", cause) + + // A registered external clone knows how to tear down its network, state, + // cgroup, and per-copy storage. Remove in reverse publication order. + zero := uint(0) + for i := len(t.registered) - 1; i >= 0; i-- { + ctr := t.registered[i] + if err := t.rt.RemoveContainer(context.WithoutCancel(t.ctx), ctr, true, false, &zero); err != nil { + logrus.Warnf("tfork: rollback remove registered clone %s: %v", ctr.ID(), err) + } + } + + t.trackRestorePIDs() + t.killCloneCgroups() + for i := len(t.pids) - 1; i >= 0; i-- { + t.killTrackedPID(t.pids[i]) + } + + if err := t.restoreSourceOnce(); err != nil { + logrus.Warnf("tfork: rollback restore source cgroup: %v", err) + } + + t.removeCloneCgroups() + for _, id := range t.cloneIDs { + _ = os.Remove(filepath.Join("/run/libpod/exits", id)) + _ = os.RemoveAll(filepath.Join("/run/crun", id)) + } + if t.runtimeBatchDir != "" { + if err := os.RemoveAll(t.runtimeBatchDir); err != nil { + logrus.Warnf("tfork: rollback remove runtime publication %s: %v", t.runtimeBatchDir, err) + } + tforkRemoveEmptyParent(filepath.Dir(t.runtimeBatchDir)) + } + if t.bundleDir != "" { + tforkBestEffortBundleReap(t.bundleDir, t.copies) + } +} + +func (t *tforkCloneTransaction) trackRestorePIDs() { + if t.imgDir == "" { + return + } + for i := 0; i < t.copies; i++ { + for _, path := range []string{ + filepath.Join(t.imgDir, "tfork.pid"), + filepath.Join(t.imgDir, fmt.Sprintf("tfork.pid.copy%d", i)), + } { + data, err := os.ReadFile(path) + if err != nil { + continue + } + pid, err := parsePIDBytes(data, path) + if err == nil { + t.trackPID(pid, "restore-child") + if child, err := readFirstChildPID(pid); err == nil { + t.trackPID(child, "clone-init") + } + } + } + } +} + +func (t *tforkCloneTransaction) killCloneCgroups() { + for _, rel := range t.cgroupPaths { + root := tforkCgroupFSPath(rel) + if root == "" { + continue + } + killPath := filepath.Join(root, "cgroup.kill") + if err := os.WriteFile(killPath, []byte("1"), 0o644); err != nil && !errors.Is(err, os.ErrNotExist) { + logrus.Debugf("tfork: rollback write %s: %v", killPath, err) + } + } +} + +func (t *tforkCloneTransaction) removeCloneCgroups() { + deadline := time.Now().Add(tforkRollbackWait) + for _, rel := range t.cgroupPaths { + root := tforkCgroupFSPath(rel) + if root == "" { + continue + } + for { + err := os.Remove(root) + if err == nil || errors.Is(err, os.ErrNotExist) { + break + } + if time.Now().After(deadline) { + logrus.Warnf("tfork: rollback cgroup remains at %s: %v", root, err) + break + } + time.Sleep(25 * time.Millisecond) + } + } +} + +func tforkCgroupFSPath(rel string) string { + clean := strings.TrimPrefix(filepath.Clean(rel), string(os.PathSeparator)) + if clean == "" || clean == "." || strings.HasPrefix(clean, "..") { + return "" + } + return filepath.Join("/sys/fs/cgroup", clean) +} + +func (t *tforkCloneTransaction) killTrackedPID(tracked tforkTrackedPID) { + current, err := libpod.ReadProcStartTime(tracked.pid) + if errors.Is(err, os.ErrNotExist) || errors.Is(err, unix.ESRCH) { + return + } + if err != nil { + logrus.Warnf("tfork: rollback cannot verify %s pid=%d: %v; refusing unsafe kill", tracked.role, tracked.pid, err) + return + } + if current != tracked.startTime { + logrus.Warnf("tfork: rollback %s pid=%d was recycled; refusing kill", tracked.role, tracked.pid) + return + } + descendants := tforkCollectDescendants(tracked.pid) + for i := len(descendants) - 1; i >= 0; i-- { + _ = syscall.Kill(descendants[i], syscall.SIGKILL) + } + _ = syscall.Kill(tracked.pid, syscall.SIGKILL) +} + +func tforkRemoveEmptyParent(path string) { + if path == "" { + return + } + entries, err := os.ReadDir(path) + if err == nil && len(entries) == 0 { + _ = os.Remove(path) + } +} + +func tforkPIDRunning(pid int) error { + if pid <= 0 { + return fmt.Errorf("invalid pid %d", pid) + } + data, err := os.ReadFile(filepath.Join("/proc", strconv.Itoa(pid), "stat")) + if err != nil { + return err + } + rp := strings.LastIndexByte(string(data), ')') + if rp < 0 || rp+2 >= len(data) { + return fmt.Errorf("malformed /proc/%d/stat", pid) + } + if data[rp+2] == 'Z' { + return fmt.Errorf("pid %d is a zombie", pid) + } + return nil +} + +func tforkInjectFault(stage string) error { + // PODMAN_TFORK_FAULT_INJECT is an opt-in integration-test hook. Keeping + // the hook at transaction boundaries exercises the real rollback path; + // production calls take the unset fast path below without changing state. + want := strings.TrimSpace(os.Getenv("PODMAN_TFORK_FAULT_INJECT")) + if want == "" { + return nil + } + for _, candidate := range strings.Split(want, ",") { + if strings.TrimSpace(candidate) == stage { + return fmt.Errorf("injected tfork fault at %s", stage) + } + } + return nil +} diff --git a/podman/pkg/domain/infra/abi/container_tfork_transaction_test.go b/podman/pkg/domain/infra/abi/container_tfork_transaction_test.go new file mode 100644 index 000000000..91c1fb2c0 --- /dev/null +++ b/podman/pkg/domain/infra/abi/container_tfork_transaction_test.go @@ -0,0 +1,73 @@ +//go:build !remote + +package abi + +import ( + "errors" + "os" + "path/filepath" + "testing" +) + +func TestTforkInjectFault(t *testing.T) { + t.Setenv("PODMAN_TFORK_FAULT_INJECT", "after_freeze, before_commit") + for _, stage := range []string{"after_freeze", "before_commit"} { + if err := tforkInjectFault(stage); err == nil { + t.Fatalf("expected injected fault at %s", stage) + } + } + if err := tforkInjectFault("after_restore"); err != nil { + t.Fatalf("unexpected fault at unselected stage: %v", err) + } +} + +func TestTforkCgroupFSPath(t *testing.T) { + for _, unsafe := range []string{"", ".", "/", "..", "../escape"} { + if got := tforkCgroupFSPath(unsafe); got != "" { + t.Errorf("tforkCgroupFSPath(%q) = %q; want empty", unsafe, got) + } + } + const rel = "machine.slice/libpod-test" + if got, want := tforkCgroupFSPath(rel), "/sys/fs/cgroup/"+rel; got != want { + t.Fatalf("tforkCgroupFSPath(%q) = %q; want %q", rel, got, want) + } +} + +func TestTforkPIDRunning(t *testing.T) { + if err := tforkPIDRunning(os.Getpid()); err != nil { + t.Fatalf("current process should be running: %v", err) + } + if err := tforkPIDRunning(-1); err == nil { + t.Fatal("negative pid unexpectedly reported running") + } +} + +func TestTforkTransactionRollbackIsIdempotent(t *testing.T) { + temp := t.TempDir() + bundle := filepath.Join(temp, "graph", "tfork-bundles", "batch") + runtimeBatch := filepath.Join(temp, "run", "tfork", "batch") + for _, path := range []string{bundle, runtimeBatch} { + if err := os.MkdirAll(path, 0o700); err != nil { + t.Fatal(err) + } + } + + restoreCalls := 0 + txn := newTforkCloneTransaction(t.Context(), nil, nil, bundle, 1) + txn.setRuntimeBatchDir(runtimeBatch) + txn.setSourceRestore(func() error { + restoreCalls++ + return nil + }) + txn.rollback(errors.New("test")) + txn.rollback(errors.New("test again")) + + if restoreCalls != 1 { + t.Fatalf("source restore called %d times; want exactly once", restoreCalls) + } + for _, path := range []string{bundle, runtimeBatch} { + if _, err := os.Stat(path); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("rollback left %s: %v", path, err) + } + } +} diff --git a/ubuntu-img/Dockerfile b/ubuntu-img/Dockerfile index 235528ac9..782c9193c 100644 --- a/ubuntu-img/Dockerfile +++ b/ubuntu-img/Dockerfile @@ -119,6 +119,7 @@ RUN \ libreoffice-style-breeze \ libreoffice-writer \ thunderbird \ + tmux \ ubuntu-wallpapers \ ubuntu-wallpapers-jammy \ vlc && \ diff --git a/ubuntu-img/docker-compose.yml b/ubuntu-img/docker-compose.yml index cf5768548..3dc4c708a 100644 --- a/ubuntu-img/docker-compose.yml +++ b/ubuntu-img/docker-compose.yml @@ -1,6 +1,6 @@ services: webtop: - image: ${IMAGE:-ghcr.io/wuklab/webtop:ubuntu-kde} + image: ${IMAGE:-localhost/gensee-tclone-webtop:tmux} container_name: webtop security_opt: - seccomp=unconfined