diff --git a/README.md b/README.md index 0b4845d72..2a3201b7b 100644 --- a/README.md +++ b/README.md @@ -178,7 +178,10 @@ check [agent-s README](agents/agent-s/README.md) - btrfs only - rootful only - amd64 only -- linux page-cache CoW currently has a memory leak that will be fixed +- Filecow layers are reclaimed with their cached inode/mapping owners rather + than eagerly when a child is deleted. Repeated unchanged forks reuse the + current layer, so retained layers are bounded by changed generations instead + of the total fork count. ## License diff --git a/criu/build.sh b/criu/build.sh index cb1d9966c..cdb1cb472 100755 --- a/criu/build.sh +++ b/criu/build.sh @@ -93,12 +93,25 @@ for mod in "${ACTIVE_MODULES[@]}"; do fi if lsmod | awk '{print $1}' | grep -qx "${mod}"; then - echo " -- ${mod}: already loaded, rmmod then insmod" - rmmod "${mod}" 2>/dev/null || true - else - echo " -- ${mod}: insmod" + echo " -- ${mod}: already loaded, trying reload" + if ! rmmod "${mod}"; then + echo "build.sh: failed to unload loaded module ${mod}; aborting to avoid stale module" >&2 + echo "build.sh: stop running tfork/podman containers that may still hold ${mod}, then retry" >&2 + exit 1 + fi + if lsmod | awk '{print $1}' | grep -qx "${mod}"; then + echo "build.sh: module ${mod} is still loaded after rmmod; aborting to avoid stale module" >&2 + echo "build.sh: stop running tfork/podman containers that may still hold ${mod}, then retry" >&2 + exit 1 + fi + fi + + echo " -- ${mod}: insmod" + if ! insmod "${ko_path}"; then + echo "build.sh: failed to insert rebuilt module ${mod} from ${ko_path}" >&2 + echo "build.sh: if the old module is still active, stop running tfork/podman containers and rerun this script" >&2 + exit 1 fi - insmod "${ko_path}" done echo " -- verify all modules loaded" diff --git a/criu/criu/clone-noasan.c b/criu/criu/clone-noasan.c index 4ba7f6f31..368f6aa1e 100644 --- a/criu/criu/clone-noasan.c +++ b/criu/criu/clone-noasan.c @@ -78,7 +78,10 @@ int clone3_with_pid_noasan(int (*fn)(void *), void *arg, int flags, int exit_sig c_args.flags = flags; c_args.set_tid = ptr_to_u64(&pid); c_args.set_tid_size = 1; + pr_info("clone3 set_tid pid=%d flags=0x%x size=1\n", pid, flags); pid = syscall(__NR_clone3, &c_args, sizeof(c_args)); + if (pid < 0) + pr_perror("clone3 set_tid failed flags=0x%x size=1", flags); if (pid == 0) exit(fn(arg)); return pid; @@ -99,6 +102,12 @@ int clone3_with_nested_pid_noasan(int (*fn)(void *), void *arg, int flags, int e BUG_ON(pid->ns_level > MAX_PID_NS_LEVEL || pid->ns_level <= 1); for (i = 0; i < pid->ns_level; i++) tids[i] = pid->ns[i].ns_pid; + pr_info("clone3 nested set_tid flags=0x%x size=%d tids=%d/%d/%d/%d\n", + flags, pid->ns_level, + tids[0], + pid->ns_level > 1 ? tids[1] : -1, + pid->ns_level > 2 ? tids[2] : -1, + pid->ns_level > 3 ? tids[3] : -1); if (!(flags & CLONE_PARENT)) { if (exit_signal != SIGCHLD) { @@ -112,6 +121,13 @@ int clone3_with_nested_pid_noasan(int (*fn)(void *), void *arg, int flags, int e c_args.set_tid = ptr_to_u64(tids); c_args.set_tid_size = pid->ns_level; pid_ret = syscall(__NR_clone3, &c_args, sizeof(c_args)); + if (pid_ret < 0) + pr_perror("clone3 nested set_tid failed flags=0x%x size=%d tids=%d/%d/%d/%d", + flags, pid->ns_level, + tids[0], + pid->ns_level > 1 ? tids[1] : -1, + pid->ns_level > 2 ? tids[2] : -1, + pid->ns_level > 3 ? tids[3] : -1); if (pid_ret == 0) exit(fn(arg)); return pid_ret; diff --git a/criu/criu/cr-dump.c b/criu/criu/cr-dump.c index d196521fc..5179e5e53 100644 --- a/criu/criu/cr-dump.c +++ b/criu/criu/cr-dump.c @@ -885,6 +885,7 @@ static int collect_pstree_ids_predump(void) * write_img_inventory(). */ + pid_init_dump(crt.i.pid, &crt.i); crt.i.pid->state = TASK_ALIVE; crt.i.pid->real = getpid(); @@ -2360,6 +2361,9 @@ int cr_dump_tasks(pid_t pid) if (dump_zombies()) goto err; + if (finalize_nested_pid_ns_ids()) + goto err; + if (dump_pstree(root_item)) goto err; diff --git a/criu/criu/cr-restore.c b/criu/criu/cr-restore.c index d85279a82..bd4fa8b0a 100644 --- a/criu/criu/cr-restore.c +++ b/criu/criu/cr-restore.c @@ -185,10 +185,39 @@ static int __restore_wait_inprogress_tasks(int participants) { int ret; futex_t *np = &task_entries->nr_in_progress; + const int tfork_restore_wait_timeout_ms = 10000; + const int tfork_restore_wait_poll_us = 100000; + + if (opts.tfork.active) { + int waited; + + for (waited = 0; waited < tfork_restore_wait_timeout_ms; + waited += tfork_restore_wait_poll_us / 1000) { + if ((int)futex_get(np) <= participants) + break; + usleep(tfork_restore_wait_poll_us); + } + + if ((int)futex_get(np) > participants) { + pr_err("tfork restore wait timed out after %dms: participants=%d nr_in_progress=%d start_stage=%d task_cr_err=%d nr_tasks=%d nr_threads=%d nr_helpers=%d\n", + tfork_restore_wait_timeout_ms, + participants, (int)futex_get(np), + (int)futex_get(&task_entries->start), + get_task_cr_err(), + task_entries->nr_tasks, task_entries->nr_threads, + task_entries->nr_helpers); + set_cr_errno(ETIMEDOUT); + return -ETIMEDOUT; + } + } else { + futex_wait_while_gt(np, participants); + } - futex_wait_while_gt(np, participants); ret = (int)futex_get(np); if (ret < 0) { + pr_err("restore wait aborted: participants=%d nr_in_progress=%d start_stage=%d task_cr_err=%d\n", + participants, ret, (int)futex_get(&task_entries->start), + get_task_cr_err()); set_cr_errno(get_task_cr_err()); return ret; } @@ -227,6 +256,17 @@ static inline void __restore_switch_stage(int next_stage) static int restore_switch_stage(int next_stage) { + if (opts.tfork.active) + pr_warn("tfork: restore_switch_stage %d participants=%d nr_tasks=%d nr_threads=%d nr_helpers=%d\n", + next_stage, stage_participants(next_stage), + task_entries->nr_tasks, task_entries->nr_threads, + task_entries->nr_helpers); + else + pr_info("restore_switch_stage %d participants=%d nr_tasks=%d nr_threads=%d nr_helpers=%d\n", + next_stage, stage_participants(next_stage), + task_entries->nr_tasks, task_entries->nr_threads, + task_entries->nr_helpers); + __restore_switch_stage(next_stage); return restore_wait_inprogress_tasks(); } @@ -1395,6 +1435,16 @@ static inline int fork_with_pid(struct pstree_item *item) ca.item = item; ca.clone_flags = rsti(item)->clone_flags; + if (opts.tfork.active && item != root_item && + !(ca.clone_flags & CLONE_NEWPID) && + item->pid->ns_level > 1 && + item->pid->ns[0].ns_pid == INIT_PID) { + pr_info("tfork: repairing missing CLONE_NEWPID for pidns init uid=%d local=%d parent_local=%d level=%d\n", + uid(item), pid, + item->parent ? localpid(item->parent) : -1, + item->pid->ns_level); + ca.clone_flags |= CLONE_NEWPID; + } BUG_ON(ca.clone_flags & CLONE_VM); @@ -1434,13 +1484,45 @@ static inline int fork_with_pid(struct pstree_item *item) strip |= CLONE_NEWUSER; if (kdat.has_clone3_set_tid) { - if (item->pid->ns_level == 1) + if (opts.tfork.active && (ca.clone_flags & CLONE_NEWPID)) { + pr_info("tfork: restore pidns init uid=%d local pid %d with fresh parent pid, dumped chain level=%d\n", + uid(item), pid, item->pid->ns_level); ret = clone3_with_pid_noasan(restore_task_with_children, &ca, ca.clone_flags & ~strip, SIGCHLD, pid); - else + } else if (item->pid->ns_level == 1) + ret = clone3_with_pid_noasan(restore_task_with_children, &ca, + ca.clone_flags & ~strip, SIGCHLD, pid); + else { + struct pid tfork_pid = {}; + struct pid *restore_pid = item->pid; + + if (opts.tfork.active && (root_ns_mask & CLONE_NEWPID) && + root_item && root_item->pid->ns_level > 1 && + item->pid->ns_level > 1) { + /* + * Copy only scalar pid identity. struct pid also + * embeds rb_node links owned by the dumped pid trees; + * copying those nodes into a temporary stack object + * corrupts the tree metadata if it ever gets reused. + */ + tfork_pid.item = item->pid->item; + tfork_pid.real = item->pid->real; + tfork_pid.local = item->pid->local; + tfork_pid.uid = item->pid->uid; + tfork_pid.state = item->pid->state; + tfork_pid.stop_signo = item->pid->stop_signo; + tfork_pid.ns_level = item->pid->ns_level - 1; + tfork_pid.leaf_ns_id = item->pid->leaf_ns_id; + memcpy(tfork_pid.ns, item->pid->ns, sizeof(tfork_pid.ns)); + restore_pid = &tfork_pid; + pr_info("tfork: restore pid uid=%d local=%d with rebased pid chain level %d -> %d\n", + uid(item), pid, item->pid->ns_level, + restore_pid->ns_level); + } ret = clone3_with_nested_pid_noasan(restore_task_with_children, &ca, ca.clone_flags & ~strip, - SIGCHLD, item->pid); + SIGCHLD, restore_pid); + } } else { BUG_ON(item->pid->ns_level >= 1); close_pid_proc(); @@ -1448,13 +1530,26 @@ static inline int fork_with_pid(struct pstree_item *item) (ca.clone_flags & ~strip) | SIGCHLD, &ca); } if (ret < 0) { + pr_err("fork_with_pid failed item uid=%d local=%d real=%d parent_local=%d flags=0x%lx stripped_flags=0x%lx ns_level=%d root_ns_mask=0x%lx tfork=%d\n", + uid(item), pid, realpid(item), + item->parent ? localpid(item->parent) : -1, + ca.clone_flags, ca.clone_flags & ~strip, + item->pid->ns_level, root_ns_mask, + opts.tfork.active ? 1 : 0); + if (item->pid->ns_level > 0) + pr_err("fork_with_pid pid chain uid=%d ns=%d/%d/%d/%d\n", + uid(item), + item->pid->ns[0].ns_pid, + item->pid->ns_level > 1 ? item->pid->ns[1].ns_pid : -1, + item->pid->ns_level > 2 ? item->pid->ns[2].ns_pid : -1, + item->pid->ns_level > 3 ? item->pid->ns[3].ns_pid : -1); pr_perror("Can't fork for %d", pid); if (errno == EEXIST) set_cr_errno(EEXIST); goto err_unlock; } - if (item == root_item) { + if (opts.tfork.active || item == root_item) { item->pid->real = ret; pr_debug("PID: real %d virt %d\n", item->pid->real, localpid(item)); } @@ -4045,6 +4140,7 @@ static int sigreturn_restore(struct task_restore_args *task_args, unsigned long task_args->vdso_rt_size = vdso_rt_size; task_args->can_map_vdso = kdat.can_map_vdso; task_args->has_clone3_set_tid = kdat.has_clone3_set_tid; + task_args->tfork_active = opts.tfork.active; new_sp = restorer_stack(task_args->t->mz); diff --git a/criu/criu/cr-tfork.c b/criu/criu/cr-tfork.c index ea64d6627..f99a6d2da 100644 --- a/criu/criu/cr-tfork.c +++ b/criu/criu/cr-tfork.c @@ -271,7 +271,7 @@ int tfork_read_cropt(void) return 0; snap_path = opts.tfork.snap_root; - if (!snap_path && opts.tfork.copies > 1 && opts.tfork.snap_roots && + if (!snap_path && opts.tfork.copies >= 1 && opts.tfork.snap_roots && opts.tfork.copy_idx < opts.tfork.snap_roots_n) snap_path = opts.tfork.snap_roots[opts.tfork.copy_idx]; @@ -966,8 +966,9 @@ int cr_tfork_tasks(pid_t pid) opts.tfork.pidfd_map[opts.tfork.pidfd_map_nr].pidfd = pidfd; opts.tfork.pidfd_map[opts.tfork.pidfd_map_nr].memfd = -1; opts.tfork.pidfd_map_nr++; - pr_info("tfork: pidfd %d for pid %d (vpid %d uid %d)\n", - pidfd, item->pid->real, localpid(item), uid(item)); + pr_info("tfork: pidfd %d for pid %d (vpid %d uid %d nsid %d level %d)\n", + pidfd, item->pid->real, localpid(item), uid(item), + item->pid->leaf_ns_id, item->pid->ns_level); } ret = run_scripts(ACT_PRE_TFORK_RESTORE); @@ -1078,7 +1079,7 @@ int cr_tfork_tasks(pid_t pid) list_for_each_entry(cgo_iter, &opts.new_cgroup_roots, node) rpc_n_cg_root++; - rpc_max = 32 + 2 * (rpc_n_ifd + rpc_n_ext + rpc_n_cg_root + rpc_max = 33 + 2 * (rpc_n_ifd + rpc_n_ext + rpc_n_cg_root + opts.tfork.snap_mount_n) + rpc_n_copy_args + 2; rpc_argv = calloc(rpc_max, sizeof(*rpc_argv)); @@ -1098,6 +1099,7 @@ int cr_tfork_tasks(pid_t pid) rpc_argv[rpc_n++] = "-o"; rpc_argv[rpc_n++] = restore_log_arg; rpc_argv[rpc_n++] = "-v2"; + rpc_argv[rpc_n++] = "--keep-pid-hierarchy"; if (opts.root) { rpc_argv[rpc_n++] = "--root"; @@ -1171,7 +1173,7 @@ int cr_tfork_tasks(pid_t pid) rpc_argv[rpc_n++] = "--tfork-snap-mounts"; rpc_argv[rpc_n++] = snap_mounts_csv; } - if (opts.tfork.copies > 1) { + if (opts.tfork.copies >= 1) { snprintf(copies_arg, sizeof(copies_arg), "%d", opts.tfork.copies); rpc_argv[rpc_n++] = "--tfork-copies"; @@ -1267,7 +1269,7 @@ int cr_tfork_tasks(pid_t pid) buf[off] = '\0'; end = buf + off; - argv_max = 8 + 2; + argv_max = 8 + 3; for (p = buf; p < end; p++) if (*p == '\0') argv_max++; @@ -1312,6 +1314,11 @@ int cr_tfork_tasks(pid_t pid) argv_new[argc_new++] = "--pidfile"; argv_new[argc_new++] = pidfile_arg; } + if ((size_t)argc_new + 1 >= argv_max) { + pr_err("tfork restore argv overflow: used=%d max=%zu\n", argc_new, argv_max); + exit(1); + } + argv_new[argc_new++] = "--keep-pid-hierarchy"; argv_new[argc_new] = NULL; execv("/proc/self/exe", argv_new); diff --git a/criu/criu/crtools.c b/criu/criu/crtools.c index df40ec14f..639ad06f5 100644 --- a/criu/criu/crtools.c +++ b/criu/criu/crtools.c @@ -332,25 +332,30 @@ int main(int argc, char *argv[], char *envp[]) if (opts.tree_id) pr_warn("Using -t with criu restore is obsoleted\n"); - if (opts.tfork.copies > 1) { + if (!opts.tfork.active && opts.tfork.copies >= 1) { + pr_err("--tfork-copies requires --tfork-restore " + "(use 'criu tfork --tfork-copies=N', not " + "'criu restore --tfork-copies=N')\n"); + return 1; + } + + if (opts.tfork.active && opts.tfork.copies >= 1) { int n = opts.tfork.copies, i; pid_t *children; int (*ready_pipes)[2]; int failed = 0; const char *base_log = opts.output; - const int ns_flags = - CLONE_NEWPID | CLONE_NEWNS; - - if (!opts.tfork.active) { - pr_err("--tfork-copies>1 requires --tfork-restore " - "(use 'criu tfork --tfork-copies=N', not " - "'criu restore --tfork-copies=N')\n"); - return 1; - } + /* + * The n-copy helper must not create/occupy PID 1 in a new + * PID namespace. CRIU restores the real root task as PID 1 + * from the image; if the helper has already consumed it, + * restore fails with EEXIST ("Can't fork for 1"). + */ + const int ns_flags = CLONE_NEWNS; if (!opts.restore_detach) { - pr_err("--tfork-copies>1 requires --restore-detached\n"); + pr_err("--tfork-copies requires --restore-detached\n"); return 1; } @@ -445,6 +450,13 @@ int main(int argc, char *argv[], char *envp[]) if (opts.tfork.snap_roots_n > 0) opts.root = opts.tfork.snap_roots[i]; + /* + * The n-copy child only creates the per-copy mount namespace. PID + * namespaces must be recreated by CRIU from the image, otherwise the + * helper would occupy PID 1 before the restored root task. + */ + opts.keep_pid_hierarchy = 0; + if (tfork_load_ncopy_fabric(i)) { pr_err("tfork-ncopy: copy %d fabric load failed\n", i); diff --git a/criu/criu/image.c b/criu/criu/image.c index 2783b5797..1a93344d5 100644 --- a/criu/criu/image.c +++ b/criu/criu/image.c @@ -374,6 +374,7 @@ int prepare_inventory(InventoryEntry *he) he->has_lsmtype = true; he->lsmtype = host_lsm_type(); + pid_init_dump(crt.i.pid, &crt.i); crt.i.pid->state = TASK_ALIVE; crt.i.pid->real = getpid(); if (get_task_ids(&crt.i)) diff --git a/criu/criu/include/namespaces.h b/criu/criu/include/namespaces.h index e442e0a39..7a5aa6437 100644 --- a/criu/criu/include/namespaces.h +++ b/criu/criu/include/namespaces.h @@ -187,6 +187,7 @@ extern int restore_mnt_ns(int rst, int *cwd_fd); extern int dump_task_ns_ids(struct pstree_item *); extern int predump_task_ns_ids(struct pstree_item *); +extern int finalize_nested_pid_ns_ids(void); extern int rst_add_ns_id(unsigned int id, struct pstree_item *, struct ns_desc *nd); extern struct ns_id *lookup_ns_by_id(unsigned int id, struct ns_desc *nd); diff --git a/criu/criu/include/pstree.h b/criu/criu/include/pstree.h index f9bdeffd1..51617015a 100644 --- a/criu/criu/include/pstree.h +++ b/criu/criu/include/pstree.h @@ -20,6 +20,23 @@ extern atomic_t pid_uid_generator; #define HELPER_UID_BASE (0x40000000) +static inline void pid_init_dump(struct pid *pid, struct pstree_item *item) +{ + *pid = (struct pid){ + .item = item, + .real = -1, + .local = -1, + .uid = atomic_inc_return(&pid_uid_generator), + .state = TASK_UNDEF, + .stop_signo = -1, + .ns_level = -1, + .leaf_ns_id = ALL_PID_NS_ID, + }; + rb_init_node(&pid->leaf_ns_node); + rb_init_node(&pid->root_ns_node); + rb_init_node(&pid->uid_node); +} + struct pstree_item { struct pstree_item *parent; struct list_head children; /* list of my children */ diff --git a/criu/criu/include/restorer.h b/criu/criu/include/restorer.h index 73e27caa3..40bb132d1 100644 --- a/criu/criu/include/restorer.h +++ b/criu/criu/include/restorer.h @@ -242,6 +242,7 @@ struct task_restore_args { int child_subreaper; int membarrier_registration_mask; bool has_clone3_set_tid; + bool tfork_active; /* * info about rseq from libc used to diff --git a/criu/criu/namespaces.c b/criu/criu/namespaces.c index df224db68..11909cea2 100644 --- a/criu/criu/namespaces.c +++ b/criu/criu/namespaces.c @@ -592,6 +592,93 @@ static unsigned int get_ns_id(int pid, struct ns_desc *nd, protobuf_c_boolean *s return __get_ns_id(pid, nd, supported, NULL); } +static unsigned int add_nested_pid_leaf_ns_id(struct pstree_item *item) +{ + struct ns_id *nsid; + + nsid = xzalloc(sizeof(*nsid)); + if (!nsid) + return 0; + + nsid->type = NS_OTHER; + nsid->kid = 0; + nsid->ns_populated = true; + nsid_add(nsid, &pid_ns_desc, ns_next_id++, localpid(item)); + + pr_info("Add nested pid leaf ns %d for task %d(%d), level %d\n", + nsid->id, localpid(item), realpid(item), item->pid->ns_level); + return nsid->id; +} + +static unsigned int ensure_task_leaf_pid_ns_id(struct pstree_item *item); + +static unsigned int task_leaf_pid_ns_id(struct pstree_item *item, unsigned int proc_pid_ns_id) +{ + struct pstree_item *parent = item->parent; + unsigned int selected; + + /* + * os4agent stores localpid as the innermost NSpid (pid->ns[0]). + * Keep pstree_entry.nsid at the same namespace level. Otherwise a + * nested pid namespace init such as bwrap can become (nsid=N, + * localpid=1) and collide with the container init in the same nsid. + */ + if (parent && ensure_task_leaf_pid_ns_id(parent) == 0) + return 0; + + if (parent && parent->pid->leaf_ns_id != ALL_PID_NS_ID) { + if (item->pid->ns_level == parent->pid->ns_level) { + selected = parent->pid->leaf_ns_id; + goto out; + } + if (item->pid->ns_level > parent->pid->ns_level && + proc_pid_ns_id != parent->pid->leaf_ns_id) { + selected = proc_pid_ns_id; + goto out; + } + if (item->pid->ns_level > parent->pid->ns_level) { + selected = add_nested_pid_leaf_ns_id(item); + goto out; + } + } + + selected = proc_pid_ns_id; + +out: + pr_info("pid leaf ns task=%d(%d) uid=%d level=%d parent_level=%d proc_nsid=%u parent_nsid=%d selected=%u\n", + localpid(item), realpid(item), uid(item), item->pid->ns_level, + parent ? parent->pid->ns_level : -1, proc_pid_ns_id, + parent ? parent->pid->leaf_ns_id : -1, selected); + return selected; +} + +static unsigned int ensure_task_leaf_pid_ns_id(struct pstree_item *item) +{ + struct pstree_item *parent = item->parent; + unsigned int proc_pid_ns_id; + + if (parent && ensure_task_leaf_pid_ns_id(parent) == 0) + return 0; + + if (item->pid->leaf_ns_id != ALL_PID_NS_ID) { + if (!parent) + return item->pid->leaf_ns_id; + if (item->pid->ns_level == parent->pid->ns_level && + item->pid->leaf_ns_id == parent->pid->leaf_ns_id) + return item->pid->leaf_ns_id; + if (item->pid->ns_level > parent->pid->ns_level && + item->pid->leaf_ns_id != parent->pid->leaf_ns_id) + return item->pid->leaf_ns_id; + } + + proc_pid_ns_id = get_ns_id(item->pid->real, &pid_ns_desc, NULL); + if (!proc_pid_ns_id) + return 0; + + item->pid->leaf_ns_id = task_leaf_pid_ns_id(item, proc_pid_ns_id); + return item->pid->leaf_ns_id; +} + int dump_one_ns_file(int lfd, u32 id, const struct fd_parms *p) { struct cr_img *img; @@ -771,15 +858,37 @@ int dump_task_ns_ids(struct pstree_item *item) int i; int pid = item->pid->real; TaskKobjIdsEntry *ids = item->ids; + struct pstree_item *parent = item->parent; + unsigned int proc_pid_ns_id; ids->has_pid_ns_id = true; - ids->pid_ns_id = get_ns_id(pid, &pid_ns_desc, NULL); + proc_pid_ns_id = get_ns_id(pid, &pid_ns_desc, NULL); + if (!proc_pid_ns_id) { + pr_err("Can't make pidns id\n"); + return -1; + } + + if (parent && ensure_task_leaf_pid_ns_id(parent) == 0) + return -1; + + ids->pid_ns_id = proc_pid_ns_id; + if (parent && item->pid->ns_level == parent->pid->ns_level) + ids->pid_ns_id = parent->pid->leaf_ns_id; + else if (parent && item->pid->ns_level > parent->pid->ns_level && + ids->pid_ns_id == parent->pid->leaf_ns_id) + ids->pid_ns_id = add_nested_pid_leaf_ns_id(item); + if (!ids->pid_ns_id) { pr_err("Can't make pidns id\n"); return -1; } item->pid->leaf_ns_id = ids->pid_ns_id; + pr_info("dump pid ns task=%d(%d) uid=%d level=%d parent_level=%d proc_nsid=%u parent_nsid=%d selected=%u\n", + localpid(item), realpid(item), uid(item), item->pid->ns_level, + parent ? parent->pid->ns_level : -1, proc_pid_ns_id, + parent ? parent->pid->leaf_ns_id : -1, ids->pid_ns_id); + for (i = 0; i < item->nr_threads; i++) item->threads[i].leaf_ns_id = ids->pid_ns_id; @@ -865,6 +974,40 @@ int dump_task_ns_ids(struct pstree_item *item) return 0; } +int finalize_nested_pid_ns_ids(void) +{ + struct pstree_item *item; + + for_each_pstree_item(item) { + struct pstree_item *parent = item->parent; + unsigned int nsid; + int i; + + if (!parent) + continue; + if (item->pid->ns_level <= parent->pid->ns_level) + continue; + if (item->pid->leaf_ns_id != parent->pid->leaf_ns_id) + continue; + + nsid = add_nested_pid_leaf_ns_id(item); + if (!nsid) + return -1; + + item->pid->leaf_ns_id = nsid; + for (i = 0; i < item->nr_threads; i++) + item->threads[i].leaf_ns_id = nsid; + if (item->ids && item->ids->has_pid_ns_id) + item->ids->pid_ns_id = nsid; + + pr_info("finalize nested pid ns task=%d(%d) uid=%d level=%d parent_nsid=%d selected=%u\n", + localpid(item), realpid(item), uid(item), item->pid->ns_level, + parent->pid->leaf_ns_id, nsid); + } + + return 0; +} + static UsernsEntry userns_entry = USERNS_ENTRY__INIT; #define INVALID_ID (~0U) diff --git a/criu/criu/pie/restorer.c b/criu/criu/pie/restorer.c index 9aaefe502..a63dbdd62 100644 --- a/criu/criu/pie/restorer.c +++ b/criu/criu/pie/restorer.c @@ -97,6 +97,7 @@ static pid_t *helpers; static int n_helpers; static pid_t *zombies; static int n_zombies; +static bool tfork_active_local; static enum faults fi_strategy; bool fault_injected(enum faults f) { @@ -162,7 +163,8 @@ static void sigchld_handler(int signal, siginfo_t *siginfo, void *data) else r = "disappeared with"; - pr_info("Task %d %s %d\n", siginfo->si_pid, r, siginfo->si_status); + pr_err("SIGCHLD during restore: task %d %s %d\n", + siginfo->si_pid, r, siginfo->si_status); futex_abort_and_wake(&task_entries_local->nr_in_progress); /* sa_restorer may be unmaped, so we can't go back to userspace*/ @@ -807,12 +809,24 @@ __visible long __export_restore_thread(struct thread_restore_args *args) } pr_info("%ld: Restored\n", sys_gettid()); - restore_finish_stage(task_entries_local, CR_STATE_RESTORE); + if (args->ta->tfork_active) + pr_debug("tfork: thread restore stage complete pid=%d tid=%ld comm=%s ns_level=%d\n", + args->pid, sys_gettid(), args->comm, args->ns_level); + ret = restore_finish_stage(task_entries_local, CR_STATE_RESTORE); + if (args->ta->tfork_active) + pr_debug("tfork: thread restore barrier returned pid=%d tid=%ld comm=%s stage=%d\n", + args->pid, sys_gettid(), args->comm, ret); if (restore_signals(args->siginfo, args->siginfo_n, false)){ goto core_restore_end; } - restore_finish_stage(task_entries_local, CR_STATE_RESTORE_SIGCHLD); + if (args->ta->tfork_active) + pr_debug("tfork: thread sigchld stage complete pid=%d tid=%ld comm=%s ns_level=%d\n", + args->pid, sys_gettid(), args->comm, args->ns_level); + ret = restore_finish_stage(task_entries_local, CR_STATE_RESTORE_SIGCHLD); + if (args->ta->tfork_active) + pr_debug("tfork: thread sigchld barrier returned pid=%d tid=%ld comm=%s stage=%d\n", + args->pid, sys_gettid(), args->comm, ret); /* * Make sure it's before creds, since it's privileged @@ -1476,6 +1490,11 @@ static int wait_zombies(struct task_restore_args *task_args) ret = sys_waitid(P_PID, task_args->zombies[i], NULL, WNOWAIT | WEXITED, NULL); if (ret == -ECHILD) { + if (task_args->tfork_active) { + pr_warn("tfork: zombie pid %d is not reparented to task %ld; skipping wait to avoid restore barrier deadlock\n", + task_args->zombies[i], sys_getpid()); + continue; + } /* A process isn't reparented to this task yet. * Let's wait when someone complete this stage * and try again. @@ -1752,6 +1771,7 @@ __visible long __export_restore_task(struct task_restore_args *args) fi_strategy = args->fault_strategy; task_entries_local = args->task_entries; + tfork_active_local = args->tfork_active; helpers = args->helpers; n_helpers = args->helpers_n; zombies = args->zombies; @@ -2465,6 +2485,18 @@ __visible long __export_restore_task(struct task_restore_args *args) c_args.set_tid = ptr_to_u64(thread_args[i].tid_in_ns); c_args.flags = clone_flags; c_args.set_tid_size = thread_args[i].ns_level; + if (args->tfork_active && thread_args[i].ns_level > 0) { + /* + * Preserve the TID visible in the clone's innermost PID namespace. + * Outer namespace TIDs are allocated by the kernel so concurrent + * copy helpers cannot collide with each other on the host. + */ + pr_debug("tfork: restore thread pid=%d with innermost tid=%d, set_tid_size %d -> 1\n", + thread_args[i].pid, + thread_args[i].tid_in_ns[0], + thread_args[i].ns_level); + c_args.set_tid_size = 1; + } /* The kernel does stack + stack_size. */ c_args.stack = new_sp - RESTORE_STACK_SIZE; c_args.stack_size = RESTORE_STACK_SIZE; @@ -2495,7 +2527,14 @@ __visible long __export_restore_task(struct task_restore_args *args) args->clone_restore_fn); } if (ret != thread_args[i].pid) { - pr_err("Unable to create a thread: %ld\n", ret); + pr_err("Unable to create a thread: %ld expected=%d ns_level=%d tids=%d/%d/%d/%d tfork=%d\n", + ret, thread_args[i].pid, + thread_args[i].ns_level, + thread_args[i].tid_in_ns[0], + thread_args[i].ns_level > 1 ? thread_args[i].tid_in_ns[1] : -1, + thread_args[i].ns_level > 2 ? thread_args[i].tid_in_ns[2] : -1, + thread_args[i].ns_level > 3 ? thread_args[i].tid_in_ns[3] : -1, + args->tfork_active ? 1 : 0); sys_close(fd); mutex_unlock(&task_entries_local->last_pid_mutex); goto core_restore_end; @@ -2524,13 +2563,33 @@ __visible long __export_restore_task(struct task_restore_args *args) if (restore_membarrier_registrations(args->membarrier_registration_mask) < 0) goto core_restore_end; pr_info("%ld: Restored\n", sys_getpid()); - - restore_finish_stage(task_entries_local, CR_STATE_RESTORE); - + if (args->tfork_active) + pr_debug("tfork: leader restore stage complete pid=%d tid=%ld comm=%s threads=%d ns_level=%d\n", + args->t->pid, sys_getpid(), args->comm, args->nr_threads, + args->t->ns_level); + + ret = restore_finish_stage(task_entries_local, CR_STATE_RESTORE); + if (args->tfork_active) + pr_debug("tfork: leader restore barrier returned pid=%d tid=%ld comm=%s stage=%ld helpers=%u zombies=%u inotify=%u\n", + args->t->pid, sys_getpid(), args->comm, ret, + args->helpers_n, args->zombies_n, args->inotify_fds_n); + + if (args->tfork_active) + pr_debug("tfork: leader wait_helpers start pid=%d tid=%ld comm=%s helpers=%u\n", + args->t->pid, sys_getpid(), args->comm, args->helpers_n); if (wait_helpers(args) < 0) goto core_restore_end; + if (args->tfork_active) + pr_debug("tfork: leader wait_helpers done pid=%d tid=%ld comm=%s\n", + args->t->pid, sys_getpid(), args->comm); + if (args->tfork_active) + pr_debug("tfork: leader wait_zombies start pid=%d tid=%ld comm=%s zombies=%u\n", + args->t->pid, sys_getpid(), args->comm, args->zombies_n); if (wait_zombies(args) < 0) goto core_restore_end; + if (args->tfork_active) + pr_debug("tfork: leader wait_zombies done pid=%d tid=%ld comm=%s\n", + args->t->pid, sys_getpid(), args->comm); ksigfillset(&to_block); ret = sys_sigprocmask(SIG_SETMASK, &to_block, NULL, sizeof(k_rtsigset_t)); @@ -2539,9 +2598,18 @@ __visible long __export_restore_task(struct task_restore_args *args) goto core_restore_end; } + if (args->tfork_active) + pr_debug("tfork: leader cleanup_inotify start pid=%d tid=%ld comm=%s inotify=%u\n", + args->t->pid, sys_getpid(), args->comm, args->inotify_fds_n); if (cleanup_current_inotify_events(args)) goto core_restore_end; + if (args->tfork_active) + pr_debug("tfork: leader cleanup_inotify done pid=%d tid=%ld comm=%s\n", + args->t->pid, sys_getpid(), args->comm); + if (args->tfork_active) + pr_debug("tfork: leader restore sigaction start pid=%d tid=%ld comm=%s\n", + args->t->pid, sys_getpid(), args->comm); if (!args->compatible_mode) { ret = sys_sigaction(SIGCHLD, &args->sigchld_act, NULL, sizeof(k_rtsigset_t)); } else { @@ -2558,16 +2626,38 @@ __visible long __export_restore_task(struct task_restore_args *args) pr_err("Failed to restore SIGCHLD: %ld\n", ret); goto core_restore_end; } + if (args->tfork_active) + pr_debug("tfork: leader restore sigaction done pid=%d tid=%ld comm=%s\n", + args->t->pid, sys_getpid(), args->comm); + if (args->tfork_active) + pr_debug("tfork: leader restore shared signals start pid=%d tid=%ld comm=%s\n", + args->t->pid, sys_getpid(), args->comm); ret = restore_signals(args->siginfo, args->siginfo_n, true); if (ret) goto core_restore_end; + if (args->tfork_active) + pr_debug("tfork: leader restore shared signals done pid=%d tid=%ld comm=%s\n", + args->t->pid, sys_getpid(), args->comm); + if (args->tfork_active) + pr_debug("tfork: leader restore private signals start pid=%d tid=%ld comm=%s\n", + args->t->pid, sys_getpid(), args->comm); ret = restore_signals(args->t->siginfo, args->t->siginfo_n, false); if (ret) goto core_restore_end; - - restore_finish_stage(task_entries_local, CR_STATE_RESTORE_SIGCHLD); + if (args->tfork_active) + pr_debug("tfork: leader restore private signals done pid=%d tid=%ld comm=%s\n", + args->t->pid, sys_getpid(), args->comm); + + if (args->tfork_active) + pr_debug("tfork: leader sigchld stage complete pid=%d tid=%ld comm=%s threads=%d ns_level=%d\n", + args->t->pid, sys_getpid(), args->comm, args->nr_threads, + args->t->ns_level); + ret = restore_finish_stage(task_entries_local, CR_STATE_RESTORE_SIGCHLD); + if (args->tfork_active) + pr_debug("tfork: leader sigchld barrier returned pid=%d tid=%ld comm=%s stage=%ld\n", + args->t->pid, sys_getpid(), args->comm, ret); rst_tcp_socks_all(args); diff --git a/criu/criu/pstree.c b/criu/criu/pstree.c index c448c9516..29f1e5d61 100644 --- a/criu/criu/pstree.c +++ b/criu/criu/pstree.c @@ -272,26 +272,26 @@ struct pstree_item *__alloc_pstree_item(bool rst) INIT_LIST_HEAD(&item->children); INIT_LIST_HEAD(&item->sibling); - item->pid->ns_level = -1; - item->pid->leaf_ns_id = ALL_PID_NS_ID; - item->pid->real = -1; - item->pid->local = -1; - if (!rst) - item->pid->uid = atomic_inc_return(&pid_uid_generator); - else + pid_init_dump(item->pid, item); + else { + item->pid->ns_level = -1; + item->pid->leaf_ns_id = ALL_PID_NS_ID; + item->pid->real = -1; + item->pid->local = -1; item->pid->uid = -1; - item->pid->state = TASK_UNDEF; - item->pid->stop_signo = -1; + item->pid->state = TASK_UNDEF; + item->pid->stop_signo = -1; + item->pid->item = item; + rb_init_node(&item->pid->leaf_ns_node); + rb_init_node(&item->pid->root_ns_node); + rb_init_node(&item->pid->uid_node); + } item->born_sid = -1; item->tfork_pidfd = -1; item->tfork_memfd = -1; item->tfork_pagemap_fd = -1; - item->pid->item = item; futex_init(&item->task_st); - rb_init_node(&item->pid->leaf_ns_node); - rb_init_node(&item->pid->root_ns_node); - rb_init_node(&item->pid->uid_node); return item; } @@ -436,7 +436,10 @@ int dump_pstree(struct pstree_item *root_item) pstree_entry__init(e); tree_entries[nr_items++] = e; - pr_info("Process: %d(%d)\n", localpid(item), realpid(item)); + pr_info("Process: %d(%d) uid=%d nsid=%d level=%d parent=%d parent_nsid=%d\n", + localpid(item), realpid(item), uid(item), item->pid->leaf_ns_id, + item->pid->ns_level, item->parent ? realpid(item->parent) : 0, + item->parent ? item->parent->pid->leaf_ns_id : -1); e->realpid = realpid(item); e->ppid = item->parent ? realpid(item->parent) : 0; @@ -729,11 +732,37 @@ static int __pstree_insert_pid(struct pid *pid_node, struct rb_node *root_parent rb_link_and_balance(&uid_root_rb, &pid_node->uid_node, parent, link); } - return 0; + return 0; err: - rb_erase(&pid_node->root_ns_node, &pid_root_rb[ALL_PID_NS_ID]); - return -1; + rb_erase(&pid_node->root_ns_node, &pid_root_rb[ALL_PID_NS_ID]); + return -1; +} + +static void pstree_remove_pid_if_linked(struct pid *pid_node) +{ + struct pid *found; + bool valid_leaf_ns = pid_node->leaf_ns_id >= 0 && + (unsigned int)pid_node->leaf_ns_id < pid_namespace_count; + + if (pid_node->uid > 0) { + found = __lookup_pid_uid(&uid_root_rb, pid_node->uid, NULL, NULL); + if (found == pid_node) + rb_erase(&pid_node->uid_node, &uid_root_rb); + } + + if (pid_node->leaf_ns_id != ALL_PID_NS_ID && valid_leaf_ns) { + found = __lookup_pid_leaf(&pid_root_rb[pid_node->leaf_ns_id], + pid_node->local, NULL, NULL); + if (found == pid_node) + rb_erase(&pid_node->leaf_ns_node, + &pid_root_rb[pid_node->leaf_ns_id]); + } + + found = __lookup_pid_root(&pid_root_rb[ALL_PID_NS_ID], + pid_node->real, NULL, NULL); + if (found == pid_node) + rb_erase(&pid_node->root_ns_node, &pid_root_rb[ALL_PID_NS_ID]); } int pstree_insert_pid(struct pid *pid_node) @@ -741,13 +770,14 @@ int pstree_insert_pid(struct pid *pid_node) return __pstree_insert_pid(pid_node, NULL, NULL); } -static struct pstree_item *get_or_create_pstree_item(pid_t real, pid_t local, int pidns_id) +static struct pstree_item *get_or_create_pstree_item(pid_t real, pid_t local, int pidns_id, bool *created) { struct pid *found; struct pstree_item *item; - struct rb_node **root_link, *root_parent; - found = __lookup_pid_root(&pid_root_rb[ALL_PID_NS_ID], real, &root_parent, &root_link); + *created = false; + + found = __lookup_pid_root(&pid_root_rb[ALL_PID_NS_ID], real, NULL, NULL); if (found) { if (pidns_id != ALL_PID_NS_ID) { BUG_ON(found->leaf_ns_id != pidns_id || found->local != local); @@ -762,11 +792,7 @@ static struct pstree_item *get_or_create_pstree_item(pid_t real, pid_t local, in item->pid->real = real; item->pid->local = local; item->pid->leaf_ns_id = pidns_id; - - if (__pstree_insert_pid(item->pid, root_parent, root_link) < 0) { - xfree(item); - return NULL; - } + *created = true; return item; } @@ -876,13 +902,24 @@ static int read_pstree_ids(struct pstree_item *pi) */ static int read_one_pstree_item(PstreeEntry *e) { - struct pstree_item *pi; - int ret = -1, i, j; + struct pstree_item *pi = NULL; + int ret = -1, i, j, next_inserted_thread = 1; + bool linked = false, pid_inserted = false, threads_allocated = false; + bool created_item = false; - pi = get_or_create_pstree_item(e->realpid, e->localpid, e->nsid); + pi = get_or_create_pstree_item(e->realpid, e->localpid, e->nsid, &created_item); if (!pi) goto err; + /* + * get_or_create_pstree_item() can only reuse an item that is still + * TASK_UNDEF. Completed items are rejected here, so the unwind below + * cannot tear down a previously parsed pstree item. + */ BUG_ON(pi->pid->state != TASK_UNDEF); + if (!created_item && (pi->threads || pi->nr_threads)) { + pr_err("Refusing to reuse partially populated pstree item for %d\n", e->realpid); + goto err; + } /* * Populate the ns-chain on pi from the thread-leader entry before @@ -915,6 +952,27 @@ static int read_one_pstree_item(PstreeEntry *e) } pi->pid->state = TASK_ALIVE; pi->pid->uid = e->uid; + pi->nr_threads = e->n_threads; + pi->threads = xmalloc(e->n_threads * sizeof(struct pid)); + if (!pi->threads) + goto err; + threads_allocated = true; + + /* note: we don't fail if we have empty ids */ + if (read_pstree_ids(pi) < 0) + goto err; + + if (pi->ids && pi->ids->has_pid_ns_id) { + if (pi->ids->pid_ns_id != pi->pid->leaf_ns_id) { + pr_warn("PID namespace id mismatch for uid %d: pstree=%d ids=%d, keeping pstree\n", + uid(pi), pi->pid->leaf_ns_id, pi->ids->pid_ns_id); + pi->ids->pid_ns_id = pi->pid->leaf_ns_id; + } + } + + if (__pstree_insert_pid(pi->pid, NULL, NULL) < 0) + goto err; + pid_inserted = true; if (e->ppid == 0) { if (root_item) { @@ -938,13 +996,9 @@ static int read_one_pstree_item(PstreeEntry *e) parent = pid->item; pi->parent = parent; list_add(&pi->sibling, &parent->children); + linked = true; } - pi->nr_threads = e->n_threads; - pi->threads = xmalloc(e->n_threads * sizeof(struct pid)); - if (!pi->threads) - goto err; - for (i = 0; i < e->n_threads; i++) { int insert_status; pi->threads[i].uid = e->threads[i]->uid; @@ -960,7 +1014,7 @@ static int read_one_pstree_item(PstreeEntry *e) pi->threads[i].state = TASK_THREAD; pi->threads[i].item = NULL; if (i == 0) { - + /* The leader is indexed through pi->pid, not this mirror. */ pi->pid->ns_level = pi->threads[0].ns_level; pi->pid->local = pi->threads[0].ns[0].ns_pid; memcpy(pi->pid->ns, pi->threads[0].ns, e->threads[0]->n_ns * sizeof(struct pid_ns)); @@ -972,17 +1026,42 @@ static int read_one_pstree_item(PstreeEntry *e) pr_err("Unexpected task %d in a tree %d\n", e->threads[i]->ns[0]->nspid, i); goto err; } + next_inserted_thread = i + 1; } task_entries->nr_threads += e->n_threads; task_entries->nr_tasks++; - /* note: we don't fail if we have empty ids */ - if (read_pstree_ids(pi) < 0) - goto err; - ret = 1; err: + if (ret < 0 && pi) { + /* + * threads[0] is the leader mirrored by pi->pid. Only + * non-leader threads are inserted independently, and + * next_inserted_thread always points one past the last + * successfully inserted non-leader slot. + */ + for (i = 1; i < next_inserted_thread; i++) + pstree_remove_pid_if_linked(&pi->threads[i]); + if (root_item == pi) + root_item = NULL; + if (linked) + list_del_init(&pi->sibling); + if (pid_inserted) + pstree_remove_pid_if_linked(pi->pid); + pi->pid->state = TASK_UNDEF; + if (threads_allocated) { + xfree(pi->threads); + pi->threads = NULL; + pi->nr_threads = 0; + } + /* + * Restore pstree items come from the shared linear arena. The + * item may no longer be the last allocation after read_pstree_ids(), + * so it cannot be released individually. Restore teardown reclaims + * the arena after this parse failure. + */ + } return ret; } @@ -1262,8 +1341,12 @@ static int new_pid_ns_truncate_pid_hierarchy(pid_t *pid_max) unsigned int ns_level_to_truncate; clone_flags = get_clone_mask(root_item->ids, root_ids); - if (!(clone_flags & CLONE_NEWPID)) + if (!(clone_flags & CLONE_NEWPID) && + !(opts.tfork.active && root_item->pid->ns_level > 1)) return 0; + if (!(clone_flags & CLONE_NEWPID)) + pr_info("pidns: forcing tfork pid hierarchy truncation for root level=%d\n", + root_item->pid->ns_level); if (root_item->pid->ns_level <= 1) { pr_err("only 1 level of pid namespace, but CLONE_NEWPID is set, " @@ -1272,6 +1355,8 @@ static int new_pid_ns_truncate_pid_hierarchy(pid_t *pid_max) } ns_level_to_truncate = root_item->pid->ns_level - 1; + pr_info("pidns: truncating %u outer pid namespace level(s) for new root pid namespace\n", + ns_level_to_truncate); for (node = rb_first(&pid_root_rb[ALL_PID_NS_ID]); node; ) { next = rb_next(node); @@ -1279,9 +1364,15 @@ static int new_pid_ns_truncate_pid_hierarchy(pid_t *pid_max) pid_node = rb_entry(node, struct pid, root_ns_node); rb_erase(node, &pid_root_rb[ALL_PID_NS_ID]); + pr_info("pidns: truncate before uid=%d real=%d local=%d level=%d\n", + pid_node->uid, pid_node->real, pid_node->local, + pid_node->ns_level); pid_node->ns_level -= ns_level_to_truncate; BUG_ON(pid_node->ns_level <= 0); pid_node->real = pid_node->ns[pid_node->ns_level - 1].ns_pid; + pr_info("pidns: truncate after uid=%d real=%d local=%d level=%d\n", + pid_node->uid, pid_node->real, pid_node->local, + pid_node->ns_level); found = __lookup_pid_root(&new_real_rbtree, pid_node->real, &parent, &link); if (found) { diff --git a/criu/criu/seize.c b/criu/criu/seize.c index f5cde74e9..db8d95b92 100644 --- a/criu/criu/seize.c +++ b/criu/criu/seize.c @@ -379,7 +379,7 @@ static int seize_cgroup_tree(char *root_path, enum freezer_state state) */ static int freezer_wait_processes(void) { - int i; + int i, collected = 0; processes_to_wait_pids = xmalloc(sizeof(pid_t) * processes_to_wait); if (processes_to_wait_pids == NULL) @@ -388,23 +388,42 @@ static int freezer_wait_processes(void) for (i = 0; i < processes_to_wait; i++) { int status; pid_t pid; + int waited_ms = 0; /* * Here we are going to skip tasks which are already traced. * Ptraced tasks looks like children for us, so if * a task isn't ptraced yet, waitpid() will return a error. */ - pid = waitpid(-1, &status, 0); - if (pid < 0) { - pr_perror("Unable to wait processes"); - xfree(processes_to_wait_pids); - processes_to_wait_pids = NULL; - return -1; + while (1) { + pid = waitpid(-1, &status, opts.tfork.active ? WNOHANG : 0); + if (pid > 0) + break; + if (opts.tfork.active && pid < 0 && errno == EINTR && waited_ms < 500) { + usleep(10 * 1000); + waited_ms += 10; + continue; + } + if (!opts.tfork.active || (pid < 0 && errno != ECHILD && errno != EINTR)) { + pr_perror("Unable to wait processes"); + xfree(processes_to_wait_pids); + processes_to_wait_pids = NULL; + return -1; + } + if (pid < 0 || waited_ms >= 500) { + pr_warn("tfork: collected %d/%d unexpected freezer processes; continuing\n", + collected, processes_to_wait); + processes_to_wait = collected; + return 0; + } + usleep(10 * 1000); + waited_ms += 10; } pr_warn("Unexpected process %d in the freezer cgroup (status 0x%x)\n", pid, status); - processes_to_wait_pids[i] = pid; + processes_to_wait_pids[collected++] = pid; } + processes_to_wait = collected; return 0; } diff --git a/criu/criu/unittest/mock.c b/criu/criu/unittest/mock.c index b2d507278..b9601fffa 100644 --- a/criu/criu/unittest/mock.c +++ b/criu/criu/unittest/mock.c @@ -97,6 +97,10 @@ int close_service_fd(int type) return 0; } +void invalidate_proc_self_fd(void) +{ +} + void compel_log_init(int log_fn, unsigned int level) { } diff --git a/criu/criu/unittest/unit.c b/criu/criu/unittest/unit.c index 54769e6f2..cf9d79b24 100644 --- a/criu/criu/unittest/unit.c +++ b/criu/criu/unittest/unit.c @@ -3,19 +3,41 @@ #include #include "log.h" +#include "pstree.h" #include "util.h" #include "criu-log.h" int parse_statement(int i, char *line, char **configuration); +atomic_t pid_uid_generator = ATOMIC_INIT(0); + int main(int argc, char *argv[], char *envp[]) { char **configuration; + struct pid first_pid = {}; + struct pid second_pid = {}; + struct pstree_item first_item = { .pid = &first_pid }; + struct pstree_item second_item = { .pid = &second_pid }; int i; configuration = malloc(10 * sizeof(char *)); log_init(NULL); + pid_init_dump(&first_pid, &first_item); + pid_init_dump(&second_pid, &second_item); + assert(first_pid.uid > 0); + assert(second_pid.uid > first_pid.uid); + assert(first_pid.item == &first_item); + assert(first_pid.real == -1); + assert(first_pid.local == -1); + assert(first_pid.state == TASK_UNDEF); + assert(first_pid.stop_signo == -1); + assert(first_pid.ns_level == -1); + assert(first_pid.leaf_ns_id == ALL_PID_NS_ID); + assert(RB_EMPTY_NODE(&first_pid.leaf_ns_node)); + assert(RB_EMPTY_NODE(&first_pid.root_ns_node)); + assert(RB_EMPTY_NODE(&first_pid.uid_node)); + i = parse_statement(0, "", configuration); assert(i == 0); diff --git a/criu/lib/pycriu/images/images.py b/criu/lib/pycriu/images/images.py index 9db506e1e..3d2e02777 100644 --- a/criu/lib/pycriu/images/images.py +++ b/criu/lib/pycriu/images/images.py @@ -43,6 +43,8 @@ import os import array +from google.protobuf.message import DecodeError + from . import magic from . import pb from . import pb2dict @@ -190,6 +192,98 @@ def count(self, f): return entries +class pstree_handler: + """Read both legacy per-task and current file-level PSTREE images.""" + + @staticmethod + def _read_payload(f): + header = f.read(4) + if not header: + return None + if len(header) != 4: + raise ValueError("truncated PSTREE entry header") + + size, = struct.unpack('i', header) + if size < 0: + raise ValueError("negative PSTREE entry size") + + payload = f.read(size) + if len(payload) != size: + raise ValueError("truncated PSTREE entry payload") + return payload + + @staticmethod + def _parse(payload, message_type): + message = message_type() + try: + message.ParseFromString(payload) + except DecodeError: + return None + return message if message.IsInitialized() else None + + def load(self, f, pretty=False, no_payload=False): + # PSTREE has no out-of-band EXTRA data, so no_payload has no effect. + payload = self._read_payload(f) + if payload is None: + return [] + + # Format detection intentionally relies on pstree_entry being proto2 + # with required fields. A file-level payload may parse as that message, + # but it cannot be initialized because its wire fields have other types. + legacy = self._parse(payload, pb.pstree_entry) + if legacy is not None: + entries = [legacy] + while True: + payload = self._read_payload(f) + if payload is None: + break + entry = self._parse(payload, pb.pstree_entry) + if entry is None: + raise ValueError("invalid legacy PSTREE entry") + entries.append(entry) + return [pb2dict.pb2dict(entry, pretty) for entry in entries] + + entry = self._parse(payload, pb.pstree_file_entry) + if entry is None: + raise ValueError("invalid PSTREE entry") + if self._read_payload(f) is not None: + raise ValueError("file-level PSTREE image has multiple entries") + + return [pb2dict.pb2dict(entry, pretty)] + + def loads(self, data, pretty=False): + return self.load(io.BytesIO(data), pretty) + + def dump(self, entries, f): + if not entries: + return + + file_level = ('tree' in entries[0] or + 'ns_max_pids' in entries[0]) + if file_level and len(entries) != 1: + raise ValueError("file-level PSTREE image requires one entry") + + message_type = (pb.pstree_file_entry if file_level + else pb.pstree_entry) + for entry in entries: + message = message_type() + pb2dict.dict2pb(entry, message) + payload = message.SerializeToString() + f.write(struct.pack('i', len(payload))) + f.write(payload) + + def dumps(self, entries): + f = io.BytesIO() + self.dump(entries, f) + return f.getvalue() + + def count(self, f): + entries = 0 + while self._read_payload(f) is not None: + entries += 1 + return entries + + # Special handler for pagemap.img class pagemap_handler: """ @@ -502,7 +596,7 @@ def skip(self, f, pbuff): tcp_stream_extra_handler()), 'STATS': entry_handler(pb.stats_entry), 'PAGEMAP': pagemap_handler(), # Special one - 'PSTREE': entry_handler(pb.pstree_entry), + 'PSTREE': pstree_handler(), 'REG_FILES': entry_handler(pb.reg_file_entry), 'NS_FILES': entry_handler(pb.ns_file_entry), 'EVENTFD_FILE': entry_handler(pb.eventfd_file_entry), diff --git a/criu/test/others/pycriu/Makefile b/criu/test/others/pycriu/Makefile index b6e3b4814..369ac5667 100644 --- a/criu/test/others/pycriu/Makefile +++ b/criu/test/others/pycriu/Makefile @@ -13,7 +13,8 @@ CRIU_SOCKET := $(BUILD_DIR)/$(SOCKET_NAME) STATUS_FIFO := $(BUILD_DIR)/startup.status STATUS_FD := 200 -run: start +run: pstree-compat + $(MAKE) --no-print-directory start cleanup() { $(MAKE) --no-print-directory stop || true; } trap cleanup EXIT INT TERM "$(PYTHON)" test_check.py @@ -21,6 +22,9 @@ run: start "$(PYTHON)" test_check_images_dir.py "$(PYTHON)" test_check_work_dir_fd.py +pstree-compat: + "$(PYTHON)" test_pstree_compat.py + start: mkdir -p "$(BUILD_DIR)" if [ -s "$(PIDFILE)" ] && kill -0 "$$(cat "$(PIDFILE)")" 2>/dev/null; then @@ -60,4 +64,4 @@ clean: fi rm -rf "$(BUILD_DIR)" -.PHONY: start stop clean run \ No newline at end of file +.PHONY: start stop clean run pstree-compat diff --git a/criu/test/others/pycriu/test_pstree_compat.py b/criu/test/others/pycriu/test_pstree_compat.py new file mode 100644 index 000000000..5cda76e13 --- /dev/null +++ b/criu/test/others/pycriu/test_pstree_compat.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 +import io +import os +import struct +import sys + +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +LIB_DIR = os.path.normpath(os.path.join(SCRIPT_DIR, "../../../lib")) +if LIB_DIR not in sys.path: + sys.path.insert(0, LIB_DIR) + +from pycriu.images import images, pb # noqa: E402 + + +def encode(*messages): + output = io.BytesIO() + for message in messages: + payload = message.SerializeToString() + output.write(struct.pack('i', len(payload))) + output.write(payload) + return output.getvalue() + + +def task(message, realpid, localpid, uid): + message.realpid = realpid + message.ppid = 0 + message.pgid = localpid + message.sid = localpid + message.nsid = 7 + message.localpid = localpid + message.uid = uid + + +def main(): + handler = images.handlers['PSTREE'] + + old_first = pb.pstree_entry() + task(old_first, 1001, 1, 11) + old_second = pb.pstree_entry() + task(old_second, 1002, 2, 12) + old_blob = encode(old_first, old_second) + old_entries = handler.loads(old_blob) + assert [entry['realpid'] for entry in old_entries] == [1001, 1002] + assert handler.loads(handler.dumps(old_entries)) == old_entries + assert handler.count(io.BytesIO(old_blob)) == 2 + + current = pb.pstree_file_entry() + ns_max = current.ns_max_pids.add() + ns_max.ns_id = 7 + ns_max.pid_max = 2 + task(current.tree.add(), 1001, 1, 11) + task(current.tree.add(), 1002, 2, 12) + current_blob = encode(current) + current_entries = handler.loads(current_blob) + assert len(current_entries) == 1 + assert [entry['realpid'] for entry in current_entries[0]['tree']] == [1001, 1002] + assert handler.loads(handler.dumps(current_entries)) == current_entries + assert handler.count(io.BytesIO(current_blob)) == 1 + + print("PASS") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/crun/src/libcrun/criu.c b/crun/src/libcrun/criu.c index 8bf068d03..20c7bdd80 100644 --- a/crun/src/libcrun/criu.c +++ b/crun/src/libcrun/criu.c @@ -46,6 +46,8 @@ # define DESCRIPTORS_FILENAME "descriptors.json" # define CRIU_RUNC_CONFIG_FILE "/etc/criu/runc.conf" # define CRIU_CRUN_CONFIG_FILE "/etc/criu/crun.conf" +# define CRIU_LOG_TAIL_LINES 80 +# define CRIU_LOG_LINE_SIZE 1024 # define CRIU_EXT_NETNS "extRootNetNS" # define CRIU_EXT_PIDNS "extRootPidNS" @@ -543,8 +545,11 @@ static void show_criu_log (const char *work_path, const char *log) { cleanup_free char *log_path = NULL; + cleanup_free char *tail = NULL; libcrun_error_t *tmp_err = NULL; - char line[1024]; + char line[CRIU_LOG_LINE_SIZE]; + size_t tail_index = 0; + size_t tail_count = 0; FILE *f; if (UNLIKELY (append_paths (&log_path, tmp_err, work_path, log, NULL)) < 0) @@ -563,12 +568,43 @@ show_criu_log (const char *work_path, const char *log) /* Log with error verbosity as this is the default. */ libcrun_error (0, "--- excerpt from CRIU log `%s`", log_path); + tail = calloc (CRIU_LOG_TAIL_LINES, CRIU_LOG_LINE_SIZE); + if (tail == NULL) + { + fclose (f); + return; + } + while (fgets (line, sizeof (line), f) != NULL) - if (strstr (line, "Error ") != NULL) - { - line[strcspn (line, "\n")] = '\0'; - libcrun_error (0, "%s", line); - } + { + char *slot = tail + tail_index * CRIU_LOG_LINE_SIZE; + strncpy (slot, line, CRIU_LOG_LINE_SIZE - 1); + slot[CRIU_LOG_LINE_SIZE - 1] = '\0'; + tail_index = (tail_index + 1) % CRIU_LOG_TAIL_LINES; + if (tail_count < CRIU_LOG_TAIL_LINES) + tail_count++; + + if (strstr (line, "Error ") != NULL + || strstr (line, "failed") != NULL || strstr (line, "FAILED") != NULL + || strstr (line, "Unable") != NULL || strstr (line, "Can't") != NULL + || strstr (line, "No such") != NULL) + { + line[strcspn (line, "\n")] = '\0'; + libcrun_error (0, "%s", line); + } + } + + if (tail_count > 0) + { + size_t start = (tail_count == CRIU_LOG_TAIL_LINES) ? tail_index : 0; + libcrun_error (0, "--- last %zu CRIU log lines", tail_count); + for (size_t i = 0; i < tail_count; i++) + { + char *entry = tail + ((start + i) % CRIU_LOG_TAIL_LINES) * CRIU_LOG_LINE_SIZE; + entry[strcspn (entry, "\n")] = '\0'; + libcrun_error (0, "%s", entry); + } + } fclose (f); libcrun_error (0, "--- end of excerpt"); @@ -1351,14 +1387,33 @@ libcrun_container_restore_linux_criu (libcrun_container_status_t *status, libcru } # define CRIU_TFORK_LOG_FILE "tfork.log" +# define CRIU_TFORK_RESTORE_LOG_FILE "tfork-restore.log" +# define CRIU_TFORK_MAX_COPY_LOGS 16 + +static void +show_criu_tfork_restore_copy_logs (const char *image_path, int copy_count) +{ + if (copy_count < 0) + copy_count = 0; + if (copy_count > CRIU_TFORK_MAX_COPY_LOGS) + copy_count = CRIU_TFORK_MAX_COPY_LOGS; + + for (int i = 0; i < copy_count; i++) + { + char log[64]; + snprintf (log, sizeof (log), "%s.copy%d", CRIU_TFORK_RESTORE_LOG_FILE, i); + show_criu_log (image_path, log); + } +} static int -read_source_state_pid (const char *path, pid_t *pid_out, libcrun_error_t *err) +read_source_state_pid_cgroup (const char *path, pid_t *pid_out, char **cgroup_path_out, libcrun_error_t *err) { cleanup_free char *buffer = NULL; char err_buffer[256]; yajl_val tree, tmp; const char *pid_path[] = { "pid", NULL }; + const char *cgroup_path[] = { "cgroup-path", NULL }; int ret; ret = read_all_file (path, &buffer, NULL, err); @@ -1377,6 +1432,15 @@ read_source_state_pid (const char *path, pid_t *pid_out, libcrun_error_t *err) } *pid_out = (pid_t) strtoull (YAJL_GET_NUMBER (tmp), NULL, 10); + + tmp = yajl_tree_get (tree, cgroup_path, yajl_t_string); + if (UNLIKELY (tmp == NULL)) + { + yajl_tree_free (tree); + return crun_make_error (err, 0, "`cgroup-path` missing in source state.json `%s`", path); + } + + *cgroup_path_out = xstrdup (YAJL_GET_STRING (tmp)); yajl_tree_free (tree); return 0; } @@ -1439,10 +1503,13 @@ libcrun_container_tfork_linux_criu (libcrun_container_t *container, libcrun_chec { runtime_spec_schema_config_schema *def = container->container_def; cleanup_wrapper struct libcriu_wrapper_s *wrapper = NULL; + cleanup_free char *freezer_path = NULL; cleanup_free char *rootfs_path = NULL; + cleanup_free char *source_cgroup_path = NULL; cleanup_close int image_fd = -1; cleanup_close int work_fd = -1; pid_t source_pid = 0; + int cgroup_mode; int ret; ret = load_wrapper (&wrapper, err); @@ -1461,7 +1528,7 @@ libcrun_container_tfork_linux_criu (libcrun_container_t *container, libcrun_chec return crun_make_error (err, 0, "--tfork-snap-root, --tfork-snap-roots, or --tfork-copy=::--tfork-snap-root=PATH is required"); - if (cr_options->tfork_copies > 1 && cr_options->tfork_snap_roots_n > 0 + if (cr_options->tfork_copies >= 1 && cr_options->tfork_snap_roots_n > 0 && (size_t) cr_options->tfork_copies != cr_options->tfork_snap_roots_n) return crun_make_error (err, 0, "--tfork-copies=%d but --tfork-snap-roots has %zu entries", @@ -1470,7 +1537,7 @@ libcrun_container_tfork_linux_criu (libcrun_container_t *container, libcrun_chec if (UNLIKELY (cr_options->image_path == NULL)) return crun_make_error (err, 0, "--image-path is required"); - ret = read_source_state_pid (cr_options->source_state, &source_pid, err); + ret = read_source_state_pid_cgroup (cr_options->source_state, &source_pid, &source_cgroup_path, err); if (UNLIKELY (ret < 0)) return ret; if (UNLIKELY (source_pid <= 0)) @@ -1514,8 +1581,24 @@ libcrun_container_tfork_linux_criu (libcrun_container_t *container, libcrun_chec libcriu_wrapper->criu_set_pid (source_pid); libcriu_wrapper->criu_set_leave_running (true); + libcriu_wrapper->criu_set_ext_unix_sk (cr_options->ext_unix_sk); libcriu_wrapper->criu_set_file_locks (true); + cgroup_mode = libcrun_get_cgroup_mode (err); + if (UNLIKELY (cgroup_mode < 0)) + return cgroup_mode; + + if (cgroup_mode == CGROUP_MODE_UNIFIED) + ret = append_paths (&freezer_path, err, CGROUP_ROOT, source_cgroup_path, NULL); + else + ret = append_paths (&freezer_path, err, CGROUP_ROOT "/freezer", source_cgroup_path, NULL); + if (UNLIKELY (ret < 0)) + return ret; + + ret = libcriu_wrapper->criu_set_freeze_cgroup (freezer_path); + if (UNLIKELY (ret < 0)) + return crun_make_error (err, -ret, "CRIU: failed setting tfork freezer %d", ret); + if (def->root != NULL && def->root->path != NULL) { ret = append_paths (&rootfs_path, err, container->context ? container->context->bundle : ".", @@ -1564,7 +1647,7 @@ libcrun_container_tfork_linux_criu (libcrun_container_t *container, libcrun_chec } } - if (cr_options->tfork_copies > 1) + if (cr_options->tfork_copies >= 1) libcriu_wrapper->criu_set_tfork_copies (cr_options->tfork_copies); if (cr_options->tfork_memdump_async) @@ -1781,6 +1864,8 @@ libcrun_container_tfork_linux_criu (libcrun_container_t *container, libcrun_chec if (UNLIKELY (ret != 0)) { show_criu_log (cr_options->work_path, CRIU_TFORK_LOG_FILE); + show_criu_log (cr_options->image_path, CRIU_TFORK_RESTORE_LOG_FILE); + show_criu_tfork_restore_copy_logs (cr_options->image_path, cr_options->tfork_copies); return crun_make_error (err, 0, "criu_tfork failed: %d", ret); } @@ -1791,7 +1876,7 @@ libcrun_container_tfork_linux_criu (libcrun_container_t *container, libcrun_chec pid_t clone_pid; const char *pidfile_name = "tfork.pid"; - if (cr_options->tfork_copies > 1) + if (cr_options->tfork_copies >= 1) pidfile_name = "tfork.pid.copy0"; ret = append_paths (&pidfile_path, err, cr_options->image_path, pidfile_name, NULL); @@ -1806,7 +1891,7 @@ libcrun_container_tfork_linux_criu (libcrun_container_t *container, libcrun_chec if (UNLIKELY (clone_pid <= 0)) return crun_make_error (err, 0, "invalid clone PID %d in `%s`", (int) clone_pid, pidfile_path); - if (cr_options->tfork_copies > 1) + if (cr_options->tfork_copies >= 1) { char children_path[64]; cleanup_free char *children_buf = NULL; diff --git a/crun/src/tfork.c b/crun/src/tfork.c index 31db969ef..3a64b2bb9 100644 --- a/crun/src/tfork.c +++ b/crun/src/tfork.c @@ -83,7 +83,7 @@ static struct argp_option options[] { "parent-path", OPTION_PARENT_PATH, "DIR", 0, "previous criu images dir, for incremental memdump chains", 0 }, { "tfork-memdump", OPTION_TFORK_MEMDUMP, 0, 0, "dump pages-*.img to image-path during tfork", 0 }, { "tfork-memdump-async", OPTION_TFORK_MEMDUMP_ASYNC, 0, 0, "async page dump (implies --tfork-memdump)", 0 }, - { "tfork-copies", OPTION_TFORK_COPIES, "N", 0, "produce N parallel clones (default 1)", 0 }, + { "tfork-copies", OPTION_TFORK_COPIES, "N", 0, "produce N clones through the n-copy helper (omitted: legacy direct single-copy)", 0 }, { "track-mem", OPTION_TRACK_MEM, 0, 0, "arm soft-dirty for chained incremental dumps", 0 }, { "manage-cgroups-mode", OPTION_MANAGE_CGROUPS_MODE, "MODE", 0, "cgroups mode: 'soft' (default), 'ignore', 'full' and 'strict'", 0 }, @@ -359,7 +359,14 @@ int crun_command_tfork (struct crun_global_arguments *global_args, int argc, char **argv, libcrun_error_t *err) { cr_options.manage_cgroups_mode = -1; - cr_options.tfork_copies = 1; + cr_options.tfork_copies = 0; + /* + * External Unix stream sockets can make Codex/tmux stacks dumpable, but they + * may hide unsupported socket topology. Keep the default fail-loud and expose + * this as an explicit escape hatch for agent integrations. + */ + if (getenv ("CRUN_TFORK_EXT_UNIX_SK") != NULL) + cr_options.ext_unix_sk = true; cr_options.leave_running = true; return crun_run_create_internal (global_args, argc, argv, container_tfork, get_options, &crun_context, &run_argp, diff --git a/linux-pagecache-cow/build_kernel.sh b/linux-pagecache-cow/build_kernel.sh index c36ce0a59..94bbaeefd 100755 --- a/linux-pagecache-cow/build_kernel.sh +++ b/linux-pagecache-cow/build_kernel.sh @@ -9,6 +9,7 @@ version="7.0.1" # Append a suffix LocalVersion="-pgcachecow" num_cores=$(($(nproc --all) - 2)) +num_cores=$(( num_cores > 1 ? num_cores : 1 )) ## Functions delete_old_kernel_contents () { diff --git a/linux-pagecache-cow/mm/filecow.c b/linux-pagecache-cow/mm/filecow.c index 136201a25..5ecf4739b 100644 --- a/linux-pagecache-cow/mm/filecow.c +++ b/linux-pagecache-cow/mm/filecow.c @@ -42,6 +42,11 @@ EXPORT_SYMBOL_GPL(filecow_stat_ra_order_layer_fallback); static atomic_long_t filecow_stat_lookup_install = ATOMIC_LONG_INIT(0); static atomic_long_t filecow_stat_lookup_miss = ATOMIC_LONG_INIT(0); static atomic_long_t filecow_stat_folios_unaccounted = ATOMIC_LONG_INIT(0); +static atomic_long_t filecow_stat_layers_allocated = ATOMIC_LONG_INIT(0); +static atomic_long_t filecow_stat_layers_freed = ATOMIC_LONG_INIT(0); +static atomic_long_t filecow_stat_layers_active = ATOMIC_LONG_INIT(0); +static atomic_long_t filecow_stat_fork_no_layer = ATOMIC_LONG_INIT(0); +static atomic_long_t filecow_stat_fork_reused_layer = ATOMIC_LONG_INIT(0); static atomic_long_t filecow_diag_evict_ok = ATOMIC_LONG_INIT(0); static atomic_long_t filecow_diag_evict_refuse = ATOMIC_LONG_INIT(0); static atomic_long_t filecow_diag_evict_skipped = ATOMIC_LONG_INIT(0); @@ -90,6 +95,16 @@ static int filecow_stats_show(struct seq_file *m, void *v) atomic_long_read(&filecow_stat_ra_order_layer_fallback)); seq_printf(m, "folios_unaccounted %ld\n", atomic_long_read(&filecow_stat_folios_unaccounted)); + seq_printf(m, "layers_allocated %ld\n", + atomic_long_read(&filecow_stat_layers_allocated)); + seq_printf(m, "layers_freed %ld\n", + atomic_long_read(&filecow_stat_layers_freed)); + seq_printf(m, "layers_active %ld\n", + atomic_long_read(&filecow_stat_layers_active)); + seq_printf(m, "fork_no_layer %ld\n", + atomic_long_read(&filecow_stat_fork_no_layer)); + seq_printf(m, "fork_reused_layer %ld\n", + atomic_long_read(&filecow_stat_fork_reused_layer)); seq_printf(m, "diag_evict_ok %ld\n", atomic_long_read(&filecow_diag_evict_ok)); seq_printf(m, "diag_evict_refuse %ld\n", @@ -372,6 +387,8 @@ struct filecow_layer *filecow_layer_alloc(struct address_space *primary) INIT_LIST_HEAD(&layer->children); INIT_LIST_HEAD(&layer->sibling_link); layer->wb_err = 0; + atomic_long_inc(&filecow_stat_layers_allocated); + atomic_long_inc(&filecow_stat_layers_active); return layer; } @@ -380,6 +397,8 @@ static void __filecow_layer_free_rcu(struct rcu_head *head) struct filecow_layer *layer = container_of(head, struct filecow_layer, rcu); WARN_ON_ONCE(!list_empty(&layer->sharers)); WARN_ON_ONCE(!list_empty(&layer->children)); + atomic_long_inc(&filecow_stat_layers_freed); + atomic_long_dec(&filecow_stat_layers_active); kmem_cache_free(filecow_layer_cache, layer); } @@ -896,6 +915,44 @@ bool filecow_aggressive_evict(struct folio *folio) } EXPORT_SYMBOL_GPL(filecow_aggressive_evict); +/* + * Return true when @mapping contains state which is newer than mapping->ro + * and therefore must be captured in a new layer. Filecow folios in i_pages + * are only local lookup aliases for an existing layer and ordinary XArray + * values are reclaim metadata. A tombstone or a normal folio, however, + * changes what a descendant must observe. + * + * The caller holds the mapping invalidate lock for write. Together with the + * frozen source, this excludes fault, write, and truncate paths which could add + * private state. Reclaim can still replace an origin folio with a tombstone, + * so the population scan refreshes has_tombstone under the XArray lock before + * taking the no-new-layer fast path. + */ +static bool filecow_mapping_has_private_state(struct address_space *mapping, + bool *has_tombstone) +{ + struct folio *folio; + bool has_private = false; + XA_STATE(xas, &mapping->i_pages, 0); + + *has_tombstone = false; + xas_lock_irq(&xas); + xas_for_each(&xas, folio, ULONG_MAX) { + if (xas_retry(&xas, folio)) + continue; + if (xa_is_tombstone(folio)) { + *has_tombstone = true; + has_private = true; + continue; + } + if (!xa_is_value(folio) && !folio_test_filecow(folio)) + has_private = true; + } + xas_unlock_irq(&xas); + + return has_private; +} + int address_space_fork(struct address_space *new, struct address_space *source) { struct filecow_layer *L; @@ -904,6 +961,9 @@ int address_space_fork(struct address_space *new, struct address_space *source) unsigned long *share_bitmap = NULL; int n = 0, capacity, i, moved = 0; int ret = 0; + bool any_shareable = false; + bool has_private_state; + bool has_tombstone; XA_STATE(xas, &source->i_pages, 0); if (!READ_ONCE(sysctl_filecow_enabled)) @@ -944,10 +1004,136 @@ int address_space_fork(struct address_space *new, struct address_space *source) ret = -EBUSY; goto out_unlock; } + has_private_state = filecow_mapping_has_private_state(source, + &has_tombstone); + + /* + * A clean mapping without a filecow layer has no in-memory state to + * preserve. The filesystem snapshot already supplies the child's data, + * so creating an empty layer here only retains one unnecessary layer per + * inode in a long-lived source's generation chain. + */ + if (!source->ro && !has_private_state) { + atomic_long_inc(&filecow_stat_fork_no_layer); + goto out_unlock; + } + + /* + * When all local entries are aliases of the current immutable layer, + * attach the child to that layer directly. Repeated forks of an + * unchanged source then consume one sharer reference per live child, + * rather than permanently extending the layer chain. + */ + if (source->ro && !has_private_state) { + L = source->ro; + spin_lock(&L->sharers_lock); + refcount_inc(&L->refs); + new->ro = L; + list_add(&new->filecow_link, &L->sharers); + spin_unlock(&L->sharers_lock); + atomic_long_inc(&filecow_stat_fork_reused_layer); + goto out_unlock; + } + + capacity = source->nrpages; + if (capacity != 0) { + batch = kvmalloc_array(capacity, sizeof(*batch), + GFP_KERNEL | __GFP_NOWARN); + indices = kvmalloc_array(capacity, sizeof(*indices), + GFP_KERNEL | __GFP_NOWARN); + if (!batch || !indices) { + ret = -ENOMEM; + goto out_free_arrays; + } + + xas_lock_irq(&xas); + xas_for_each(&xas, folio, ULONG_MAX) { + if (xas_retry(&xas, folio)) + continue; + if (xa_is_tombstone(folio)) { + has_private_state = true; + has_tombstone = true; + continue; + } + if (xa_is_value(folio)) + continue; + if (folio_test_large(folio)) { + atomic_long_inc(&filecow_stat_large_seen); + atomic_long_add(folio_nr_pages(folio), + &filecow_stat_large_skipped); + continue; + } + + if (folio_test_filecow(folio)) + continue; + if (WARN_ON_ONCE(n >= capacity)) + continue; + batch[n] = folio; + indices[n] = xas.xa_index; + n++; + } + xas_unlock_irq(&xas); + + if (n > 0) { + share_bitmap = kvmalloc_array(BITS_TO_LONGS(n), + sizeof(long), + GFP_KERNEL | __GFP_ZERO); + if (!share_bitmap) { + /* + * Keep a generation under memory pressure. The + * population loop will check each folio once without + * a result bitmap. + */ + atomic_long_inc(&filecow_stat_perfolio_hook_used); + any_shareable = true; + } else if (source->a_ops && + source->a_ops->folio_extents_shared_bulk) { + source->a_ops->folio_extents_shared_bulk(source, new, + indices, n, + share_bitmap); + atomic_long_inc(&filecow_stat_bulk_hook_used); + any_shareable = !bitmap_empty(share_bitmap, n); + } else { + atomic_long_inc(&filecow_stat_perfolio_hook_used); + for (i = 0; i < n; i++) { + if (!can_share_folio(batch[i], source, new)) + continue; + any_shareable = true; + __set_bit(i, share_bitmap); + } + } + } + } + + /* + * Ordinary cached folios whose extents are not shared with the snapshot + * must remain private to the source. The snapshot already has their + * correct on-disk contents, so an empty filecow generation would retain + * memory without preserving any state. + */ + if (!has_tombstone && !any_shareable) { + kvfree(share_bitmap); + kvfree(batch); + kvfree(indices); + if (!source->ro) { + atomic_long_inc(&filecow_stat_fork_no_layer); + goto out_unlock; + } + + L = source->ro; + spin_lock(&L->sharers_lock); + refcount_inc(&L->refs); + new->ro = L; + list_add(&new->filecow_link, &L->sharers); + spin_unlock(&L->sharers_lock); + atomic_long_inc(&filecow_stat_fork_reused_layer); + goto out_unlock; + } + L = filecow_layer_alloc(source); if (!L) { ret = -ENOMEM; - goto out_unlock; + goto out_free_arrays; } L->below = source->ro; if (source->ro) { @@ -963,6 +1149,7 @@ int address_space_fork(struct address_space *new, struct address_space *source) list_for_each_entry(next_as, &L_old->sharers, filecow_link) { struct inode *cand = next_as->host; + if (cand && !(inode_state_read_once(cand) & (I_FREEING | I_WILL_FREE | I_CLEAR))) { new_primary = cand; @@ -973,54 +1160,6 @@ int address_space_fork(struct address_space *new, struct address_space *source) } spin_unlock(&L_old->sharers_lock); } - capacity = source->nrpages; - if (capacity == 0) - goto install; - - batch = kvmalloc_array(capacity, sizeof(*batch), - GFP_KERNEL | __GFP_NOWARN); - indices = kvmalloc_array(capacity, sizeof(*indices), - GFP_KERNEL | __GFP_NOWARN); - if (!batch || !indices) { - ret = -ENOMEM; - goto out_free_layer; - } - - xas_lock_irq(&xas); - xas_for_each(&xas, folio, ULONG_MAX) { - if (n >= capacity) - break; - if (xas_retry(&xas, folio)) - continue; - if (xa_is_value(folio)) - continue; - if (folio_test_large(folio)) { - atomic_long_inc(&filecow_stat_large_seen); - atomic_long_add(folio_nr_pages(folio), - &filecow_stat_large_skipped); - continue; - } - - if (folio_test_filecow(folio)) - continue; - batch[n] = folio; - indices[n] = xas.xa_index; - n++; - } - xas_unlock_irq(&xas); - - if (n > 0 && source->a_ops && - source->a_ops->folio_extents_shared_bulk) { - share_bitmap = kvmalloc(BITS_TO_LONGS(n) * sizeof(long), - GFP_KERNEL | __GFP_ZERO); - if (share_bitmap) { - source->a_ops->folio_extents_shared_bulk( - source, new, indices, n, share_bitmap); - atomic_long_inc(&filecow_stat_bulk_hook_used); - } - } - if (!share_bitmap) - atomic_long_inc(&filecow_stat_perfolio_hook_used); for (i = 0; i < n; i++) { void *prev; @@ -1059,7 +1198,6 @@ int address_space_fork(struct address_space *new, struct address_space *source) kvfree(batch); kvfree(indices); -install: spin_lock(&L->sharers_lock); refcount_set(&L->refs, 2 + moved); source->ro = L; @@ -1083,14 +1221,8 @@ int address_space_fork(struct address_space *new, struct address_space *source) } return 0; -out_free_layer: - if (L->below) { - spin_lock(&L->below->sharers_lock); - list_add(&source->filecow_link, &L->below->sharers); - spin_unlock(&L->below->sharers_lock); - } - - kmem_cache_free(filecow_layer_cache, L); +out_free_arrays: + kvfree(share_bitmap); kvfree(batch); kvfree(indices); out_unlock: diff --git a/linux-pagecache-cow/tools/testing/selftests/Makefile b/linux-pagecache-cow/tools/testing/selftests/Makefile index 450f13ba4..06b77806a 100644 --- a/linux-pagecache-cow/tools/testing/selftests/Makefile +++ b/linux-pagecache-cow/tools/testing/selftests/Makefile @@ -30,6 +30,7 @@ TARGETS += dt TARGETS += efivarfs TARGETS += exec TARGETS += fchmodat2 +TARGETS += filecow TARGETS += filesystems TARGETS += filesystems/binderfs TARGETS += filesystems/epoll diff --git a/linux-pagecache-cow/tools/testing/selftests/filecow/Makefile b/linux-pagecache-cow/tools/testing/selftests/filecow/Makefile new file mode 100644 index 000000000..1ebc86dd4 --- /dev/null +++ b/linux-pagecache-cow/tools/testing/selftests/filecow/Makefile @@ -0,0 +1,5 @@ +# SPDX-License-Identifier: GPL-2.0 + +TEST_PROGS := filecow_layer_lifetime.sh + +include ../lib.mk diff --git a/linux-pagecache-cow/tools/testing/selftests/filecow/filecow_layer_lifetime.sh b/linux-pagecache-cow/tools/testing/selftests/filecow/filecow_layer_lifetime.sh new file mode 100755 index 000000000..9eba6dbe4 --- /dev/null +++ b/linux-pagecache-cow/tools/testing/selftests/filecow/filecow_layer_lifetime.sh @@ -0,0 +1,108 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 + +set -euo pipefail + +readonly KSFT_SKIP=4 +readonly STATS=/proc/filecow_stats +readonly LOOPS="${FILECOW_TEST_LOOPS:-100}" +readonly TEST_ROOT="${FILECOW_TEST_ROOT:-}" + +skip() +{ + echo "1..0 # SKIP $*" + exit "$KSFT_SKIP" +} + +fail() +{ + echo "not ok 1 - filecow layer lifetime" + echo "# $*" + exit 1 +} + +stat_value() +{ + awk -v key="$1" '$1 == key { print $2; found = 1 } + END { if (!found) exit 1 }' "$STATS" +} + +[[ $EUID -eq 0 ]] || skip "must be run as root" +[[ -n "$TEST_ROOT" ]] || + skip "set FILECOW_TEST_ROOT to an existing btrfs directory" +[[ "$LOOPS" =~ ^[1-9][0-9]*$ ]] || skip "FILECOW_TEST_LOOPS must be positive" +[[ -r "$STATS" ]] || skip "$STATS is unavailable" +command -v btrfs >/dev/null || skip "btrfs-progs is unavailable" +[[ "$(findmnt -n -o FSTYPE -T "$TEST_ROOT")" == btrfs ]] || + skip "$TEST_ROOT is not on btrfs" + +for key in layers_allocated layers_active fork_no_layer fork_reused_layer; do + stat_value "$key" >/dev/null || + skip "running kernel does not expose the $key counter" +done + +workdir="$(mktemp -d "${TEST_ROOT%/}/filecow-layer-lifetime.XXXXXX")" +source_subvol="$workdir/source" +child_subvol="$workdir/child" + +cleanup() +{ + exec 8<&- 9<&- + if btrfs subvolume show "$child_subvol" >/dev/null 2>&1; then + btrfs subvolume delete "$child_subvol" >/dev/null + fi + if btrfs subvolume show "$source_subvol" >/dev/null 2>&1; then + btrfs subvolume delete "$source_subvol" >/dev/null + fi + rmdir "$workdir" 2>/dev/null || true +} +trap cleanup EXIT + +btrfs subvolume create "$source_subvol" >/dev/null +printf 'filecow-layer-lifetime\n' >"$source_subvol/cached" +dd if=/dev/zero bs=4096 count=16 status=none >>"$source_subvol/cached" +: >"$source_subvol/empty" +sync -f "$source_subvol/cached" + +# Populate one source folio. The first snapshot needs a layer for it; later +# unchanged snapshots must share that layer rather than extend the chain. +cat "$source_subvol/cached" >/dev/null +exec 8<"$source_subvol/cached" +exec 9<"$source_subvol/empty" + +allocated_before="$(stat_value layers_allocated)" +active_before="$(stat_value layers_active)" +no_layer_before="$(stat_value fork_no_layer)" +reused_before="$(stat_value fork_reused_layer)" + +for ((i = 0; i < LOOPS; i++)); do + btrfs subvolume snapshot "$source_subvol" "$child_subvol" >/dev/null + [[ "$(head -n 1 "$child_subvol/cached")" == filecow-layer-lifetime ]] || + fail "snapshot data mismatch in iteration $i" + stat "$child_subvol/empty" >/dev/null + btrfs subvolume delete "$child_subvol" >/dev/null +done + +allocated_after="$(stat_value layers_allocated)" +active_after="$(stat_value layers_active)" +no_layer_after="$(stat_value fork_no_layer)" +reused_after="$(stat_value fork_reused_layer)" + +allocated_delta=$((allocated_after - allocated_before)) +active_delta=$((active_after - active_before)) +no_layer_delta=$((no_layer_after - no_layer_before)) +reused_delta=$((reused_after - reused_before)) + +((allocated_delta <= 1)) || + fail "unchanged forks allocated $allocated_delta layers; expected at most 1" +((active_delta <= 1)) || + fail "unchanged forks retained $active_delta layers; expected at most 1" +((no_layer_delta >= LOOPS)) || + fail "empty inode skipped only $no_layer_delta layers; expected at least $LOOPS" +((reused_delta >= LOOPS - 1)) || + fail "cached inode reused only $reused_delta layers; expected at least $((LOOPS - 1))" + +echo "ok 1 - filecow layer lifetime" +echo "# loops=$LOOPS allocated_delta=$allocated_delta active_delta=$active_delta" +echo "# fork_no_layer_delta=$no_layer_delta fork_reused_layer_delta=$reused_delta" +echo "1..1" diff --git a/podman/pkg/domain/infra/abi/container_tfork.go b/podman/pkg/domain/infra/abi/container_tfork.go index 8aa6926d5..00543ddeb 100644 --- a/podman/pkg/domain/infra/abi/container_tfork.go +++ b/podman/pkg/domain/infra/abi/container_tfork.go @@ -9,6 +9,7 @@ import ( "os" "os/exec" "path/filepath" + "strconv" "strings" "syscall" "time" @@ -16,7 +17,6 @@ import ( "github.com/containers/podman/v5/libpod" "github.com/containers/podman/v5/libpod/define" - "strconv" "github.com/containers/podman/v5/pkg/domain/entities" "github.com/containers/podman/v5/utils" @@ -27,6 +27,27 @@ import ( "golang.org/x/sys/unix" ) +const ( + tforkSourceFreezeTimeout = 10 * time.Second + tforkSourceThawTimeout = 10 * time.Second + tforkCloneReadyTimeout = 60 * time.Second + tforkCgroupPollInterval = 50 * time.Millisecond + tforkClonePollInterval = 200 * time.Millisecond +) + +func tforkCloneReadyTimeoutFromEnv() time.Duration { + value := strings.TrimSpace(os.Getenv("PODMAN_TFORK_CLONE_READY_TIMEOUT_SECS")) + if value == "" { + return tforkCloneReadyTimeout + } + seconds, err := strconv.Atoi(value) + if err != nil || seconds <= 0 { + logrus.Warnf("tfork: ignoring invalid PODMAN_TFORK_CLONE_READY_TIMEOUT_SECS=%q", value) + return tforkCloneReadyTimeout + } + return time.Duration(seconds) * time.Second +} + func (ic *ContainerEngine) containerCloneLive(ctx context.Context, opts entities.ContainerCloneOptions) (rep *entities.ContainerCreateReport, retErr error) { src, err := ic.Libpod.LookupContainer(opts.ID) if err != nil { @@ -45,6 +66,14 @@ func (ic *ContainerEngine) containerCloneLive(ctx context.Context, opts entities if copies <= 0 { copies = 1 } + requestedCopies := copies + useSingleCopyConmon := requestedCopies == 1 && os.Getenv("PODMAN_TFORK_SINGLE_COPY_CONMON") == "1" + // PODMAN_TFORK_SINGLE_COPY_DIRECT is a debugging escape hatch that skips + // the n-copy restore helper for single-copy experiments. Production paths + // keep the n-copy helper even for copies=1 so attach/status handling is + // consistent with multi-copy forks. + useSingleCopyDirect := requestedCopies == 1 && os.Getenv("PODMAN_TFORK_SINGLE_COPY_DIRECT") == "1" + useNcopyRestore := copies > 1 || (requestedCopies == 1 && !useSingleCopyConmon && !useSingleCopyDirect) var srcRootfs string if cfg := src.Config(); cfg != nil && cfg.ExternalSetup && cfg.Rootfs != "" { @@ -69,6 +98,20 @@ func (ic *ContainerEngine) containerCloneLive(ctx context.Context, opts entities snapRO := filepath.Join(bundleDir, "snap-ro") + thawSource, err := tforkFreezeSourceCgroup(src, tforkSourceFreezeTimeout) + if err != nil { + return nil, fmt.Errorf("freeze source cgroup before tfork snapshot: %w", err) + } + sourceThawed := false + defer func() { + if sourceThawed { + return + } + if err := thawSource(); err != nil { + logrus.Warnf("tfork: thaw source cgroup after clone setup: %v", err) + } + }() + if out, err := exec.Command("sync").CombinedOutput(); err != nil { return nil, fmt.Errorf("sync: %s: %w", out, err) } @@ -280,7 +323,7 @@ func (ic *ContainerEngine) containerCloneLive(ctx context.Context, opts entities "--source-state", srcStatePath, "--image-path", imgDir, } - if copies == 1 { + if !useNcopyRestore { crunArgs = append(crunArgs, "--tfork-snap-root", cloneRootfsList[0]) crunArgs = append(crunArgs, "--tfork-snap-mount", "/") if cloneCgroupPaths[0] != "" { @@ -400,7 +443,9 @@ func (ic *ContainerEngine) containerCloneLive(ctx context.Context, opts entities } }() hasTTY := len(ttySrcFds) > 0 - useConmon := copies == 1 + // Keep the legacy single-copy conmon bootstrap opt-in because it can fail + // before crun starts and report only `conmon reported pid=-1`. + useConmon := copies == 1 && useSingleCopyConmon if useConmon { conmonInheritFds := inheritFds @@ -443,8 +488,8 @@ func (ic *ContainerEngine) containerCloneLive(ctx context.Context, opts entities var perCopyExtraFiles []*os.File var perCopyReadEnds []*os.File var perCopyArgs []string - skipTtySrcFds := copies > 1 && hasTTY - if copies > 1 { + skipTtySrcFds := useNcopyRestore && hasTTY + if useNcopyRestore { ifdsForPerCopy, stdioKeys := splitStdioInheritFds(inheritFds) inheritFds = ifdsForPerCopy extraFDBase := 3 @@ -497,6 +542,7 @@ func (ic *ContainerEngine) containerCloneLive(ctx context.Context, opts entities logF.Close() return nil, fmt.Errorf("start crun tfork: %w", err) } + crunDone := make(chan error, 1) var crunAborted bool defer func() { if !crunAborted && retErr != nil { @@ -507,7 +553,7 @@ func (ic *ContainerEngine) containerCloneLive(ctx context.Context, opts entities _ = f.Close() } go func() { - _ = crunCmd.Wait() + crunDone <- crunCmd.Wait() logF.Close() }() perCopyAttachSocks := make([]*os.File, copies) @@ -549,17 +595,27 @@ func (ic *ContainerEngine) containerCloneLive(ctx context.Context, opts entities } pidFileFor := func(i int) string { - if copies == 1 { + if !useNcopyRestore { return filepath.Join(imgDir, "tfork.pid") } return filepath.Join(imgDir, fmt.Sprintf("tfork.pid.copy%d", i)) } statePath := fmt.Sprintf("/run/crun/%s/status", cloneIDs[0]) - needState := copies == 1 - deadline := time.Now().Add(60 * time.Second) + needState := !useNcopyRestore + cloneReadyTimeout := tforkCloneReadyTimeoutFromEnv() + deadline := time.Now().Add(cloneReadyTimeout) readyCopies := 0 stateReady := !needState + crunExited := false + var crunErr error for readyCopies < copies || !stateReady { + if !crunExited { + select { + case crunErr = <-crunDone: + crunExited = true + default: + } + } readyCopies = 0 for i := 0; i < copies; i++ { if _, err := os.Stat(pidFileFor(i)); err == nil { @@ -574,7 +630,7 @@ func (ic *ContainerEngine) containerCloneLive(ctx context.Context, opts entities if readyCopies >= copies && stateReady { break } - if crunCmd.ProcessState != nil && crunCmd.ProcessState.Exited() && readyCopies < copies { + if crunExited && readyCopies < copies { tforkAbortCrunCmd(crunCmd, src, bundleDir, copies) crunAborted = true return nil, fmt.Errorf("crun tfork exited before %d clones came up (got %d); see %s", @@ -586,17 +642,41 @@ func (ic *ContainerEngine) containerCloneLive(ctx context.Context, opts entities return nil, fmt.Errorf("timeout waiting for %d tfork.pid* files in %s (got %d, stateReady=%v); see %s", copies, imgDir, readyCopies, stateReady, logPath) } - time.Sleep(200 * time.Millisecond) + time.Sleep(tforkClonePollInterval) + } + if !crunExited { + select { + case crunErr = <-crunDone: + crunExited = true + case <-time.After(cloneReadyTimeout): + tforkAbortCrunCmd(crunCmd, src, bundleDir, copies) + crunAborted = true + return nil, fmt.Errorf("timeout waiting %s for crun tfork to finish after %d clones came up; see %s", + cloneReadyTimeout, copies, logPath) + } + } + if crunErr != nil { + tforkAbortCrunCmd(crunCmd, src, bundleDir, copies) + crunAborted = true + return nil, fmt.Errorf("crun tfork failed after %d clones came up: %w; see %s", + copies, crunErr, logPath) } logrus.Infof("tfork: batch %s up (N=%d); crun-tfork.log at %s", batchID, copies, logPath) } + if err := thawSource(); err != nil { + logrus.Warnf("tfork: clones are up, but thawing source cgroup after restore failed: %v", err) + return nil, fmt.Errorf("thaw source cgroup after tfork restore: %w", err) + } + sourceThawed = true + srcCfg := src.Config() if srcCfg == nil { return nil, fmt.Errorf("source %q: could not read libpod config", src.ID()) } + visibleCloneIDs := make([]string, 0, requestedCopies) for i, cloneID := range cloneIDs { - clonePID, err := readTforkClonePID(cloneID, imgDir, i, copies) + clonePID, err := readTforkClonePID(cloneID, imgDir, i, copies, useNcopyRestore) if err != nil { return nil, fmt.Errorf("read clone %d PID: %w", i, err) } @@ -661,9 +741,184 @@ func (ic *ContainerEngine) containerCloneLive(ctx context.Context, opts entities } } logrus.Infof("tfork: clone %s (%s) registered in libpod state, pid=%d", cloneID, cloneNames[i], clonePID) + visibleCloneIDs = append(visibleCloneIDs, cloneID) } - return &entities.ContainerCreateReport{Id: strings.Join(cloneIDs, "\n")}, nil + return &entities.ContainerCreateReport{Id: strings.Join(visibleCloneIDs, "\n")}, nil +} + +type tforkCgroupFreezer struct { + root string + statePath string + eventsPath string + version string +} + +func tforkFreezeSourceCgroup(src *libpod.Container, timeout time.Duration) (func() error, error) { + if src == nil { + return nil, fmt.Errorf("source container is nil") + } + cgPath, err := src.CgroupPath() + if err != nil { + return nil, fmt.Errorf("read source cgroup path: %w", err) + } + cgPath = strings.TrimPrefix(filepath.Clean(cgPath), string(os.PathSeparator)) + if cgPath == "" || cgPath == "." { + return nil, fmt.Errorf("source cgroup path is empty") + } + freezer, err := tforkSourceCgroupFreezer(cgPath) + if err != nil { + return nil, err + } + + originalFrozen, err := freezer.frozen() + if err != nil { + return nil, err + } + if err := freezer.freeze(); err != nil { + return nil, err + } + if err := freezer.waitFrozen(true, timeout); err != nil { + if !originalFrozen { + _ = freezer.thaw() + } + return nil, err + } + logrus.Infof("tfork: froze source cgroup for snapshot consistency: %s (%s)", freezer.root, freezer.version) + + thaw := func() error { + if originalFrozen { + return freezer.freeze() + } + if err := freezer.thaw(); err != nil { + return err + } + if err := freezer.waitFrozen(false, tforkSourceThawTimeout); err != nil { + return err + } + logrus.Infof("tfork: thawed source cgroup after clone restore: %s (%s)", freezer.root, freezer.version) + return nil + } + return thaw, nil +} + +func tforkSourceCgroupFreezer(cgPath string) (*tforkCgroupFreezer, error) { + v2Root := filepath.Join("/sys/fs/cgroup", cgPath) + v2FreezePath := filepath.Join(v2Root, "cgroup.freeze") + v2EventsPath := filepath.Join(v2Root, "cgroup.events") + if _, err := os.Stat(v2FreezePath); err == nil { + return &tforkCgroupFreezer{ + root: v2Root, + statePath: v2FreezePath, + eventsPath: v2EventsPath, + version: "cgroup v2", + }, nil + } else if !os.IsNotExist(err) { + return nil, fmt.Errorf("stat %s: %w", v2FreezePath, err) + } + + v1Root := filepath.Join("/sys/fs/cgroup/freezer", cgPath) + v1StatePath := filepath.Join(v1Root, "freezer.state") + if _, err := os.Stat(v1StatePath); err == nil { + return &tforkCgroupFreezer{ + root: v1Root, + statePath: v1StatePath, + version: "cgroup v1 freezer", + }, nil + } else if !os.IsNotExist(err) { + return nil, fmt.Errorf("stat %s: %w", v1StatePath, err) + } + + return nil, fmt.Errorf("source cgroup freezer not found for %q; expected cgroup v2 %s or cgroup v1 %s", + cgPath, v2FreezePath, v1StatePath) +} + +func (f *tforkCgroupFreezer) frozen() (bool, error) { + data, err := os.ReadFile(f.statePath) + if err != nil { + return false, fmt.Errorf("read %s: %w", f.statePath, err) + } + frozen, ok := f.parseFrozen(data) + if !ok { + return false, fmt.Errorf("could not parse frozen state from %s", f.statePath) + } + return frozen, nil +} + +func (f *tforkCgroupFreezer) freeze() error { + value := []byte("1") + if f.version == "cgroup v1 freezer" { + value = []byte("FROZEN") + } + if err := os.WriteFile(f.statePath, value, 0o644); err != nil { + return fmt.Errorf("write %s=%s: %w", f.statePath, value, err) + } + return nil +} + +func (f *tforkCgroupFreezer) thaw() error { + value := []byte("0") + if f.version == "cgroup v1 freezer" { + value = []byte("THAWED") + } + if err := os.WriteFile(f.statePath, value, 0o644); err != nil { + return fmt.Errorf("write %s=%s: %w", f.statePath, value, err) + } + return nil +} + +func (f *tforkCgroupFreezer) waitFrozen(wantFrozen bool, timeout time.Duration) error { + deadline := time.Now().Add(timeout) + for { + path := f.statePath + if f.eventsPath != "" { + path = f.eventsPath + } + data, err := os.ReadFile(path) + if err != nil { + return fmt.Errorf("read %s: %w", path, err) + } + frozen, ok := f.parseFrozen(data) + if ok && frozen == wantFrozen { + return nil + } + if time.Now().After(deadline) { + return fmt.Errorf("timeout waiting %s for %s to report frozen=%t", timeout, path, wantFrozen) + } + time.Sleep(tforkCgroupPollInterval) + } +} + +func (f *tforkCgroupFreezer) parseFrozen(data []byte) (bool, bool) { + if f.version == "cgroup v1 freezer" { + switch strings.TrimSpace(string(data)) { + case "FROZEN": + return true, true + case "THAWED": + return false, true + } + return false, false + } + + switch strings.TrimSpace(string(data)) { + case "0": + return false, true + case "1": + return true, true + } + + for _, line := range strings.Split(string(data), "\n") { + fields := strings.Fields(line) + if len(fields) == 2 && fields[0] == "frozen" { + switch fields[1] { + case "0": + return false, true + case "1": + return true, true + } + } + } + return false, false } func spawnTforkDumpdHolder() (int, uint64, error) { @@ -1046,7 +1301,7 @@ func spawnTforkExitWatcher(ctx context.Context, rt *libpod.Runtime, ctr *libpod. return nil } -func readTforkClonePID(cloneID string, imgDir string, copyIdx, copies int) (int, error) { +func readTforkClonePID(cloneID string, imgDir string, copyIdx, copies int, useNcopyRestore bool) (int, error) { statePath := fmt.Sprintf("/run/crun/%s/status", cloneID) if data, err := os.ReadFile(statePath); err == nil { var st struct { @@ -1057,7 +1312,7 @@ func readTforkClonePID(cloneID string, imgDir string, copyIdx, copies int) (int, } } var pidFile string - if copies == 1 { + if !useNcopyRestore { pidFile = filepath.Join(imgDir, "tfork.pid") } else { pidFile = filepath.Join(imgDir, fmt.Sprintf("tfork.pid.copy%d", copyIdx)) @@ -1070,7 +1325,7 @@ func readTforkClonePID(cloneID string, imgDir string, copyIdx, copies int) (int, if err != nil { return 0, err } - if copies == 1 { + if !useNcopyRestore { return rcPID, nil } initPID, err := readFirstChildPID(rcPID) diff --git a/ubuntu-img/Dockerfile b/ubuntu-img/Dockerfile index 235528ac9..782c9193c 100644 --- a/ubuntu-img/Dockerfile +++ b/ubuntu-img/Dockerfile @@ -119,6 +119,7 @@ RUN \ libreoffice-style-breeze \ libreoffice-writer \ thunderbird \ + tmux \ ubuntu-wallpapers \ ubuntu-wallpapers-jammy \ vlc && \ diff --git a/ubuntu-img/docker-compose.yml b/ubuntu-img/docker-compose.yml index cf5768548..3dc4c708a 100644 --- a/ubuntu-img/docker-compose.yml +++ b/ubuntu-img/docker-compose.yml @@ -1,6 +1,6 @@ services: webtop: - image: ${IMAGE:-ghcr.io/wuklab/webtop:ubuntu-kde} + image: ${IMAGE:-localhost/gensee-tclone-webtop:tmux} container_name: webtop security_opt: - seccomp=unconfined