diff --git a/criu/build.sh b/criu/build.sh index cb1d9966c..cdb1cb472 100755 --- a/criu/build.sh +++ b/criu/build.sh @@ -93,12 +93,25 @@ for mod in "${ACTIVE_MODULES[@]}"; do fi if lsmod | awk '{print $1}' | grep -qx "${mod}"; then - echo " -- ${mod}: already loaded, rmmod then insmod" - rmmod "${mod}" 2>/dev/null || true - else - echo " -- ${mod}: insmod" + echo " -- ${mod}: already loaded, trying reload" + if ! rmmod "${mod}"; then + echo "build.sh: failed to unload loaded module ${mod}; aborting to avoid stale module" >&2 + echo "build.sh: stop running tfork/podman containers that may still hold ${mod}, then retry" >&2 + exit 1 + fi + if lsmod | awk '{print $1}' | grep -qx "${mod}"; then + echo "build.sh: module ${mod} is still loaded after rmmod; aborting to avoid stale module" >&2 + echo "build.sh: stop running tfork/podman containers that may still hold ${mod}, then retry" >&2 + exit 1 + fi + fi + + echo " -- ${mod}: insmod" + if ! insmod "${ko_path}"; then + echo "build.sh: failed to insert rebuilt module ${mod} from ${ko_path}" >&2 + echo "build.sh: if the old module is still active, stop running tfork/podman containers and rerun this script" >&2 + exit 1 fi - insmod "${ko_path}" done echo " -- verify all modules loaded" diff --git a/criu/criu/action-scripts.c b/criu/criu/action-scripts.c index 6f7900186..3742d863b 100644 --- a/criu/criu/action-scripts.c +++ b/criu/criu/action-scripts.c @@ -32,6 +32,9 @@ static const char *action_names[ACT_MAX] = { [ACT_ORPHAN_PTS_MASTER] = "orphan-pts-master", [ACT_STATUS_READY] = "status-ready", [ACT_QUERY_EXT_FILES] = "query-ext-files", + [ACT_POST_TFORK_FREEZE] = "post-tfork-freeze", + [ACT_PRE_TFORK_RESTORE] = "pre-tfork-restore", + [ACT_TFORK_SOURCE_DETACHED] = "tfork-source-detached", }; struct script { diff --git a/criu/criu/clone-noasan.c b/criu/criu/clone-noasan.c index 4ba7f6f31..368f6aa1e 100644 --- a/criu/criu/clone-noasan.c +++ b/criu/criu/clone-noasan.c @@ -78,7 +78,10 @@ int clone3_with_pid_noasan(int (*fn)(void *), void *arg, int flags, int exit_sig c_args.flags = flags; c_args.set_tid = ptr_to_u64(&pid); c_args.set_tid_size = 1; + pr_info("clone3 set_tid pid=%d flags=0x%x size=1\n", pid, flags); pid = syscall(__NR_clone3, &c_args, sizeof(c_args)); + if (pid < 0) + pr_perror("clone3 set_tid failed flags=0x%x size=1", flags); if (pid == 0) exit(fn(arg)); return pid; @@ -99,6 +102,12 @@ int clone3_with_nested_pid_noasan(int (*fn)(void *), void *arg, int flags, int e BUG_ON(pid->ns_level > MAX_PID_NS_LEVEL || pid->ns_level <= 1); for (i = 0; i < pid->ns_level; i++) tids[i] = pid->ns[i].ns_pid; + pr_info("clone3 nested set_tid flags=0x%x size=%d tids=%d/%d/%d/%d\n", + flags, pid->ns_level, + tids[0], + pid->ns_level > 1 ? tids[1] : -1, + pid->ns_level > 2 ? tids[2] : -1, + pid->ns_level > 3 ? tids[3] : -1); if (!(flags & CLONE_PARENT)) { if (exit_signal != SIGCHLD) { @@ -112,6 +121,13 @@ int clone3_with_nested_pid_noasan(int (*fn)(void *), void *arg, int flags, int e c_args.set_tid = ptr_to_u64(tids); c_args.set_tid_size = pid->ns_level; pid_ret = syscall(__NR_clone3, &c_args, sizeof(c_args)); + if (pid_ret < 0) + pr_perror("clone3 nested set_tid failed flags=0x%x size=%d tids=%d/%d/%d/%d", + flags, pid->ns_level, + tids[0], + pid->ns_level > 1 ? tids[1] : -1, + pid->ns_level > 2 ? tids[2] : -1, + pid->ns_level > 3 ? tids[3] : -1); if (pid_ret == 0) exit(fn(arg)); return pid_ret; diff --git a/criu/criu/cr-dump.c b/criu/criu/cr-dump.c index d196521fc..5179e5e53 100644 --- a/criu/criu/cr-dump.c +++ b/criu/criu/cr-dump.c @@ -885,6 +885,7 @@ static int collect_pstree_ids_predump(void) * write_img_inventory(). */ + pid_init_dump(crt.i.pid, &crt.i); crt.i.pid->state = TASK_ALIVE; crt.i.pid->real = getpid(); @@ -2360,6 +2361,9 @@ int cr_dump_tasks(pid_t pid) if (dump_zombies()) goto err; + if (finalize_nested_pid_ns_ids()) + goto err; + if (dump_pstree(root_item)) goto err; diff --git a/criu/criu/cr-restore.c b/criu/criu/cr-restore.c index d85279a82..354cfe965 100644 --- a/criu/criu/cr-restore.c +++ b/criu/criu/cr-restore.c @@ -185,10 +185,77 @@ static int __restore_wait_inprogress_tasks(int participants) { int ret; futex_t *np = &task_entries->nr_in_progress; + const int tfork_restore_wait_timeout_ms = 10000; + + if (opts.tfork.active) { + struct timespec started, now, timeout; + int wait_ret = 0; + + /* + * All paths that decrement this barrier use + * futex_dec_and_wake(). Waiting on the observed value avoids + * paying up to one 100ms polling interval at every restore stage. + */ + if (clock_gettime(CLOCK_MONOTONIC, &started)) { + pr_perror("tfork restore wait: clock_gettime"); + return -errno; + } + while ((int)futex_get(np) > participants) { + int64_t elapsed_ns, remaining_ns; + uint32_t observed = futex_get(np); + + if (observed & FUTEX_ABORT_FLAG) + break; + if (clock_gettime(CLOCK_MONOTONIC, &now)) { + pr_perror("tfork restore wait: clock_gettime"); + return -errno; + } + elapsed_ns = + (int64_t)(now.tv_sec - started.tv_sec) * NSEC_PER_SEC + + (now.tv_nsec - started.tv_nsec); + remaining_ns = + (int64_t)tfork_restore_wait_timeout_ms * 1000000 - + elapsed_ns; + if (remaining_ns <= 0) { + wait_ret = -ETIMEDOUT; + break; + } + timeout.tv_sec = remaining_ns / NSEC_PER_SEC; + timeout.tv_nsec = remaining_ns % NSEC_PER_SEC; + wait_ret = sys_futex( + (uint32_t *)&np->raw.counter, FUTEX_WAIT, + observed, &timeout, NULL, 0); + if (wait_ret == 0 || wait_ret == -EINTR || + wait_ret == -EWOULDBLOCK) + continue; + if (wait_ret == -ETIMEDOUT) + break; + pr_err("tfork restore futex wait failed: %d\n", wait_ret); + set_cr_errno(-wait_ret); + return wait_ret; + } + + if (wait_ret == -ETIMEDOUT && + (int)futex_get(np) > participants) { + pr_err("tfork restore wait timed out after %dms: participants=%d nr_in_progress=%d start_stage=%d task_cr_err=%d nr_tasks=%d nr_threads=%d nr_helpers=%d\n", + tfork_restore_wait_timeout_ms, + participants, (int)futex_get(np), + (int)futex_get(&task_entries->start), + get_task_cr_err(), + task_entries->nr_tasks, task_entries->nr_threads, + task_entries->nr_helpers); + set_cr_errno(ETIMEDOUT); + return -ETIMEDOUT; + } + } else { + futex_wait_while_gt(np, participants); + } - futex_wait_while_gt(np, participants); ret = (int)futex_get(np); if (ret < 0) { + pr_err("restore wait aborted: participants=%d nr_in_progress=%d start_stage=%d task_cr_err=%d\n", + participants, ret, (int)futex_get(&task_entries->start), + get_task_cr_err()); set_cr_errno(get_task_cr_err()); return ret; } @@ -227,6 +294,17 @@ static inline void __restore_switch_stage(int next_stage) static int restore_switch_stage(int next_stage) { + if (opts.tfork.active) + pr_warn("tfork: restore_switch_stage %d participants=%d nr_tasks=%d nr_threads=%d nr_helpers=%d\n", + next_stage, stage_participants(next_stage), + task_entries->nr_tasks, task_entries->nr_threads, + task_entries->nr_helpers); + else + pr_info("restore_switch_stage %d participants=%d nr_tasks=%d nr_threads=%d nr_helpers=%d\n", + next_stage, stage_participants(next_stage), + task_entries->nr_tasks, task_entries->nr_threads, + task_entries->nr_helpers); + __restore_switch_stage(next_stage); return restore_wait_inprogress_tasks(); } @@ -1303,14 +1381,15 @@ static int set_next_pid(void *arg) return 0; } -static inline int fork_with_pid(struct pstree_item *item) +static inline int fork_with_pid_mode(struct pstree_item *item, bool parallel_sibling) { struct cr_clone_arg ca; struct ns_id *pid_ns = NULL; bool external_pidns = false; + bool last_pid_locked = false; int ret = -1; pid_t pid = localpid(item); - unsigned long strip; + unsigned long strip, syscall_clone_flags; if (item->pid->state != TASK_HELPER) { if (open_core(uid(item), &ca.core)) @@ -1395,13 +1474,31 @@ static inline int fork_with_pid(struct pstree_item *item) ca.item = item; ca.clone_flags = rsti(item)->clone_flags; + if (opts.tfork.active && item != root_item && + !(ca.clone_flags & CLONE_NEWPID) && + item->pid->ns_level > 1 && + item->pid->ns[0].ns_pid == INIT_PID) { + pr_info("tfork: repairing missing CLONE_NEWPID for pidns init uid=%d local=%d parent_local=%d level=%d\n", + uid(item), pid, + item->parent ? localpid(item->parent) : -1, + item->pid->ns_level); + ca.clone_flags |= CLONE_NEWPID; + } BUG_ON(ca.clone_flags & CLONE_VM); pr_info("Forking task with %d(%d) (flags 0x%lx)\n", realpid(item), pid, ca.clone_flags); if (!(ca.clone_flags & CLONE_NEWPID)) { - lock_last_pid(); + /* + * clone3(set_tid) reserves the requested PID atomically. Parallel + * sibling helpers must not serialize on the legacy ns_last_pid lock; + * the lock remains mandatory for the fallback set_next_pid path. + */ + if (!parallel_sibling || !kdat.has_clone3_set_tid) { + lock_last_pid(); + last_pid_locked = true; + } if (!kdat.has_clone3_set_tid) { if (external_pidns) { @@ -1432,15 +1529,50 @@ static inline int fork_with_pid(struct pstree_item *item) strip = CLONE_NEWNET | CLONE_NEWCGROUP | CLONE_NEWTIME; if (!(item == root_item && is_simple_userns_tree())) strip |= CLONE_NEWUSER; + syscall_clone_flags = ca.clone_flags; + if (parallel_sibling) + syscall_clone_flags |= CLONE_PARENT; if (kdat.has_clone3_set_tid) { - if (item->pid->ns_level == 1) + if (opts.tfork.active && (syscall_clone_flags & CLONE_NEWPID)) { + pr_info("tfork: restore pidns init uid=%d local pid %d with fresh parent pid, dumped chain level=%d\n", + uid(item), pid, item->pid->ns_level); ret = clone3_with_pid_noasan(restore_task_with_children, &ca, - ca.clone_flags & ~strip, SIGCHLD, pid); - else + syscall_clone_flags & ~strip, SIGCHLD, pid); + } else if (item->pid->ns_level == 1) + ret = clone3_with_pid_noasan(restore_task_with_children, &ca, + syscall_clone_flags & ~strip, SIGCHLD, pid); + else { + struct pid tfork_pid = {}; + struct pid *restore_pid = item->pid; + + if (opts.tfork.active && (root_ns_mask & CLONE_NEWPID) && + root_item && root_item->pid->ns_level > 1 && + item->pid->ns_level > 1) { + /* + * Copy only scalar pid identity. struct pid also + * embeds rb_node links owned by the dumped pid trees; + * copying those nodes into a temporary stack object + * corrupts the tree metadata if it ever gets reused. + */ + tfork_pid.item = item->pid->item; + tfork_pid.real = item->pid->real; + tfork_pid.local = item->pid->local; + tfork_pid.uid = item->pid->uid; + tfork_pid.state = item->pid->state; + tfork_pid.stop_signo = item->pid->stop_signo; + tfork_pid.ns_level = item->pid->ns_level - 1; + tfork_pid.leaf_ns_id = item->pid->leaf_ns_id; + memcpy(tfork_pid.ns, item->pid->ns, sizeof(tfork_pid.ns)); + restore_pid = &tfork_pid; + pr_info("tfork: restore pid uid=%d local=%d with rebased pid chain level %d -> %d\n", + uid(item), pid, item->pid->ns_level, + restore_pid->ns_level); + } ret = clone3_with_nested_pid_noasan(restore_task_with_children, &ca, - ca.clone_flags & ~strip, - SIGCHLD, item->pid); + syscall_clone_flags & ~strip, + SIGCHLD, restore_pid); + } } else { BUG_ON(item->pid->ns_level >= 1); close_pid_proc(); @@ -1448,13 +1580,26 @@ static inline int fork_with_pid(struct pstree_item *item) (ca.clone_flags & ~strip) | SIGCHLD, &ca); } if (ret < 0) { + pr_err("fork_with_pid failed item uid=%d local=%d real=%d parent_local=%d flags=0x%lx stripped_flags=0x%lx ns_level=%d root_ns_mask=0x%lx tfork=%d\n", + uid(item), pid, realpid(item), + item->parent ? localpid(item->parent) : -1, + ca.clone_flags, ca.clone_flags & ~strip, + item->pid->ns_level, root_ns_mask, + opts.tfork.active ? 1 : 0); + if (item->pid->ns_level > 0) + pr_err("fork_with_pid pid chain uid=%d ns=%d/%d/%d/%d\n", + uid(item), + item->pid->ns[0].ns_pid, + item->pid->ns_level > 1 ? item->pid->ns[1].ns_pid : -1, + item->pid->ns_level > 2 ? item->pid->ns[2].ns_pid : -1, + item->pid->ns_level > 3 ? item->pid->ns[3].ns_pid : -1); pr_perror("Can't fork for %d", pid); if (errno == EEXIST) set_cr_errno(EEXIST); goto err_unlock; } - if (item == root_item) { + if (opts.tfork.active || item == root_item) { item->pid->real = ret; pr_debug("PID: real %d virt %d\n", item->pid->real, localpid(item)); } @@ -1462,7 +1607,7 @@ static inline int fork_with_pid(struct pstree_item *item) arch_shstk_unlock(item, ca.core, ret); err_unlock: - if (!(ca.clone_flags & CLONE_NEWPID)) + if (last_pid_locked) unlock_last_pid(); if (ca.core) @@ -1470,6 +1615,11 @@ static inline int fork_with_pid(struct pstree_item *item) return ret; } +static inline int fork_with_pid(struct pstree_item *item) +{ + return fork_with_pid_mode(item, false); +} + static pid_t userns_maps_helper_pid = -1; static bool item_parent_uns_is_host(struct pstree_item *item) @@ -1739,6 +1889,161 @@ static int mount_proc(void) return ret; } +#define TFORK_PARALLEL_SIBLING_MAX_WORKERS 16 +#define TFORK_PARALLEL_SIBLING_MIN_CHILDREN 32 + +struct tfork_parallel_sibling_arg { + int worker_index; + int worker_count; + bool before_setsid; +}; + +static int tfork_parallel_sibling_worker_count(void) +{ + const char *value; + char *end = NULL; + long workers; + + /* + * Restrict the prototype to one bounded helper pool for root children. + * Recursing into every wide subtree can multiply helpers without bound. + */ + if (!opts.tfork.active || current != root_item) + return 0; + value = getenv("CRIU_TFORK_PARALLEL_SIBLINGS"); + if (!value || !value[0]) + return 0; + workers = strtol(value, &end, 10); + if (!*end && workers == 0) + return 0; + if (*end || workers < 2) { + pr_warn("tfork: ignoring invalid CRIU_TFORK_PARALLEL_SIBLINGS=%s\n", value); + return 0; + } + if (workers > TFORK_PARALLEL_SIBLING_MAX_WORKERS) + workers = TFORK_PARALLEL_SIBLING_MAX_WORKERS; + return workers; +} + +static bool tfork_parallel_sibling_matches(struct pstree_item *child, bool before_setsid) +{ + return restore_before_setsid(child) == before_setsid; +} + +static int tfork_parallel_sibling_main(void *opaque) +{ + struct tfork_parallel_sibling_arg *arg = opaque; + struct pstree_item *child; + sigset_t unblock; + int ordinal = 0; + + /* The root blocks SIGCHLD while it owns/reaps the temporary helpers. */ + sigemptyset(&unblock); + sigaddset(&unblock, SIGCHLD); + if (sigprocmask(SIG_UNBLOCK, &unblock, NULL)) { + pr_perror("tfork: parallel sibling helper cannot unblock SIGCHLD"); + return 1; + } + + list_for_each_entry(child, ¤t->children, sibling) { + if (!tfork_parallel_sibling_matches(child, arg->before_setsid)) + continue; + if ((ordinal++ % arg->worker_count) != arg->worker_index) + continue; + if (arg->before_setsid) + BUG_ON(child->born_sid != -1 && getsid(0) != child->born_sid); + if (fork_with_pid_mode(child, true) < 0) + return 1; + } + return 0; +} + +/* + * Return 0 when the matching children were created, 1 when the guarded path + * is ineligible (the caller should use the serial loop), and -1 after a + * partial/failed parallel attempt. + */ +static int tfork_create_siblings_parallel(bool before_setsid, int requested_workers) +{ + struct tfork_parallel_sibling_arg args[TFORK_PARALLEL_SIBLING_MAX_WORKERS]; + pid_t helpers[TFORK_PARALLEL_SIBLING_MAX_WORKERS] = {}; + struct pstree_item *child; + sigset_t oldmask; + int child_count = 0, workers, launched = 0, i, ret = -1; + + if (!kdat.has_clone3_set_tid) + return 1; + + list_for_each_entry(child, ¤t->children, sibling) { + if (!tfork_parallel_sibling_matches(child, before_setsid)) + continue; + child_count++; + /* + * CLONE_PARENT is added only to the helper's clone3 syscall so + * that the restored child remains a child of current. Nested PID + * namespace creation and pre-existing CLONE_PARENT semantics need + * a separate dependency proof and stay on the serial path. + */ + if (rsti(child)->clone_flags & (CLONE_PARENT | CLONE_NEWPID | CLONE_VM | CLONE_THREAD)) + return 1; + } + /* Keep small process trees on the cheaper and better-tested serial path. */ + if (child_count < TFORK_PARALLEL_SIBLING_MIN_CHILDREN) + return 1; + + workers = requested_workers; + if (workers > child_count) + workers = child_count; + if (workers > TFORK_PARALLEL_SIBLING_MAX_WORKERS) + workers = TFORK_PARALLEL_SIBLING_MAX_WORKERS; + + if (block_sigmask(&oldmask, SIGCHLD)) + return -1; + + pr_info("tfork: creating %d %s-setsid siblings with %d temporary helpers\n", + child_count, before_setsid ? "pre" : "post", workers); + for (i = 0; i < workers; i++) { + pid_t helper_pid = pstree_get_free_pid(current); + + args[i].worker_index = i; + args[i].worker_count = workers; + args[i].before_setsid = before_setsid; + helpers[i] = clone3_with_pid_noasan(tfork_parallel_sibling_main, &args[i], + 0, SIGCHLD, helper_pid); + if (helpers[i] < 0) { + pr_perror("tfork: cannot create parallel sibling helper at vpid %d", helper_pid); + goto kill_helpers; + } + launched++; + } + + ret = 0; + for (i = 0; i < launched; i++) { + int status = 0; + pid_t waited; + + do { + waited = waitpid(helpers[i], &status, 0); + } while (waited < 0 && errno == EINTR); + if (waited != helpers[i] || !WIFEXITED(status) || WEXITSTATUS(status) != 0) { + pr_err("tfork: parallel sibling helper %d failed (waited=%d status=0x%x)\n", + helpers[i], waited, status); + ret = -1; + } + } + if (restore_sigmask(&oldmask)) + ret = -1; + return ret; + +kill_helpers: + for (i = 0; i < launched; i++) + kill(helpers[i], SIGKILL); + for (i = 0; i < launched; i++) + waitpid(helpers[i], NULL, 0); + restore_sigmask(&oldmask); + return -1; +} + /* * Tasks cannot change sid (session id) arbitrary, but can either * inherit one from ancestor, or create a new one with id equal to @@ -1749,30 +2054,43 @@ static int create_children_and_session(void) { int ret; struct pstree_item *child; + int parallel_workers = tfork_parallel_sibling_worker_count(); pr_info("Restoring children in alien sessions:\n"); - list_for_each_entry(child, ¤t->children, sibling) { - if (!restore_before_setsid(child)) - continue; + ret = parallel_workers > 1 ? + tfork_create_siblings_parallel(true, parallel_workers) : 1; + if (ret < 0) + return ret; + if (ret > 0) { + list_for_each_entry(child, ¤t->children, sibling) { + if (!restore_before_setsid(child)) + continue; - BUG_ON(child->born_sid != -1 && getsid(0) != child->born_sid); + BUG_ON(child->born_sid != -1 && getsid(0) != child->born_sid); - ret = fork_with_pid(child); - if (ret < 0) - return ret; + ret = fork_with_pid(child); + if (ret < 0) + return ret; + } } if (current->parent) restore_sid(); pr_info("Restoring children in our session:\n"); - list_for_each_entry(child, ¤t->children, sibling) { - if (restore_before_setsid(child)) - continue; + ret = parallel_workers > 1 ? + tfork_create_siblings_parallel(false, parallel_workers) : 1; + if (ret < 0) + return ret; + if (ret > 0) { + list_for_each_entry(child, ¤t->children, sibling) { + if (restore_before_setsid(child)) + continue; - ret = fork_with_pid(child); - if (ret < 0) - return ret; + ret = fork_with_pid(child); + if (ret < 0) + return ret; + } } return 0; @@ -4045,6 +4363,7 @@ static int sigreturn_restore(struct task_restore_args *task_args, unsigned long task_args->vdso_rt_size = vdso_rt_size; task_args->can_map_vdso = kdat.can_map_vdso; task_args->has_clone3_set_tid = kdat.has_clone3_set_tid; + task_args->tfork_active = opts.tfork.active; new_sp = restorer_stack(task_args->t->mz); diff --git a/criu/criu/cr-tfork.c b/criu/criu/cr-tfork.c index ea64d6627..90fe0db8c 100644 --- a/criu/criu/cr-tfork.c +++ b/criu/criu/cr-tfork.c @@ -271,7 +271,7 @@ int tfork_read_cropt(void) return 0; snap_path = opts.tfork.snap_root; - if (!snap_path && opts.tfork.copies > 1 && opts.tfork.snap_roots && + if (!snap_path && opts.tfork.copies >= 1 && opts.tfork.snap_roots && opts.tfork.copy_idx < opts.tfork.snap_roots_n) snap_path = opts.tfork.snap_roots[opts.tfork.copy_idx]; @@ -615,6 +615,8 @@ static int cr_tfork_finish(int ret) pstree_switch_state(root_item, TASK_ALIVE); timing_stop(TIME_FROZEN); + if (run_scripts(ACT_TFORK_SOURCE_DETACHED)) + pr_warn("tfork: source-detached notification failed\n"); seccomp_free_entries(); free_file_locks(); @@ -966,8 +968,9 @@ int cr_tfork_tasks(pid_t pid) opts.tfork.pidfd_map[opts.tfork.pidfd_map_nr].pidfd = pidfd; opts.tfork.pidfd_map[opts.tfork.pidfd_map_nr].memfd = -1; opts.tfork.pidfd_map_nr++; - pr_info("tfork: pidfd %d for pid %d (vpid %d uid %d)\n", - pidfd, item->pid->real, localpid(item), uid(item)); + pr_info("tfork: pidfd %d for pid %d (vpid %d uid %d nsid %d level %d)\n", + pidfd, item->pid->real, localpid(item), uid(item), + item->pid->leaf_ns_id, item->pid->ns_level); } ret = run_scripts(ACT_PRE_TFORK_RESTORE); @@ -1078,7 +1081,7 @@ int cr_tfork_tasks(pid_t pid) list_for_each_entry(cgo_iter, &opts.new_cgroup_roots, node) rpc_n_cg_root++; - rpc_max = 32 + 2 * (rpc_n_ifd + rpc_n_ext + rpc_n_cg_root + rpc_max = 33 + 2 * (rpc_n_ifd + rpc_n_ext + rpc_n_cg_root + opts.tfork.snap_mount_n) + rpc_n_copy_args + 2; rpc_argv = calloc(rpc_max, sizeof(*rpc_argv)); @@ -1098,6 +1101,7 @@ int cr_tfork_tasks(pid_t pid) rpc_argv[rpc_n++] = "-o"; rpc_argv[rpc_n++] = restore_log_arg; rpc_argv[rpc_n++] = "-v2"; + rpc_argv[rpc_n++] = "--keep-pid-hierarchy"; if (opts.root) { rpc_argv[rpc_n++] = "--root"; @@ -1171,7 +1175,7 @@ int cr_tfork_tasks(pid_t pid) rpc_argv[rpc_n++] = "--tfork-snap-mounts"; rpc_argv[rpc_n++] = snap_mounts_csv; } - if (opts.tfork.copies > 1) { + if (opts.tfork.copies >= 1) { snprintf(copies_arg, sizeof(copies_arg), "%d", opts.tfork.copies); rpc_argv[rpc_n++] = "--tfork-copies"; @@ -1267,7 +1271,7 @@ int cr_tfork_tasks(pid_t pid) buf[off] = '\0'; end = buf + off; - argv_max = 8 + 2; + argv_max = 8 + 3; for (p = buf; p < end; p++) if (*p == '\0') argv_max++; @@ -1312,6 +1316,11 @@ int cr_tfork_tasks(pid_t pid) argv_new[argc_new++] = "--pidfile"; argv_new[argc_new++] = pidfile_arg; } + if ((size_t)argc_new + 1 >= argv_max) { + pr_err("tfork restore argv overflow: used=%d max=%zu\n", argc_new, argv_max); + exit(1); + } + argv_new[argc_new++] = "--keep-pid-hierarchy"; argv_new[argc_new] = NULL; execv("/proc/self/exe", argv_new); diff --git a/criu/criu/crtools.c b/criu/criu/crtools.c index df40ec14f..639ad06f5 100644 --- a/criu/criu/crtools.c +++ b/criu/criu/crtools.c @@ -332,25 +332,30 @@ int main(int argc, char *argv[], char *envp[]) if (opts.tree_id) pr_warn("Using -t with criu restore is obsoleted\n"); - if (opts.tfork.copies > 1) { + if (!opts.tfork.active && opts.tfork.copies >= 1) { + pr_err("--tfork-copies requires --tfork-restore " + "(use 'criu tfork --tfork-copies=N', not " + "'criu restore --tfork-copies=N')\n"); + return 1; + } + + if (opts.tfork.active && opts.tfork.copies >= 1) { int n = opts.tfork.copies, i; pid_t *children; int (*ready_pipes)[2]; int failed = 0; const char *base_log = opts.output; - const int ns_flags = - CLONE_NEWPID | CLONE_NEWNS; - - if (!opts.tfork.active) { - pr_err("--tfork-copies>1 requires --tfork-restore " - "(use 'criu tfork --tfork-copies=N', not " - "'criu restore --tfork-copies=N')\n"); - return 1; - } + /* + * The n-copy helper must not create/occupy PID 1 in a new + * PID namespace. CRIU restores the real root task as PID 1 + * from the image; if the helper has already consumed it, + * restore fails with EEXIST ("Can't fork for 1"). + */ + const int ns_flags = CLONE_NEWNS; if (!opts.restore_detach) { - pr_err("--tfork-copies>1 requires --restore-detached\n"); + pr_err("--tfork-copies requires --restore-detached\n"); return 1; } @@ -445,6 +450,13 @@ int main(int argc, char *argv[], char *envp[]) if (opts.tfork.snap_roots_n > 0) opts.root = opts.tfork.snap_roots[i]; + /* + * The n-copy child only creates the per-copy mount namespace. PID + * namespaces must be recreated by CRIU from the image, otherwise the + * helper would occupy PID 1 before the restored root task. + */ + opts.keep_pid_hierarchy = 0; + if (tfork_load_ncopy_fabric(i)) { pr_err("tfork-ncopy: copy %d fabric load failed\n", i); diff --git a/criu/criu/image.c b/criu/criu/image.c index 2783b5797..1a93344d5 100644 --- a/criu/criu/image.c +++ b/criu/criu/image.c @@ -374,6 +374,7 @@ int prepare_inventory(InventoryEntry *he) he->has_lsmtype = true; he->lsmtype = host_lsm_type(); + pid_init_dump(crt.i.pid, &crt.i); crt.i.pid->state = TASK_ALIVE; crt.i.pid->real = getpid(); if (get_task_ids(&crt.i)) diff --git a/criu/criu/include/action-scripts.h b/criu/criu/include/action-scripts.h index 88f3e5799..d6ccc0495 100644 --- a/criu/criu/include/action-scripts.h +++ b/criu/criu/include/action-scripts.h @@ -20,6 +20,7 @@ enum script_actions { ACT_QUERY_EXT_FILES, ACT_POST_TFORK_FREEZE, ACT_PRE_TFORK_RESTORE, + ACT_TFORK_SOURCE_DETACHED, ACT_MAX }; diff --git a/criu/criu/include/namespaces.h b/criu/criu/include/namespaces.h index e442e0a39..7a5aa6437 100644 --- a/criu/criu/include/namespaces.h +++ b/criu/criu/include/namespaces.h @@ -187,6 +187,7 @@ extern int restore_mnt_ns(int rst, int *cwd_fd); extern int dump_task_ns_ids(struct pstree_item *); extern int predump_task_ns_ids(struct pstree_item *); +extern int finalize_nested_pid_ns_ids(void); extern int rst_add_ns_id(unsigned int id, struct pstree_item *, struct ns_desc *nd); extern struct ns_id *lookup_ns_by_id(unsigned int id, struct ns_desc *nd); diff --git a/criu/criu/include/pstree.h b/criu/criu/include/pstree.h index f9bdeffd1..515b6cd0e 100644 --- a/criu/criu/include/pstree.h +++ b/criu/criu/include/pstree.h @@ -20,6 +20,23 @@ extern atomic_t pid_uid_generator; #define HELPER_UID_BASE (0x40000000) +static inline void pid_init_dump(struct pid *pid, struct pstree_item *item) +{ + *pid = (struct pid){ + .item = item, + .real = -1, + .local = -1, + .uid = atomic_inc_return(&pid_uid_generator), + .state = TASK_UNDEF, + .stop_signo = -1, + .ns_level = -1, + .leaf_ns_id = ALL_PID_NS_ID, + }; + rb_init_node(&pid->leaf_ns_node); + rb_init_node(&pid->root_ns_node); + rb_init_node(&pid->uid_node); +} + struct pstree_item { struct pstree_item *parent; struct list_head children; /* list of my children */ @@ -65,6 +82,7 @@ enum { #define FDS_EVENT (1 << FDS_EVENT_BIT) extern struct pstree_item *current; +extern int pstree_get_free_pid(struct pstree_item *item); struct rst_info; /* See alloc_pstree_item() for details */ diff --git a/criu/criu/include/restorer.h b/criu/criu/include/restorer.h index 73e27caa3..40bb132d1 100644 --- a/criu/criu/include/restorer.h +++ b/criu/criu/include/restorer.h @@ -242,6 +242,7 @@ struct task_restore_args { int child_subreaper; int membarrier_registration_mask; bool has_clone3_set_tid; + bool tfork_active; /* * info about rseq from libc used to diff --git a/criu/criu/namespaces.c b/criu/criu/namespaces.c index df224db68..11909cea2 100644 --- a/criu/criu/namespaces.c +++ b/criu/criu/namespaces.c @@ -592,6 +592,93 @@ static unsigned int get_ns_id(int pid, struct ns_desc *nd, protobuf_c_boolean *s return __get_ns_id(pid, nd, supported, NULL); } +static unsigned int add_nested_pid_leaf_ns_id(struct pstree_item *item) +{ + struct ns_id *nsid; + + nsid = xzalloc(sizeof(*nsid)); + if (!nsid) + return 0; + + nsid->type = NS_OTHER; + nsid->kid = 0; + nsid->ns_populated = true; + nsid_add(nsid, &pid_ns_desc, ns_next_id++, localpid(item)); + + pr_info("Add nested pid leaf ns %d for task %d(%d), level %d\n", + nsid->id, localpid(item), realpid(item), item->pid->ns_level); + return nsid->id; +} + +static unsigned int ensure_task_leaf_pid_ns_id(struct pstree_item *item); + +static unsigned int task_leaf_pid_ns_id(struct pstree_item *item, unsigned int proc_pid_ns_id) +{ + struct pstree_item *parent = item->parent; + unsigned int selected; + + /* + * os4agent stores localpid as the innermost NSpid (pid->ns[0]). + * Keep pstree_entry.nsid at the same namespace level. Otherwise a + * nested pid namespace init such as bwrap can become (nsid=N, + * localpid=1) and collide with the container init in the same nsid. + */ + if (parent && ensure_task_leaf_pid_ns_id(parent) == 0) + return 0; + + if (parent && parent->pid->leaf_ns_id != ALL_PID_NS_ID) { + if (item->pid->ns_level == parent->pid->ns_level) { + selected = parent->pid->leaf_ns_id; + goto out; + } + if (item->pid->ns_level > parent->pid->ns_level && + proc_pid_ns_id != parent->pid->leaf_ns_id) { + selected = proc_pid_ns_id; + goto out; + } + if (item->pid->ns_level > parent->pid->ns_level) { + selected = add_nested_pid_leaf_ns_id(item); + goto out; + } + } + + selected = proc_pid_ns_id; + +out: + pr_info("pid leaf ns task=%d(%d) uid=%d level=%d parent_level=%d proc_nsid=%u parent_nsid=%d selected=%u\n", + localpid(item), realpid(item), uid(item), item->pid->ns_level, + parent ? parent->pid->ns_level : -1, proc_pid_ns_id, + parent ? parent->pid->leaf_ns_id : -1, selected); + return selected; +} + +static unsigned int ensure_task_leaf_pid_ns_id(struct pstree_item *item) +{ + struct pstree_item *parent = item->parent; + unsigned int proc_pid_ns_id; + + if (parent && ensure_task_leaf_pid_ns_id(parent) == 0) + return 0; + + if (item->pid->leaf_ns_id != ALL_PID_NS_ID) { + if (!parent) + return item->pid->leaf_ns_id; + if (item->pid->ns_level == parent->pid->ns_level && + item->pid->leaf_ns_id == parent->pid->leaf_ns_id) + return item->pid->leaf_ns_id; + if (item->pid->ns_level > parent->pid->ns_level && + item->pid->leaf_ns_id != parent->pid->leaf_ns_id) + return item->pid->leaf_ns_id; + } + + proc_pid_ns_id = get_ns_id(item->pid->real, &pid_ns_desc, NULL); + if (!proc_pid_ns_id) + return 0; + + item->pid->leaf_ns_id = task_leaf_pid_ns_id(item, proc_pid_ns_id); + return item->pid->leaf_ns_id; +} + int dump_one_ns_file(int lfd, u32 id, const struct fd_parms *p) { struct cr_img *img; @@ -771,15 +858,37 @@ int dump_task_ns_ids(struct pstree_item *item) int i; int pid = item->pid->real; TaskKobjIdsEntry *ids = item->ids; + struct pstree_item *parent = item->parent; + unsigned int proc_pid_ns_id; ids->has_pid_ns_id = true; - ids->pid_ns_id = get_ns_id(pid, &pid_ns_desc, NULL); + proc_pid_ns_id = get_ns_id(pid, &pid_ns_desc, NULL); + if (!proc_pid_ns_id) { + pr_err("Can't make pidns id\n"); + return -1; + } + + if (parent && ensure_task_leaf_pid_ns_id(parent) == 0) + return -1; + + ids->pid_ns_id = proc_pid_ns_id; + if (parent && item->pid->ns_level == parent->pid->ns_level) + ids->pid_ns_id = parent->pid->leaf_ns_id; + else if (parent && item->pid->ns_level > parent->pid->ns_level && + ids->pid_ns_id == parent->pid->leaf_ns_id) + ids->pid_ns_id = add_nested_pid_leaf_ns_id(item); + if (!ids->pid_ns_id) { pr_err("Can't make pidns id\n"); return -1; } item->pid->leaf_ns_id = ids->pid_ns_id; + pr_info("dump pid ns task=%d(%d) uid=%d level=%d parent_level=%d proc_nsid=%u parent_nsid=%d selected=%u\n", + localpid(item), realpid(item), uid(item), item->pid->ns_level, + parent ? parent->pid->ns_level : -1, proc_pid_ns_id, + parent ? parent->pid->leaf_ns_id : -1, ids->pid_ns_id); + for (i = 0; i < item->nr_threads; i++) item->threads[i].leaf_ns_id = ids->pid_ns_id; @@ -865,6 +974,40 @@ int dump_task_ns_ids(struct pstree_item *item) return 0; } +int finalize_nested_pid_ns_ids(void) +{ + struct pstree_item *item; + + for_each_pstree_item(item) { + struct pstree_item *parent = item->parent; + unsigned int nsid; + int i; + + if (!parent) + continue; + if (item->pid->ns_level <= parent->pid->ns_level) + continue; + if (item->pid->leaf_ns_id != parent->pid->leaf_ns_id) + continue; + + nsid = add_nested_pid_leaf_ns_id(item); + if (!nsid) + return -1; + + item->pid->leaf_ns_id = nsid; + for (i = 0; i < item->nr_threads; i++) + item->threads[i].leaf_ns_id = nsid; + if (item->ids && item->ids->has_pid_ns_id) + item->ids->pid_ns_id = nsid; + + pr_info("finalize nested pid ns task=%d(%d) uid=%d level=%d parent_nsid=%d selected=%u\n", + localpid(item), realpid(item), uid(item), item->pid->ns_level, + parent->pid->leaf_ns_id, nsid); + } + + return 0; +} + static UsernsEntry userns_entry = USERNS_ENTRY__INIT; #define INVALID_ID (~0U) diff --git a/criu/criu/pie/restorer.c b/criu/criu/pie/restorer.c index 9aaefe502..a63dbdd62 100644 --- a/criu/criu/pie/restorer.c +++ b/criu/criu/pie/restorer.c @@ -97,6 +97,7 @@ static pid_t *helpers; static int n_helpers; static pid_t *zombies; static int n_zombies; +static bool tfork_active_local; static enum faults fi_strategy; bool fault_injected(enum faults f) { @@ -162,7 +163,8 @@ static void sigchld_handler(int signal, siginfo_t *siginfo, void *data) else r = "disappeared with"; - pr_info("Task %d %s %d\n", siginfo->si_pid, r, siginfo->si_status); + pr_err("SIGCHLD during restore: task %d %s %d\n", + siginfo->si_pid, r, siginfo->si_status); futex_abort_and_wake(&task_entries_local->nr_in_progress); /* sa_restorer may be unmaped, so we can't go back to userspace*/ @@ -807,12 +809,24 @@ __visible long __export_restore_thread(struct thread_restore_args *args) } pr_info("%ld: Restored\n", sys_gettid()); - restore_finish_stage(task_entries_local, CR_STATE_RESTORE); + if (args->ta->tfork_active) + pr_debug("tfork: thread restore stage complete pid=%d tid=%ld comm=%s ns_level=%d\n", + args->pid, sys_gettid(), args->comm, args->ns_level); + ret = restore_finish_stage(task_entries_local, CR_STATE_RESTORE); + if (args->ta->tfork_active) + pr_debug("tfork: thread restore barrier returned pid=%d tid=%ld comm=%s stage=%d\n", + args->pid, sys_gettid(), args->comm, ret); if (restore_signals(args->siginfo, args->siginfo_n, false)){ goto core_restore_end; } - restore_finish_stage(task_entries_local, CR_STATE_RESTORE_SIGCHLD); + if (args->ta->tfork_active) + pr_debug("tfork: thread sigchld stage complete pid=%d tid=%ld comm=%s ns_level=%d\n", + args->pid, sys_gettid(), args->comm, args->ns_level); + ret = restore_finish_stage(task_entries_local, CR_STATE_RESTORE_SIGCHLD); + if (args->ta->tfork_active) + pr_debug("tfork: thread sigchld barrier returned pid=%d tid=%ld comm=%s stage=%d\n", + args->pid, sys_gettid(), args->comm, ret); /* * Make sure it's before creds, since it's privileged @@ -1476,6 +1490,11 @@ static int wait_zombies(struct task_restore_args *task_args) ret = sys_waitid(P_PID, task_args->zombies[i], NULL, WNOWAIT | WEXITED, NULL); if (ret == -ECHILD) { + if (task_args->tfork_active) { + pr_warn("tfork: zombie pid %d is not reparented to task %ld; skipping wait to avoid restore barrier deadlock\n", + task_args->zombies[i], sys_getpid()); + continue; + } /* A process isn't reparented to this task yet. * Let's wait when someone complete this stage * and try again. @@ -1752,6 +1771,7 @@ __visible long __export_restore_task(struct task_restore_args *args) fi_strategy = args->fault_strategy; task_entries_local = args->task_entries; + tfork_active_local = args->tfork_active; helpers = args->helpers; n_helpers = args->helpers_n; zombies = args->zombies; @@ -2465,6 +2485,18 @@ __visible long __export_restore_task(struct task_restore_args *args) c_args.set_tid = ptr_to_u64(thread_args[i].tid_in_ns); c_args.flags = clone_flags; c_args.set_tid_size = thread_args[i].ns_level; + if (args->tfork_active && thread_args[i].ns_level > 0) { + /* + * Preserve the TID visible in the clone's innermost PID namespace. + * Outer namespace TIDs are allocated by the kernel so concurrent + * copy helpers cannot collide with each other on the host. + */ + pr_debug("tfork: restore thread pid=%d with innermost tid=%d, set_tid_size %d -> 1\n", + thread_args[i].pid, + thread_args[i].tid_in_ns[0], + thread_args[i].ns_level); + c_args.set_tid_size = 1; + } /* The kernel does stack + stack_size. */ c_args.stack = new_sp - RESTORE_STACK_SIZE; c_args.stack_size = RESTORE_STACK_SIZE; @@ -2495,7 +2527,14 @@ __visible long __export_restore_task(struct task_restore_args *args) args->clone_restore_fn); } if (ret != thread_args[i].pid) { - pr_err("Unable to create a thread: %ld\n", ret); + pr_err("Unable to create a thread: %ld expected=%d ns_level=%d tids=%d/%d/%d/%d tfork=%d\n", + ret, thread_args[i].pid, + thread_args[i].ns_level, + thread_args[i].tid_in_ns[0], + thread_args[i].ns_level > 1 ? thread_args[i].tid_in_ns[1] : -1, + thread_args[i].ns_level > 2 ? thread_args[i].tid_in_ns[2] : -1, + thread_args[i].ns_level > 3 ? thread_args[i].tid_in_ns[3] : -1, + args->tfork_active ? 1 : 0); sys_close(fd); mutex_unlock(&task_entries_local->last_pid_mutex); goto core_restore_end; @@ -2524,13 +2563,33 @@ __visible long __export_restore_task(struct task_restore_args *args) if (restore_membarrier_registrations(args->membarrier_registration_mask) < 0) goto core_restore_end; pr_info("%ld: Restored\n", sys_getpid()); - - restore_finish_stage(task_entries_local, CR_STATE_RESTORE); - + if (args->tfork_active) + pr_debug("tfork: leader restore stage complete pid=%d tid=%ld comm=%s threads=%d ns_level=%d\n", + args->t->pid, sys_getpid(), args->comm, args->nr_threads, + args->t->ns_level); + + ret = restore_finish_stage(task_entries_local, CR_STATE_RESTORE); + if (args->tfork_active) + pr_debug("tfork: leader restore barrier returned pid=%d tid=%ld comm=%s stage=%ld helpers=%u zombies=%u inotify=%u\n", + args->t->pid, sys_getpid(), args->comm, ret, + args->helpers_n, args->zombies_n, args->inotify_fds_n); + + if (args->tfork_active) + pr_debug("tfork: leader wait_helpers start pid=%d tid=%ld comm=%s helpers=%u\n", + args->t->pid, sys_getpid(), args->comm, args->helpers_n); if (wait_helpers(args) < 0) goto core_restore_end; + if (args->tfork_active) + pr_debug("tfork: leader wait_helpers done pid=%d tid=%ld comm=%s\n", + args->t->pid, sys_getpid(), args->comm); + if (args->tfork_active) + pr_debug("tfork: leader wait_zombies start pid=%d tid=%ld comm=%s zombies=%u\n", + args->t->pid, sys_getpid(), args->comm, args->zombies_n); if (wait_zombies(args) < 0) goto core_restore_end; + if (args->tfork_active) + pr_debug("tfork: leader wait_zombies done pid=%d tid=%ld comm=%s\n", + args->t->pid, sys_getpid(), args->comm); ksigfillset(&to_block); ret = sys_sigprocmask(SIG_SETMASK, &to_block, NULL, sizeof(k_rtsigset_t)); @@ -2539,9 +2598,18 @@ __visible long __export_restore_task(struct task_restore_args *args) goto core_restore_end; } + if (args->tfork_active) + pr_debug("tfork: leader cleanup_inotify start pid=%d tid=%ld comm=%s inotify=%u\n", + args->t->pid, sys_getpid(), args->comm, args->inotify_fds_n); if (cleanup_current_inotify_events(args)) goto core_restore_end; + if (args->tfork_active) + pr_debug("tfork: leader cleanup_inotify done pid=%d tid=%ld comm=%s\n", + args->t->pid, sys_getpid(), args->comm); + if (args->tfork_active) + pr_debug("tfork: leader restore sigaction start pid=%d tid=%ld comm=%s\n", + args->t->pid, sys_getpid(), args->comm); if (!args->compatible_mode) { ret = sys_sigaction(SIGCHLD, &args->sigchld_act, NULL, sizeof(k_rtsigset_t)); } else { @@ -2558,16 +2626,38 @@ __visible long __export_restore_task(struct task_restore_args *args) pr_err("Failed to restore SIGCHLD: %ld\n", ret); goto core_restore_end; } + if (args->tfork_active) + pr_debug("tfork: leader restore sigaction done pid=%d tid=%ld comm=%s\n", + args->t->pid, sys_getpid(), args->comm); + if (args->tfork_active) + pr_debug("tfork: leader restore shared signals start pid=%d tid=%ld comm=%s\n", + args->t->pid, sys_getpid(), args->comm); ret = restore_signals(args->siginfo, args->siginfo_n, true); if (ret) goto core_restore_end; + if (args->tfork_active) + pr_debug("tfork: leader restore shared signals done pid=%d tid=%ld comm=%s\n", + args->t->pid, sys_getpid(), args->comm); + if (args->tfork_active) + pr_debug("tfork: leader restore private signals start pid=%d tid=%ld comm=%s\n", + args->t->pid, sys_getpid(), args->comm); ret = restore_signals(args->t->siginfo, args->t->siginfo_n, false); if (ret) goto core_restore_end; - - restore_finish_stage(task_entries_local, CR_STATE_RESTORE_SIGCHLD); + if (args->tfork_active) + pr_debug("tfork: leader restore private signals done pid=%d tid=%ld comm=%s\n", + args->t->pid, sys_getpid(), args->comm); + + if (args->tfork_active) + pr_debug("tfork: leader sigchld stage complete pid=%d tid=%ld comm=%s threads=%d ns_level=%d\n", + args->t->pid, sys_getpid(), args->comm, args->nr_threads, + args->t->ns_level); + ret = restore_finish_stage(task_entries_local, CR_STATE_RESTORE_SIGCHLD); + if (args->tfork_active) + pr_debug("tfork: leader sigchld barrier returned pid=%d tid=%ld comm=%s stage=%ld\n", + args->t->pid, sys_getpid(), args->comm, ret); rst_tcp_socks_all(args); diff --git a/criu/criu/pstree.c b/criu/criu/pstree.c index c448c9516..7371fcb48 100644 --- a/criu/criu/pstree.c +++ b/criu/criu/pstree.c @@ -272,26 +272,26 @@ struct pstree_item *__alloc_pstree_item(bool rst) INIT_LIST_HEAD(&item->children); INIT_LIST_HEAD(&item->sibling); - item->pid->ns_level = -1; - item->pid->leaf_ns_id = ALL_PID_NS_ID; - item->pid->real = -1; - item->pid->local = -1; - if (!rst) - item->pid->uid = atomic_inc_return(&pid_uid_generator); - else + pid_init_dump(item->pid, item); + else { + item->pid->ns_level = -1; + item->pid->leaf_ns_id = ALL_PID_NS_ID; + item->pid->real = -1; + item->pid->local = -1; item->pid->uid = -1; - item->pid->state = TASK_UNDEF; - item->pid->stop_signo = -1; + item->pid->state = TASK_UNDEF; + item->pid->stop_signo = -1; + item->pid->item = item; + rb_init_node(&item->pid->leaf_ns_node); + rb_init_node(&item->pid->root_ns_node); + rb_init_node(&item->pid->uid_node); + } item->born_sid = -1; item->tfork_pidfd = -1; item->tfork_memfd = -1; item->tfork_pagemap_fd = -1; - item->pid->item = item; futex_init(&item->task_st); - rb_init_node(&item->pid->leaf_ns_node); - rb_init_node(&item->pid->root_ns_node); - rb_init_node(&item->pid->uid_node); return item; } @@ -436,7 +436,10 @@ int dump_pstree(struct pstree_item *root_item) pstree_entry__init(e); tree_entries[nr_items++] = e; - pr_info("Process: %d(%d)\n", localpid(item), realpid(item)); + pr_info("Process: %d(%d) uid=%d nsid=%d level=%d parent=%d parent_nsid=%d\n", + localpid(item), realpid(item), uid(item), item->pid->leaf_ns_id, + item->pid->ns_level, item->parent ? realpid(item->parent) : 0, + item->parent ? item->parent->pid->leaf_ns_id : -1); e->realpid = realpid(item); e->ppid = item->parent ? realpid(item->parent) : 0; @@ -729,11 +732,37 @@ static int __pstree_insert_pid(struct pid *pid_node, struct rb_node *root_parent rb_link_and_balance(&uid_root_rb, &pid_node->uid_node, parent, link); } - return 0; + return 0; err: - rb_erase(&pid_node->root_ns_node, &pid_root_rb[ALL_PID_NS_ID]); - return -1; + rb_erase(&pid_node->root_ns_node, &pid_root_rb[ALL_PID_NS_ID]); + return -1; +} + +static void pstree_remove_pid_if_linked(struct pid *pid_node) +{ + struct pid *found; + bool valid_leaf_ns = pid_node->leaf_ns_id >= 0 && + (unsigned int)pid_node->leaf_ns_id < pid_namespace_count; + + if (pid_node->uid > 0) { + found = __lookup_pid_uid(&uid_root_rb, pid_node->uid, NULL, NULL); + if (found == pid_node) + rb_erase(&pid_node->uid_node, &uid_root_rb); + } + + if (pid_node->leaf_ns_id != ALL_PID_NS_ID && valid_leaf_ns) { + found = __lookup_pid_leaf(&pid_root_rb[pid_node->leaf_ns_id], + pid_node->local, NULL, NULL); + if (found == pid_node) + rb_erase(&pid_node->leaf_ns_node, + &pid_root_rb[pid_node->leaf_ns_id]); + } + + found = __lookup_pid_root(&pid_root_rb[ALL_PID_NS_ID], + pid_node->real, NULL, NULL); + if (found == pid_node) + rb_erase(&pid_node->root_ns_node, &pid_root_rb[ALL_PID_NS_ID]); } int pstree_insert_pid(struct pid *pid_node) @@ -741,13 +770,14 @@ int pstree_insert_pid(struct pid *pid_node) return __pstree_insert_pid(pid_node, NULL, NULL); } -static struct pstree_item *get_or_create_pstree_item(pid_t real, pid_t local, int pidns_id) +static struct pstree_item *get_or_create_pstree_item(pid_t real, pid_t local, int pidns_id, bool *created) { struct pid *found; struct pstree_item *item; - struct rb_node **root_link, *root_parent; - found = __lookup_pid_root(&pid_root_rb[ALL_PID_NS_ID], real, &root_parent, &root_link); + *created = false; + + found = __lookup_pid_root(&pid_root_rb[ALL_PID_NS_ID], real, NULL, NULL); if (found) { if (pidns_id != ALL_PID_NS_ID) { BUG_ON(found->leaf_ns_id != pidns_id || found->local != local); @@ -762,11 +792,7 @@ static struct pstree_item *get_or_create_pstree_item(pid_t real, pid_t local, in item->pid->real = real; item->pid->local = local; item->pid->leaf_ns_id = pidns_id; - - if (__pstree_insert_pid(item->pid, root_parent, root_link) < 0) { - xfree(item); - return NULL; - } + *created = true; return item; } @@ -876,13 +902,24 @@ static int read_pstree_ids(struct pstree_item *pi) */ static int read_one_pstree_item(PstreeEntry *e) { - struct pstree_item *pi; - int ret = -1, i, j; + struct pstree_item *pi = NULL; + int ret = -1, i, j, next_inserted_thread = 1; + bool linked = false, pid_inserted = false, threads_allocated = false; + bool created_item = false; - pi = get_or_create_pstree_item(e->realpid, e->localpid, e->nsid); + pi = get_or_create_pstree_item(e->realpid, e->localpid, e->nsid, &created_item); if (!pi) goto err; + /* + * get_or_create_pstree_item() can only reuse an item that is still + * TASK_UNDEF. Completed items are rejected here, so the unwind below + * cannot tear down a previously parsed pstree item. + */ BUG_ON(pi->pid->state != TASK_UNDEF); + if (!created_item && (pi->threads || pi->nr_threads)) { + pr_err("Refusing to reuse partially populated pstree item for %d\n", e->realpid); + goto err; + } /* * Populate the ns-chain on pi from the thread-leader entry before @@ -915,6 +952,27 @@ static int read_one_pstree_item(PstreeEntry *e) } pi->pid->state = TASK_ALIVE; pi->pid->uid = e->uid; + pi->nr_threads = e->n_threads; + pi->threads = xmalloc(e->n_threads * sizeof(struct pid)); + if (!pi->threads) + goto err; + threads_allocated = true; + + /* note: we don't fail if we have empty ids */ + if (read_pstree_ids(pi) < 0) + goto err; + + if (pi->ids && pi->ids->has_pid_ns_id) { + if (pi->ids->pid_ns_id != pi->pid->leaf_ns_id) { + pr_warn("PID namespace id mismatch for uid %d: pstree=%d ids=%d, keeping pstree\n", + uid(pi), pi->pid->leaf_ns_id, pi->ids->pid_ns_id); + pi->ids->pid_ns_id = pi->pid->leaf_ns_id; + } + } + + if (__pstree_insert_pid(pi->pid, NULL, NULL) < 0) + goto err; + pid_inserted = true; if (e->ppid == 0) { if (root_item) { @@ -938,13 +996,9 @@ static int read_one_pstree_item(PstreeEntry *e) parent = pid->item; pi->parent = parent; list_add(&pi->sibling, &parent->children); + linked = true; } - pi->nr_threads = e->n_threads; - pi->threads = xmalloc(e->n_threads * sizeof(struct pid)); - if (!pi->threads) - goto err; - for (i = 0; i < e->n_threads; i++) { int insert_status; pi->threads[i].uid = e->threads[i]->uid; @@ -960,7 +1014,7 @@ static int read_one_pstree_item(PstreeEntry *e) pi->threads[i].state = TASK_THREAD; pi->threads[i].item = NULL; if (i == 0) { - + /* The leader is indexed through pi->pid, not this mirror. */ pi->pid->ns_level = pi->threads[0].ns_level; pi->pid->local = pi->threads[0].ns[0].ns_pid; memcpy(pi->pid->ns, pi->threads[0].ns, e->threads[0]->n_ns * sizeof(struct pid_ns)); @@ -972,17 +1026,42 @@ static int read_one_pstree_item(PstreeEntry *e) pr_err("Unexpected task %d in a tree %d\n", e->threads[i]->ns[0]->nspid, i); goto err; } + next_inserted_thread = i + 1; } task_entries->nr_threads += e->n_threads; task_entries->nr_tasks++; - /* note: we don't fail if we have empty ids */ - if (read_pstree_ids(pi) < 0) - goto err; - ret = 1; err: + if (ret < 0 && pi) { + /* + * threads[0] is the leader mirrored by pi->pid. Only + * non-leader threads are inserted independently, and + * next_inserted_thread always points one past the last + * successfully inserted non-leader slot. + */ + for (i = 1; i < next_inserted_thread; i++) + pstree_remove_pid_if_linked(&pi->threads[i]); + if (root_item == pi) + root_item = NULL; + if (linked) + list_del_init(&pi->sibling); + if (pid_inserted) + pstree_remove_pid_if_linked(pi->pid); + pi->pid->state = TASK_UNDEF; + if (threads_allocated) { + xfree(pi->threads); + pi->threads = NULL; + pi->nr_threads = 0; + } + /* + * Restore pstree items come from the shared linear arena. The + * item may no longer be the last allocation after read_pstree_ids(), + * so it cannot be released individually. Restore teardown reclaims + * the arena after this parse failure. + */ + } return ret; } @@ -1024,7 +1103,7 @@ static int read_pstree_image(void) return ret < 0 ? -1 : 0; } -static int helper_get_free_pid(struct pstree_item *item) +int pstree_get_free_pid(struct pstree_item *item) { int pidns = item ? item->pid->leaf_ns_id : ALL_PID_NS_ID; @@ -1082,7 +1161,7 @@ static int prepare_pstree_ids(pid_t pid) } } if (leader->pid->state != TASK_UNDEF) { - helper_pid = helper_get_free_pid(item); + helper_pid = pstree_get_free_pid(item); if (helper_pid < 0) break; @@ -1262,8 +1341,12 @@ static int new_pid_ns_truncate_pid_hierarchy(pid_t *pid_max) unsigned int ns_level_to_truncate; clone_flags = get_clone_mask(root_item->ids, root_ids); - if (!(clone_flags & CLONE_NEWPID)) + if (!(clone_flags & CLONE_NEWPID) && + !(opts.tfork.active && root_item->pid->ns_level > 1)) return 0; + if (!(clone_flags & CLONE_NEWPID)) + pr_info("pidns: forcing tfork pid hierarchy truncation for root level=%d\n", + root_item->pid->ns_level); if (root_item->pid->ns_level <= 1) { pr_err("only 1 level of pid namespace, but CLONE_NEWPID is set, " @@ -1272,6 +1355,8 @@ static int new_pid_ns_truncate_pid_hierarchy(pid_t *pid_max) } ns_level_to_truncate = root_item->pid->ns_level - 1; + pr_info("pidns: truncating %u outer pid namespace level(s) for new root pid namespace\n", + ns_level_to_truncate); for (node = rb_first(&pid_root_rb[ALL_PID_NS_ID]); node; ) { next = rb_next(node); @@ -1279,9 +1364,15 @@ static int new_pid_ns_truncate_pid_hierarchy(pid_t *pid_max) pid_node = rb_entry(node, struct pid, root_ns_node); rb_erase(node, &pid_root_rb[ALL_PID_NS_ID]); + pr_info("pidns: truncate before uid=%d real=%d local=%d level=%d\n", + pid_node->uid, pid_node->real, pid_node->local, + pid_node->ns_level); pid_node->ns_level -= ns_level_to_truncate; BUG_ON(pid_node->ns_level <= 0); pid_node->real = pid_node->ns[pid_node->ns_level - 1].ns_pid; + pr_info("pidns: truncate after uid=%d real=%d local=%d level=%d\n", + pid_node->uid, pid_node->real, pid_node->local, + pid_node->ns_level); found = __lookup_pid_root(&new_real_rbtree, pid_node->real, &parent, &link); if (found) { diff --git a/criu/criu/seize.c b/criu/criu/seize.c index f5cde74e9..db8d95b92 100644 --- a/criu/criu/seize.c +++ b/criu/criu/seize.c @@ -379,7 +379,7 @@ static int seize_cgroup_tree(char *root_path, enum freezer_state state) */ static int freezer_wait_processes(void) { - int i; + int i, collected = 0; processes_to_wait_pids = xmalloc(sizeof(pid_t) * processes_to_wait); if (processes_to_wait_pids == NULL) @@ -388,23 +388,42 @@ static int freezer_wait_processes(void) for (i = 0; i < processes_to_wait; i++) { int status; pid_t pid; + int waited_ms = 0; /* * Here we are going to skip tasks which are already traced. * Ptraced tasks looks like children for us, so if * a task isn't ptraced yet, waitpid() will return a error. */ - pid = waitpid(-1, &status, 0); - if (pid < 0) { - pr_perror("Unable to wait processes"); - xfree(processes_to_wait_pids); - processes_to_wait_pids = NULL; - return -1; + while (1) { + pid = waitpid(-1, &status, opts.tfork.active ? WNOHANG : 0); + if (pid > 0) + break; + if (opts.tfork.active && pid < 0 && errno == EINTR && waited_ms < 500) { + usleep(10 * 1000); + waited_ms += 10; + continue; + } + if (!opts.tfork.active || (pid < 0 && errno != ECHILD && errno != EINTR)) { + pr_perror("Unable to wait processes"); + xfree(processes_to_wait_pids); + processes_to_wait_pids = NULL; + return -1; + } + if (pid < 0 || waited_ms >= 500) { + pr_warn("tfork: collected %d/%d unexpected freezer processes; continuing\n", + collected, processes_to_wait); + processes_to_wait = collected; + return 0; + } + usleep(10 * 1000); + waited_ms += 10; } pr_warn("Unexpected process %d in the freezer cgroup (status 0x%x)\n", pid, status); - processes_to_wait_pids[i] = pid; + processes_to_wait_pids[collected++] = pid; } + processes_to_wait = collected; return 0; } diff --git a/criu/criu/unittest/mock.c b/criu/criu/unittest/mock.c index b2d507278..b9601fffa 100644 --- a/criu/criu/unittest/mock.c +++ b/criu/criu/unittest/mock.c @@ -97,6 +97,10 @@ int close_service_fd(int type) return 0; } +void invalidate_proc_self_fd(void) +{ +} + void compel_log_init(int log_fn, unsigned int level) { } diff --git a/criu/criu/unittest/unit.c b/criu/criu/unittest/unit.c index 54769e6f2..cf9d79b24 100644 --- a/criu/criu/unittest/unit.c +++ b/criu/criu/unittest/unit.c @@ -3,19 +3,41 @@ #include #include "log.h" +#include "pstree.h" #include "util.h" #include "criu-log.h" int parse_statement(int i, char *line, char **configuration); +atomic_t pid_uid_generator = ATOMIC_INIT(0); + int main(int argc, char *argv[], char *envp[]) { char **configuration; + struct pid first_pid = {}; + struct pid second_pid = {}; + struct pstree_item first_item = { .pid = &first_pid }; + struct pstree_item second_item = { .pid = &second_pid }; int i; configuration = malloc(10 * sizeof(char *)); log_init(NULL); + pid_init_dump(&first_pid, &first_item); + pid_init_dump(&second_pid, &second_item); + assert(first_pid.uid > 0); + assert(second_pid.uid > first_pid.uid); + assert(first_pid.item == &first_item); + assert(first_pid.real == -1); + assert(first_pid.local == -1); + assert(first_pid.state == TASK_UNDEF); + assert(first_pid.stop_signo == -1); + assert(first_pid.ns_level == -1); + assert(first_pid.leaf_ns_id == ALL_PID_NS_ID); + assert(RB_EMPTY_NODE(&first_pid.leaf_ns_node)); + assert(RB_EMPTY_NODE(&first_pid.root_ns_node)); + assert(RB_EMPTY_NODE(&first_pid.uid_node)); + i = parse_statement(0, "", configuration); assert(i == 0); diff --git a/criu/lib/pycriu/images/images.py b/criu/lib/pycriu/images/images.py index 9db506e1e..3d2e02777 100644 --- a/criu/lib/pycriu/images/images.py +++ b/criu/lib/pycriu/images/images.py @@ -43,6 +43,8 @@ import os import array +from google.protobuf.message import DecodeError + from . import magic from . import pb from . import pb2dict @@ -190,6 +192,98 @@ def count(self, f): return entries +class pstree_handler: + """Read both legacy per-task and current file-level PSTREE images.""" + + @staticmethod + def _read_payload(f): + header = f.read(4) + if not header: + return None + if len(header) != 4: + raise ValueError("truncated PSTREE entry header") + + size, = struct.unpack('i', header) + if size < 0: + raise ValueError("negative PSTREE entry size") + + payload = f.read(size) + if len(payload) != size: + raise ValueError("truncated PSTREE entry payload") + return payload + + @staticmethod + def _parse(payload, message_type): + message = message_type() + try: + message.ParseFromString(payload) + except DecodeError: + return None + return message if message.IsInitialized() else None + + def load(self, f, pretty=False, no_payload=False): + # PSTREE has no out-of-band EXTRA data, so no_payload has no effect. + payload = self._read_payload(f) + if payload is None: + return [] + + # Format detection intentionally relies on pstree_entry being proto2 + # with required fields. A file-level payload may parse as that message, + # but it cannot be initialized because its wire fields have other types. + legacy = self._parse(payload, pb.pstree_entry) + if legacy is not None: + entries = [legacy] + while True: + payload = self._read_payload(f) + if payload is None: + break + entry = self._parse(payload, pb.pstree_entry) + if entry is None: + raise ValueError("invalid legacy PSTREE entry") + entries.append(entry) + return [pb2dict.pb2dict(entry, pretty) for entry in entries] + + entry = self._parse(payload, pb.pstree_file_entry) + if entry is None: + raise ValueError("invalid PSTREE entry") + if self._read_payload(f) is not None: + raise ValueError("file-level PSTREE image has multiple entries") + + return [pb2dict.pb2dict(entry, pretty)] + + def loads(self, data, pretty=False): + return self.load(io.BytesIO(data), pretty) + + def dump(self, entries, f): + if not entries: + return + + file_level = ('tree' in entries[0] or + 'ns_max_pids' in entries[0]) + if file_level and len(entries) != 1: + raise ValueError("file-level PSTREE image requires one entry") + + message_type = (pb.pstree_file_entry if file_level + else pb.pstree_entry) + for entry in entries: + message = message_type() + pb2dict.dict2pb(entry, message) + payload = message.SerializeToString() + f.write(struct.pack('i', len(payload))) + f.write(payload) + + def dumps(self, entries): + f = io.BytesIO() + self.dump(entries, f) + return f.getvalue() + + def count(self, f): + entries = 0 + while self._read_payload(f) is not None: + entries += 1 + return entries + + # Special handler for pagemap.img class pagemap_handler: """ @@ -502,7 +596,7 @@ def skip(self, f, pbuff): tcp_stream_extra_handler()), 'STATS': entry_handler(pb.stats_entry), 'PAGEMAP': pagemap_handler(), # Special one - 'PSTREE': entry_handler(pb.pstree_entry), + 'PSTREE': pstree_handler(), 'REG_FILES': entry_handler(pb.reg_file_entry), 'NS_FILES': entry_handler(pb.ns_file_entry), 'EVENTFD_FILE': entry_handler(pb.eventfd_file_entry), diff --git a/criu/test/others/pycriu/Makefile b/criu/test/others/pycriu/Makefile index b6e3b4814..369ac5667 100644 --- a/criu/test/others/pycriu/Makefile +++ b/criu/test/others/pycriu/Makefile @@ -13,7 +13,8 @@ CRIU_SOCKET := $(BUILD_DIR)/$(SOCKET_NAME) STATUS_FIFO := $(BUILD_DIR)/startup.status STATUS_FD := 200 -run: start +run: pstree-compat + $(MAKE) --no-print-directory start cleanup() { $(MAKE) --no-print-directory stop || true; } trap cleanup EXIT INT TERM "$(PYTHON)" test_check.py @@ -21,6 +22,9 @@ run: start "$(PYTHON)" test_check_images_dir.py "$(PYTHON)" test_check_work_dir_fd.py +pstree-compat: + "$(PYTHON)" test_pstree_compat.py + start: mkdir -p "$(BUILD_DIR)" if [ -s "$(PIDFILE)" ] && kill -0 "$$(cat "$(PIDFILE)")" 2>/dev/null; then @@ -60,4 +64,4 @@ clean: fi rm -rf "$(BUILD_DIR)" -.PHONY: start stop clean run \ No newline at end of file +.PHONY: start stop clean run pstree-compat diff --git a/criu/test/others/pycriu/test_pstree_compat.py b/criu/test/others/pycriu/test_pstree_compat.py new file mode 100644 index 000000000..5cda76e13 --- /dev/null +++ b/criu/test/others/pycriu/test_pstree_compat.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 +import io +import os +import struct +import sys + +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +LIB_DIR = os.path.normpath(os.path.join(SCRIPT_DIR, "../../../lib")) +if LIB_DIR not in sys.path: + sys.path.insert(0, LIB_DIR) + +from pycriu.images import images, pb # noqa: E402 + + +def encode(*messages): + output = io.BytesIO() + for message in messages: + payload = message.SerializeToString() + output.write(struct.pack('i', len(payload))) + output.write(payload) + return output.getvalue() + + +def task(message, realpid, localpid, uid): + message.realpid = realpid + message.ppid = 0 + message.pgid = localpid + message.sid = localpid + message.nsid = 7 + message.localpid = localpid + message.uid = uid + + +def main(): + handler = images.handlers['PSTREE'] + + old_first = pb.pstree_entry() + task(old_first, 1001, 1, 11) + old_second = pb.pstree_entry() + task(old_second, 1002, 2, 12) + old_blob = encode(old_first, old_second) + old_entries = handler.loads(old_blob) + assert [entry['realpid'] for entry in old_entries] == [1001, 1002] + assert handler.loads(handler.dumps(old_entries)) == old_entries + assert handler.count(io.BytesIO(old_blob)) == 2 + + current = pb.pstree_file_entry() + ns_max = current.ns_max_pids.add() + ns_max.ns_id = 7 + ns_max.pid_max = 2 + task(current.tree.add(), 1001, 1, 11) + task(current.tree.add(), 1002, 2, 12) + current_blob = encode(current) + current_entries = handler.loads(current_blob) + assert len(current_entries) == 1 + assert [entry['realpid'] for entry in current_entries[0]['tree']] == [1001, 1002] + assert handler.loads(handler.dumps(current_entries)) == current_entries + assert handler.count(io.BytesIO(current_blob)) == 1 + + print("PASS") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/criu/test/others/tfork-phase-b-ab.sh b/criu/test/others/tfork-phase-b-ab.sh new file mode 100755 index 000000000..d24f07dc6 --- /dev/null +++ b/criu/test/others/tfork-phase-b-ab.sh @@ -0,0 +1,110 @@ +#!/usr/bin/env bash +set -euo pipefail + +PODMAN=${PODMAN:-podman} +PODMAN_GLOBAL_ARGS=${PODMAN_GLOBAL_ARGS:-} +OLD_CRIU_ROOT=${OLD_CRIU_ROOT:?set OLD_CRIU_ROOT} +NEW_CRIU_ROOT=${NEW_CRIU_ROOT:?set NEW_CRIU_ROOT} +SAMPLES=${SAMPLES:-20} +IMAGE=${IMAGE:-docker.io/library/alpine:3.19} +PREFIX=${PREFIX:-tfork-phase-b-$RANDOM} +OUTPUT=${OUTPUT:-/tmp/tfork-phase-b-ab.tsv} +OS4AGENT_CRUN=${OS4AGENT_CRUN:-crun} +WORKLOAD_PROCESSES=${WORKLOAD_PROCESSES:-1} +LOG_DIR=${LOG_DIR:-} + +read -r -a podman_global_args <<<"$PODMAN_GLOBAL_ARGS" +source_name=${PREFIX}-source + +podman_cmd() { + "$PODMAN" "${podman_global_args[@]}" "$@" +} + +cleanup() { + podman_cmd ps -a --format '{{.Names}}' | + awk -v prefix="$PREFIX" 'index($0, prefix) == 1' | + while read -r name; do + podman_cmd rm -f -t 0 "$name" >/dev/null 2>&1 || true + done +} +trap cleanup EXIT + +run_clone() { + local variant=$1 + local root=$2 + local index=$3 + local name=${PREFIX}-${variant}-${index} + local started ended elapsed rootfs bundle + + started=$(date +%s%N) + env \ + PATH="$root/criu:$PATH" \ + LD_LIBRARY_PATH="$root/lib/c${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" \ + OS4AGENT_CRUN="$OS4AGENT_CRUN" \ + "$PODMAN" "${podman_global_args[@]}" container clone \ + --live --tfork-overlay-btrfs "$source_name" "$name" >/dev/null + ended=$(date +%s%N) + elapsed=$(( (ended - started) / 1000000 )) + + [[ $(podman_cmd exec "$name" cat /tmp/sentinel) == source-before-fork ]] + if [[ -n $LOG_DIR ]]; then + rootfs=$(podman_cmd inspect --format '{{.Rootfs}}' "$name") + bundle=$(dirname "$rootfs") + test -f "$bundle/img/tfork.log" + cp "$bundle/img/tfork.log" \ + "$LOG_DIR/${variant}-${index}.tfork.log" + if [[ -f $bundle/img/tfork-restore.log.copy0 ]]; then + cp "$bundle/img/tfork-restore.log.copy0" \ + "$LOG_DIR/${variant}-${index}.restore.log" + fi + fi + podman_cmd kill -s KILL "$name" >/dev/null + podman_cmd rm -f -t 0 "$name" >/dev/null + printf '%s\t%d\t%d\n' "$variant" "$index" "$elapsed" | tee -a "$OUTPUT" +} + +cleanup +: >"$OUTPUT" +if [[ -n $LOG_DIR ]]; then + mkdir -p "$LOG_DIR" +fi +podman_cmd run -d --name "$source_name" \ + --log-driver k8s-file \ + --security-opt seccomp=unconfined \ + --security-opt apparmor=unconfined \ + "$IMAGE" sh -c \ + 'count=$1 + i=1 + while [ "$i" -lt "$count" ]; do + sleep 86400 & + i=$((i + 1)) + done + echo source-before-fork >/tmp/sentinel + exec tail -f /dev/null' sh "$WORKLOAD_PROCESSES" >/dev/null + +for ((attempt = 0; attempt < 100; attempt++)); do + actual_processes=$(podman_cmd top "$source_name" pid | + awk 'NR > 1 { count++ } END { print count + 0 }') + if ((actual_processes == WORKLOAD_PROCESSES)); then + break + fi + sleep 0.05 +done +if ((actual_processes != WORKLOAD_PROCESSES)); then + printf 'expected %d source processes, found %d\n' \ + "$WORKLOAD_PROCESSES" "$actual_processes" >&2 + exit 1 +fi + +for ((i = 1; i <= SAMPLES; i++)); do + if ((i % 2)); then + run_clone old "$OLD_CRIU_ROOT" "$i" + run_clone new "$NEW_CRIU_ROOT" "$i" + else + run_clone new "$NEW_CRIU_ROOT" "$i" + run_clone old "$OLD_CRIU_ROOT" "$i" + fi +done + +podman_cmd rm -f -t 0 "$source_name" >/dev/null +trap - EXIT diff --git a/crun/src/libcrun/container.h b/crun/src/libcrun/container.h index 7b6649e2e..851d91b3a 100644 --- a/crun/src/libcrun/container.h +++ b/crun/src/libcrun/container.h @@ -224,6 +224,8 @@ struct libcrun_checkpoint_restore_s bool track_mem; int tfork_copies; int tfork_dumpd_parent_pid; + int tfork_pre_restore_fd; + int tfork_source_detached_fd; unsigned int tfork_ghost_limit; bool tfork_full_memcopy; diff --git a/crun/src/libcrun/criu.c b/crun/src/libcrun/criu.c index 8bf068d03..681c30cb9 100644 --- a/crun/src/libcrun/criu.c +++ b/crun/src/libcrun/criu.c @@ -46,6 +46,8 @@ # define DESCRIPTORS_FILENAME "descriptors.json" # define CRIU_RUNC_CONFIG_FILE "/etc/criu/runc.conf" # define CRIU_CRUN_CONFIG_FILE "/etc/criu/crun.conf" +# define CRIU_LOG_TAIL_LINES 80 +# define CRIU_LOG_LINE_SIZE 1024 # define CRIU_EXT_NETNS "extRootNetNS" # define CRIU_EXT_PIDNS "extRootPidNS" @@ -58,6 +60,8 @@ char *chroot_realpath (const char *chroot, const char *path, char resolved_path[]); static const char *console_socket = NULL; +static int tfork_pre_restore_fd = -1; +static int tfork_source_detached_fd = -1; # define LIBCRIU_MIN_VERSION 31500 @@ -239,6 +243,39 @@ load_wrapper (struct libcriu_wrapper_s **wrapper_out, libcrun_error_t *err) static int criu_notify (char *action, __attribute__ ((unused)) criu_notify_arg_t na) { + if (action == NULL) + return 0; + + if (strcmp (action, "post-tfork-freeze") == 0 && tfork_pre_restore_fd >= 0) + { + char byte; + ssize_t n; + + do + n = read (tfork_pre_restore_fd, &byte, 1); + while (n < 0 && errno == EINTR); + if (n != 1) + return -1; + close (tfork_pre_restore_fd); + tfork_pre_restore_fd = -1; + return 0; + } + + if (strcmp (action, "tfork-source-detached") == 0 && tfork_source_detached_fd >= 0) + { + char byte = 1; + ssize_t n; + + do + n = write (tfork_source_detached_fd, &byte, 1); + while (n < 0 && errno == EINTR); + if (n != 1) + return -1; + close (tfork_source_detached_fd); + tfork_source_detached_fd = -1; + return 0; + } + if (strncmp (action, "orphan-pts-master", 17) == 0) { /* CRIU sends us the master FD via the 'orphan-pts-master' @@ -543,8 +580,11 @@ static void show_criu_log (const char *work_path, const char *log) { cleanup_free char *log_path = NULL; + cleanup_free char *tail = NULL; libcrun_error_t *tmp_err = NULL; - char line[1024]; + char line[CRIU_LOG_LINE_SIZE]; + size_t tail_index = 0; + size_t tail_count = 0; FILE *f; if (UNLIKELY (append_paths (&log_path, tmp_err, work_path, log, NULL)) < 0) @@ -563,12 +603,43 @@ show_criu_log (const char *work_path, const char *log) /* Log with error verbosity as this is the default. */ libcrun_error (0, "--- excerpt from CRIU log `%s`", log_path); + tail = calloc (CRIU_LOG_TAIL_LINES, CRIU_LOG_LINE_SIZE); + if (tail == NULL) + { + fclose (f); + return; + } + while (fgets (line, sizeof (line), f) != NULL) - if (strstr (line, "Error ") != NULL) - { - line[strcspn (line, "\n")] = '\0'; - libcrun_error (0, "%s", line); - } + { + char *slot = tail + tail_index * CRIU_LOG_LINE_SIZE; + strncpy (slot, line, CRIU_LOG_LINE_SIZE - 1); + slot[CRIU_LOG_LINE_SIZE - 1] = '\0'; + tail_index = (tail_index + 1) % CRIU_LOG_TAIL_LINES; + if (tail_count < CRIU_LOG_TAIL_LINES) + tail_count++; + + if (strstr (line, "Error ") != NULL + || strstr (line, "failed") != NULL || strstr (line, "FAILED") != NULL + || strstr (line, "Unable") != NULL || strstr (line, "Can't") != NULL + || strstr (line, "No such") != NULL) + { + line[strcspn (line, "\n")] = '\0'; + libcrun_error (0, "%s", line); + } + } + + if (tail_count > 0) + { + size_t start = (tail_count == CRIU_LOG_TAIL_LINES) ? tail_index : 0; + libcrun_error (0, "--- last %zu CRIU log lines", tail_count); + for (size_t i = 0; i < tail_count; i++) + { + char *entry = tail + ((start + i) % CRIU_LOG_TAIL_LINES) * CRIU_LOG_LINE_SIZE; + entry[strcspn (entry, "\n")] = '\0'; + libcrun_error (0, "%s", entry); + } + } fclose (f); libcrun_error (0, "--- end of excerpt"); @@ -1351,14 +1422,33 @@ libcrun_container_restore_linux_criu (libcrun_container_status_t *status, libcru } # define CRIU_TFORK_LOG_FILE "tfork.log" +# define CRIU_TFORK_RESTORE_LOG_FILE "tfork-restore.log" +# define CRIU_TFORK_MAX_COPY_LOGS 16 + +static void +show_criu_tfork_restore_copy_logs (const char *image_path, int copy_count) +{ + if (copy_count < 0) + copy_count = 0; + if (copy_count > CRIU_TFORK_MAX_COPY_LOGS) + copy_count = CRIU_TFORK_MAX_COPY_LOGS; + + for (int i = 0; i < copy_count; i++) + { + char log[64]; + snprintf (log, sizeof (log), "%s.copy%d", CRIU_TFORK_RESTORE_LOG_FILE, i); + show_criu_log (image_path, log); + } +} static int -read_source_state_pid (const char *path, pid_t *pid_out, libcrun_error_t *err) +read_source_state_pid_cgroup (const char *path, pid_t *pid_out, char **cgroup_path_out, libcrun_error_t *err) { cleanup_free char *buffer = NULL; char err_buffer[256]; yajl_val tree, tmp; const char *pid_path[] = { "pid", NULL }; + const char *cgroup_path[] = { "cgroup-path", NULL }; int ret; ret = read_all_file (path, &buffer, NULL, err); @@ -1377,6 +1467,15 @@ read_source_state_pid (const char *path, pid_t *pid_out, libcrun_error_t *err) } *pid_out = (pid_t) strtoull (YAJL_GET_NUMBER (tmp), NULL, 10); + + tmp = yajl_tree_get (tree, cgroup_path, yajl_t_string); + if (UNLIKELY (tmp == NULL)) + { + yajl_tree_free (tree); + return crun_make_error (err, 0, "`cgroup-path` missing in source state.json `%s`", path); + } + + *cgroup_path_out = xstrdup (YAJL_GET_STRING (tmp)); yajl_tree_free (tree); return 0; } @@ -1439,10 +1538,13 @@ libcrun_container_tfork_linux_criu (libcrun_container_t *container, libcrun_chec { runtime_spec_schema_config_schema *def = container->container_def; cleanup_wrapper struct libcriu_wrapper_s *wrapper = NULL; + cleanup_free char *freezer_path = NULL; cleanup_free char *rootfs_path = NULL; + cleanup_free char *source_cgroup_path = NULL; cleanup_close int image_fd = -1; cleanup_close int work_fd = -1; pid_t source_pid = 0; + int cgroup_mode; int ret; ret = load_wrapper (&wrapper, err); @@ -1461,7 +1563,7 @@ libcrun_container_tfork_linux_criu (libcrun_container_t *container, libcrun_chec return crun_make_error (err, 0, "--tfork-snap-root, --tfork-snap-roots, or --tfork-copy=::--tfork-snap-root=PATH is required"); - if (cr_options->tfork_copies > 1 && cr_options->tfork_snap_roots_n > 0 + if (cr_options->tfork_copies >= 1 && cr_options->tfork_snap_roots_n > 0 && (size_t) cr_options->tfork_copies != cr_options->tfork_snap_roots_n) return crun_make_error (err, 0, "--tfork-copies=%d but --tfork-snap-roots has %zu entries", @@ -1470,7 +1572,7 @@ libcrun_container_tfork_linux_criu (libcrun_container_t *container, libcrun_chec if (UNLIKELY (cr_options->image_path == NULL)) return crun_make_error (err, 0, "--image-path is required"); - ret = read_source_state_pid (cr_options->source_state, &source_pid, err); + ret = read_source_state_pid_cgroup (cr_options->source_state, &source_pid, &source_cgroup_path, err); if (UNLIKELY (ret < 0)) return ret; if (UNLIKELY (source_pid <= 0)) @@ -1514,8 +1616,24 @@ libcrun_container_tfork_linux_criu (libcrun_container_t *container, libcrun_chec libcriu_wrapper->criu_set_pid (source_pid); libcriu_wrapper->criu_set_leave_running (true); + libcriu_wrapper->criu_set_ext_unix_sk (cr_options->ext_unix_sk); libcriu_wrapper->criu_set_file_locks (true); + cgroup_mode = libcrun_get_cgroup_mode (err); + if (UNLIKELY (cgroup_mode < 0)) + return cgroup_mode; + + if (cgroup_mode == CGROUP_MODE_UNIFIED) + ret = append_paths (&freezer_path, err, CGROUP_ROOT, source_cgroup_path, NULL); + else + ret = append_paths (&freezer_path, err, CGROUP_ROOT "/freezer", source_cgroup_path, NULL); + if (UNLIKELY (ret < 0)) + return ret; + + ret = libcriu_wrapper->criu_set_freeze_cgroup (freezer_path); + if (UNLIKELY (ret < 0)) + return crun_make_error (err, -ret, "CRIU: failed setting tfork freezer %d", ret); + if (def->root != NULL && def->root->path != NULL) { ret = append_paths (&rootfs_path, err, container->context ? container->context->bundle : ".", @@ -1564,7 +1682,7 @@ libcrun_container_tfork_linux_criu (libcrun_container_t *container, libcrun_chec } } - if (cr_options->tfork_copies > 1) + if (cr_options->tfork_copies >= 1) libcriu_wrapper->criu_set_tfork_copies (cr_options->tfork_copies); if (cr_options->tfork_memdump_async) @@ -1600,6 +1718,13 @@ libcrun_container_tfork_linux_criu (libcrun_container_t *container, libcrun_chec libcriu_wrapper->criu_set_tfork_full_memcopy (true); } + if (libcriu_wrapper->criu_set_network_lock && cr_options->network_lock_method > 0) + { + ret = libcriu_wrapper->criu_set_network_lock (cr_options->network_lock_method); + if (UNLIKELY (ret < 0)) + return crun_make_error (err, 0, "CRIU: failed setting tfork network lock"); + } + if (cr_options->track_mem || cr_options->tfork_memdump) libcriu_wrapper->criu_set_track_mem (true); @@ -1777,10 +1902,17 @@ libcrun_container_tfork_linux_criu (libcrun_container_t *container, libcrun_chec if (UNLIKELY (ret < 0)) return ret; + tfork_pre_restore_fd = cr_options->tfork_pre_restore_fd; + tfork_source_detached_fd = cr_options->tfork_source_detached_fd; + libcriu_wrapper->criu_set_notify_cb (criu_notify); ret = libcriu_wrapper->criu_tfork(); + tfork_pre_restore_fd = -1; + tfork_source_detached_fd = -1; if (UNLIKELY (ret != 0)) { show_criu_log (cr_options->work_path, CRIU_TFORK_LOG_FILE); + show_criu_log (cr_options->image_path, CRIU_TFORK_RESTORE_LOG_FILE); + show_criu_tfork_restore_copy_logs (cr_options->image_path, cr_options->tfork_copies); return crun_make_error (err, 0, "criu_tfork failed: %d", ret); } @@ -1791,7 +1923,7 @@ libcrun_container_tfork_linux_criu (libcrun_container_t *container, libcrun_chec pid_t clone_pid; const char *pidfile_name = "tfork.pid"; - if (cr_options->tfork_copies > 1) + if (cr_options->tfork_copies >= 1) pidfile_name = "tfork.pid.copy0"; ret = append_paths (&pidfile_path, err, cr_options->image_path, pidfile_name, NULL); @@ -1806,7 +1938,7 @@ libcrun_container_tfork_linux_criu (libcrun_container_t *container, libcrun_chec if (UNLIKELY (clone_pid <= 0)) return crun_make_error (err, 0, "invalid clone PID %d in `%s`", (int) clone_pid, pidfile_path); - if (cr_options->tfork_copies > 1) + if (cr_options->tfork_copies >= 1) { char children_path[64]; cleanup_free char *children_buf = NULL; diff --git a/crun/src/tfork.c b/crun/src/tfork.c index 31db969ef..f0524dbc3 100644 --- a/crun/src/tfork.c +++ b/crun/src/tfork.c @@ -56,7 +56,10 @@ enum OPTION_TFORK_TCP_CLOSE, OPTION_TFORK_SNAP_MOUNT, OPTION_TFORK_DUMPD_PARENT, + OPTION_TFORK_PRE_RESTORE_FD, + OPTION_TFORK_SOURCE_DETACHED_FD, OPTION_TFORK_FULL_MEMCOPY, + OPTION_NETWORK_LOCK_METHOD, OPTION_PID_FILE, OPTION_CONSOLE_SOCKET, OPTION_NO_PIVOT, @@ -83,7 +86,7 @@ static struct argp_option options[] { "parent-path", OPTION_PARENT_PATH, "DIR", 0, "previous criu images dir, for incremental memdump chains", 0 }, { "tfork-memdump", OPTION_TFORK_MEMDUMP, 0, 0, "dump pages-*.img to image-path during tfork", 0 }, { "tfork-memdump-async", OPTION_TFORK_MEMDUMP_ASYNC, 0, 0, "async page dump (implies --tfork-memdump)", 0 }, - { "tfork-copies", OPTION_TFORK_COPIES, "N", 0, "produce N parallel clones (default 1)", 0 }, + { "tfork-copies", OPTION_TFORK_COPIES, "N", 0, "produce N clones through the n-copy helper (omitted: legacy direct single-copy)", 0 }, { "track-mem", OPTION_TRACK_MEM, 0, 0, "arm soft-dirty for chained incremental dumps", 0 }, { "manage-cgroups-mode", OPTION_MANAGE_CGROUPS_MODE, "MODE", 0, "cgroups mode: 'soft' (default), 'ignore', 'full' and 'strict'", 0 }, @@ -101,8 +104,14 @@ static struct argp_option options[] "dump ESTABLISHED TCP sockets as closed; the clone wakes with sockets in closed state (apps reconnect). Required for chromium / electron / any networked workload that holds long-poll connections.", 0 }, { "tfork-dumpd-parent", OPTION_TFORK_DUMPD_PARENT, "PID", 0, "host-pidns PID for dumpd to reparent itself to (so `podman stop`/`rm` cleans dumpd up via its parent). Only meaningful with --tfork-memdump-async. If unset, defaults to getppid() at exec time (conmon when invoked from podman). Same-pidns is enforced by criu — pass a host PID. See ../criu/Documentation/CRIU_TFORK_RPC_PEER_DISCONNECT_LEAK.md.", 0 }, + { "tfork-pre-restore-fd", OPTION_TFORK_PRE_RESTORE_FD, "FD", 0, + "block at the Phase A/Phase B boundary until one byte can be read from FD", 0 }, + { "tfork-source-detached-fd", OPTION_TFORK_SOURCE_DETACHED_FD, "FD", 0, + "write one byte to FD after CRIU has safely detached from the source tree", 0 }, { "tfork-full-memcopy", OPTION_TFORK_FULL_MEMCOPY, 0, 0, "ablation knob: replace anon-private vma_cherrypick CoW with a userspace physical copy of every anon page (via pread from /proc//mem in the restorer). File-backed VMAs, memfds, and SysV IPC are unaffected. For measuring the anon-CoW signal vs full-copy baseline.", 0 }, + { "network-lock", OPTION_NETWORK_LOCK_METHOD, "METHOD", 0, + "network lock backend: 'iptables', 'nftables', or 'skip'", 0 }, { "cgroup-root", OPTION_CGROUP_ROOT, "[CTRL:]PATH", 0, "rewrite dumped cgroup paths under PATH on restore. Under tfork the rewrite synthesizes cgns_prefix so prepare_cgns() unshares CLONE_NEWCGROUP at the clone's own cgroup boundary. Required for recursive clone-of-clone (gen-1 → gen-2) so /proc/self/cgroup reads `/` inside gen-1.", 0 }, { "pid-file", OPTION_PID_FILE, "FILE", 0, "where to write the PID of the container", 0 }, @@ -189,10 +198,37 @@ parse_opt (int key, char *arg, struct argp_state *state) } break; + case OPTION_TFORK_PRE_RESTORE_FD: + { + char *endp; + const char *p = argp_mandatory_argument (arg, state); + long val = strtol (p, &endp, 10); + if (*endp != '\0' || val < 0 || val > INT_MAX) + libcrun_fail_with_error (0, "--tfork-pre-restore-fd: invalid FD `%s`", p); + cr_options.tfork_pre_restore_fd = (int) val; + } + break; + + case OPTION_TFORK_SOURCE_DETACHED_FD: + { + char *endp; + const char *p = argp_mandatory_argument (arg, state); + long val = strtol (p, &endp, 10); + if (*endp != '\0' || val < 0 || val > INT_MAX) + libcrun_fail_with_error (0, "--tfork-source-detached-fd: invalid FD `%s`", p); + cr_options.tfork_source_detached_fd = (int) val; + } + break; + case OPTION_TFORK_FULL_MEMCOPY: cr_options.tfork_full_memcopy = true; break; + case OPTION_NETWORK_LOCK_METHOD: + cr_options.network_lock_method + = crun_parse_network_lock_method (argp_mandatory_argument (arg, state)); + break; + case OPTION_TRACK_MEM: cr_options.track_mem = true; break; @@ -359,7 +395,16 @@ int crun_command_tfork (struct crun_global_arguments *global_args, int argc, char **argv, libcrun_error_t *err) { cr_options.manage_cgroups_mode = -1; - cr_options.tfork_copies = 1; + cr_options.tfork_copies = 0; + cr_options.tfork_pre_restore_fd = -1; + cr_options.tfork_source_detached_fd = -1; + /* + * External Unix stream sockets can make Codex/tmux stacks dumpable, but they + * may hide unsupported socket topology. Keep the default fail-loud and expose + * this as an explicit escape hatch for agent integrations. + */ + if (getenv ("CRUN_TFORK_EXT_UNIX_SK") != NULL) + cr_options.ext_unix_sk = true; cr_options.leave_running = true; return crun_run_create_internal (global_args, argc, argv, container_tfork, get_options, &crun_context, &run_argp, diff --git a/linux-pagecache-cow/build_kernel.sh b/linux-pagecache-cow/build_kernel.sh index c36ce0a59..94bbaeefd 100755 --- a/linux-pagecache-cow/build_kernel.sh +++ b/linux-pagecache-cow/build_kernel.sh @@ -9,6 +9,7 @@ version="7.0.1" # Append a suffix LocalVersion="-pgcachecow" num_cores=$(($(nproc --all) - 2)) +num_cores=$(( num_cores > 1 ? num_cores : 1 )) ## Functions delete_old_kernel_contents () { diff --git a/podman/cmd/podman/containers/clone.go b/podman/cmd/podman/containers/clone.go index e0264ade5..8851e973b 100644 --- a/podman/cmd/podman/containers/clone.go +++ b/podman/cmd/podman/containers/clone.go @@ -1,7 +1,9 @@ package containers import ( + jsonencoding "encoding/json" "fmt" + "os" "github.com/containers/podman/v5/cmd/podman/common" "github.com/containers/podman/v5/cmd/podman/registry" @@ -59,6 +61,15 @@ func cloneFlags(cmd *cobra.Command) { tforkOverlayBtrfsFlagName := "tfork-overlay-btrfs" flags.BoolVar(&ctrClone.TforkOverlayBtrfs, tforkOverlayBtrfsFlagName, false, "use overlay-on-btrfs for per-clone rootfs (default: per-clone btrfs subvolume snapshot; overlay shares lower's page cache across siblings; requires --live)") + tforkMetadataFlagName := "tfork-metadata" + flags.BoolVar(&ctrClone.TforkMetadata, tforkMetadataFlagName, false, "print live-clone identity, PID, and rootfs metadata as JSON (only with --live)") + + tforkInjectFileFlagName := "tfork-inject-file" + flags.StringSliceVar(&ctrClone.TforkInjectFiles, tforkInjectFileFlagName, nil, "inject COPY_INDEX:HOST_PATH:CONTAINER_PATH after live restore and before publication") + + tforkInjectSourceFileFlagName := "tfork-inject-source-file" + flags.StringSliceVar(&ctrClone.TforkInjectSourceFiles, tforkInjectSourceFileFlagName, nil, "inject HOST_PATH:CONTAINER_PATH into the frozen source after rootfs cloning") + tforkGhostLimitFlagName := "tfork-ghost-limit" flags.UintVar(&ctrClone.TforkGhostLimit, tforkGhostLimitFlagName, 256<<20, "raise CRIU's ghost-file size cap (bytes); GUI apps need >1MiB default (only with --live)") @@ -68,6 +79,9 @@ func cloneFlags(cmd *cobra.Command) { tforkFullMemcopyFlagName := "tfork-full-memcopy" flags.BoolVar(&ctrClone.TforkFullMemcopy, tforkFullMemcopyFlagName, false, "ablation: physical-copy anon-private VMAs instead of CoW (for measuring the anon-CoW signal; only with --live)") + tforkNetworkLockFlagName := "tfork-network-lock" + flags.StringVar(&ctrClone.TforkNetworkLock, tforkNetworkLockFlagName, "nftables", "network lock backend: iptables or nftables (only with --live)") + common.DefineCreateDefaults(&ctrClone.CreateOpts) common.DefineCreateFlags(cmd, &ctrClone.CreateOpts, entities.CloneMode) } @@ -140,6 +154,15 @@ func clone(cmd *cobra.Command, args []string) error { if ctrClone.TforkOverlayBtrfs { return fmt.Errorf("--tfork-overlay-btrfs requires --live: %w", define.ErrInvalidArg) } + if ctrClone.TforkMetadata { + return fmt.Errorf("--tfork-metadata requires --live: %w", define.ErrInvalidArg) + } + if len(ctrClone.TforkInjectFiles) > 0 { + return fmt.Errorf("--tfork-inject-file requires --live: %w", define.ErrInvalidArg) + } + if len(ctrClone.TforkInjectSourceFiles) > 0 { + return fmt.Errorf("--tfork-inject-source-file requires --live: %w", define.ErrInvalidArg) + } } ctrClone.ID = args[0] @@ -148,6 +171,9 @@ func clone(cmd *cobra.Command, args []string) error { if err != nil { return err } + if ctrClone.TforkMetadata { + return jsonencoding.NewEncoder(os.Stdout).Encode(rep.TforkClones) + } fmt.Println(rep.Id) return nil } diff --git a/podman/hack/tfork-network-lock-ab.sh b/podman/hack/tfork-network-lock-ab.sh new file mode 100755 index 000000000..208415bc7 --- /dev/null +++ b/podman/hack/tfork-network-lock-ab.sh @@ -0,0 +1,107 @@ +#!/usr/bin/env bash +set -euo pipefail + +PODMAN=${PODMAN:-podman} +PODMAN_GLOBAL_ARGS=${PODMAN_GLOBAL_ARGS:-} +CRIU_ROOT=${CRIU_ROOT:?set CRIU_ROOT} +OS4AGENT_CRUN=${OS4AGENT_CRUN:-crun} +SAMPLES=${SAMPLES:-10} +WORKLOAD_PROCESSES=${WORKLOAD_PROCESSES:-100} +IMAGE=${IMAGE:-docker.io/library/alpine:3.19} +PREFIX=${PREFIX:-tfork-network-lock-$RANDOM} +OUTPUT=${OUTPUT:-/tmp/tfork-network-lock-ab.tsv} +LOG_DIR=${LOG_DIR:-} + +read -r -a podman_global_args <<<"$PODMAN_GLOBAL_ARGS" +source_name=${PREFIX}-source + +podman_cmd() { + "$PODMAN" "${podman_global_args[@]}" "$@" +} + +cleanup() { + podman_cmd ps -a --format '{{.Names}}' | + awk -v prefix="$PREFIX" 'index($0, prefix) == 1' | + while read -r name; do + podman_cmd rm -f -t 0 "$name" >/dev/null 2>&1 || true + done +} +trap cleanup EXIT + +run_clone() { + local backend=$1 + local index=$2 + local name=${PREFIX}-${backend}-${index} + local started ended elapsed rootfs bundle + + started=$(date +%s%N) + env \ + PATH="$CRIU_ROOT/criu:$PATH" \ + LD_LIBRARY_PATH="$CRIU_ROOT/lib/c${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" \ + OS4AGENT_CRUN="$OS4AGENT_CRUN" \ + "$PODMAN" "${podman_global_args[@]}" container clone \ + --live --tfork-overlay-btrfs \ + --tfork-network-lock="$backend" \ + "$source_name" "$name" >/dev/null + ended=$(date +%s%N) + elapsed=$(( (ended - started) / 1000000 )) + + [[ $(podman_cmd exec "$name" cat /tmp/sentinel) == source-before-fork ]] + if [[ -n $LOG_DIR ]]; then + rootfs=$(podman_cmd inspect --format '{{.Rootfs}}' "$name") + bundle=$(dirname "$rootfs") + test -f "$bundle/img/tfork.log" + cp "$bundle/img/tfork.log" \ + "$LOG_DIR/${backend}-${index}.tfork.log" + fi + podman_cmd kill -s KILL "$name" >/dev/null + podman_cmd rm -f -t 0 "$name" >/dev/null + printf '%s\t%d\t%d\n' "$backend" "$index" "$elapsed" | + tee -a "$OUTPUT" +} + +cleanup +: >"$OUTPUT" +if [[ -n $LOG_DIR ]]; then + mkdir -p "$LOG_DIR" +fi +podman_cmd run -d --name "$source_name" \ + --log-driver k8s-file \ + --security-opt seccomp=unconfined \ + --security-opt apparmor=unconfined \ + "$IMAGE" sh -c \ + 'count=$1 + i=1 + while [ "$i" -lt "$count" ]; do + sleep 86400 & + i=$((i + 1)) + done + echo source-before-fork >/tmp/sentinel + exec tail -f /dev/null' sh "$WORKLOAD_PROCESSES" >/dev/null + +for ((attempt = 0; attempt < 100; attempt++)); do + actual_processes=$(podman_cmd top "$source_name" pid | + awk 'NR > 1 { count++ } END { print count + 0 }') + if ((actual_processes == WORKLOAD_PROCESSES)); then + break + fi + sleep 0.05 +done +if ((actual_processes != WORKLOAD_PROCESSES)); then + printf 'expected %d source processes, found %d\n' \ + "$WORKLOAD_PROCESSES" "$actual_processes" >&2 + exit 1 +fi + +for ((i = 1; i <= SAMPLES; i++)); do + if ((i % 2)); then + run_clone iptables "$i" + run_clone nftables "$i" + else + run_clone nftables "$i" + run_clone iptables "$i" + fi +done + +podman_cmd rm -f -t 0 "$source_name" >/dev/null +trap - EXIT diff --git a/podman/hack/tfork-network-lock-smoke.sh b/podman/hack/tfork-network-lock-smoke.sh new file mode 100755 index 000000000..83abf5f88 --- /dev/null +++ b/podman/hack/tfork-network-lock-smoke.sh @@ -0,0 +1,103 @@ +#!/usr/bin/env bash +set -euo pipefail + +PODMAN=${PODMAN:-podman} +PODMAN_GLOBAL_ARGS=${PODMAN_GLOBAL_ARGS:-} +CRIU_ROOT=${CRIU_ROOT:?set CRIU_ROOT} +OS4AGENT_CRUN=${OS4AGENT_CRUN:-crun} +IMAGE=${IMAGE:-docker.io/library/alpine:3.19} +PREFIX=${PREFIX:-tfork-network-smoke-$RANDOM} + +read -r -a podman_global_args <<<"$PODMAN_GLOBAL_ARGS" +source_name=${PREFIX}-source +clone_name=${PREFIX}-clone +unsafe_name=${PREFIX}-unsafe + +podman_cmd() { + "$PODMAN" "${podman_global_args[@]}" "$@" +} + +cleanup() { + podman_cmd rm -f -t 0 "$clone_name" >/dev/null 2>&1 || true + podman_cmd rm -f -t 0 "$unsafe_name" >/dev/null 2>&1 || true + podman_cmd rm -f -t 0 "$source_name" >/dev/null 2>&1 || true +} +trap cleanup EXIT + +assert_no_criu_table() { + local name=$1 + local pid + + pid=$(podman_cmd inspect --format '{{.State.Pid}}' "$name") + if nsenter -t "$pid" -n nft list tables 2>/dev/null | + grep -qi criu; then + printf 'stale CRIU nftables table in %s\n' "$name" >&2 + return 1 + fi +} + +cleanup +podman_cmd run -d --name "$source_name" \ + --log-driver k8s-file \ + --security-opt seccomp=unconfined \ + --security-opt apparmor=unconfined \ + "$IMAGE" sh -c \ + 'echo source-before-fork >/tmp/sentinel + while :; do + printf "HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok" | + nc -l -p 18080 + done' >/dev/null + +for ((attempt = 0; attempt < 100; attempt++)); do + if [[ $(podman_cmd exec "$source_name" \ + wget -qO- http://127.0.0.1:18080 2>/dev/null || true) == ok ]]; then + break + fi + sleep 0.05 +done +if ((attempt == 100)); then + printf 'source listener did not become ready\n' >&2 + exit 1 +fi +assert_no_criu_table "$source_name" + +if env \ + PATH="$CRIU_ROOT/criu:$PATH" \ + LD_LIBRARY_PATH="$CRIU_ROOT/lib/c${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" \ + OS4AGENT_CRUN="$OS4AGENT_CRUN" \ + "$PODMAN" "${podman_global_args[@]}" container clone \ + --live --tfork-overlay-btrfs \ + --tfork-network-lock=skip \ + "$source_name" "$unsafe_name" >/dev/null 2>&1; then + printf 'unsafe skip network backend was accepted\n' >&2 + exit 1 +fi +[[ $(podman_cmd exec "$source_name" cat /tmp/sentinel) == source-before-fork ]] + +env \ + PATH="$CRIU_ROOT/criu:$PATH" \ + LD_LIBRARY_PATH="$CRIU_ROOT/lib/c${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" \ + OS4AGENT_CRUN="$OS4AGENT_CRUN" \ + "$PODMAN" "${podman_global_args[@]}" container clone \ + --live --tfork-overlay-btrfs \ + --tfork-network-lock=nftables \ + "$source_name" "$clone_name" >/dev/null + +[[ $(podman_cmd exec "$clone_name" cat /tmp/sentinel) == source-before-fork ]] +[[ $(podman_cmd exec "$source_name" \ + wget -qO- http://127.0.0.1:18080) == ok ]] +[[ $(podman_cmd exec "$clone_name" \ + wget -qO- http://127.0.0.1:18080) == ok ]] +assert_no_criu_table "$source_name" +assert_no_criu_table "$clone_name" + +podman_cmd exec "$clone_name" sh -c 'echo clone-only >/tmp/divergence' +test "$(podman_cmd exec "$clone_name" cat /tmp/divergence)" = clone-only +if podman_cmd exec "$source_name" test -e /tmp/divergence; then + printf 'clone filesystem write leaked into source\n' >&2 + exit 1 +fi + +cleanup +trap - EXIT +printf 'tfork nftables network-lock smoke: PASS\n' diff --git a/podman/hack/tfork-transaction-smoke.sh b/podman/hack/tfork-transaction-smoke.sh new file mode 100755 index 000000000..8008fb68f --- /dev/null +++ b/podman/hack/tfork-transaction-smoke.sh @@ -0,0 +1,102 @@ +#!/usr/bin/env bash +set -euo pipefail + +PODMAN=${PODMAN:-podman} +PODMAN_GLOBAL_ARGS=${PODMAN_GLOBAL_ARGS:-} +IMAGE=${IMAGE:-docker.io/library/alpine:3.19} +PREFIX=${PREFIX:-tfork-txn-$RANDOM} +TFORK_RUNTIME_ROOT=${TFORK_RUNTIME_ROOT:-/run/libpod/tfork} + +source_name=${PREFIX}-source +clone_name=${PREFIX}-clone +read -r -a podman_global_args <<<"$PODMAN_GLOBAL_ARGS" + +podman_cmd() { + "$PODMAN" "${podman_global_args[@]}" "$@" +} + +count_dirs() { + local path=$1 + if [[ ! -d $path ]]; then + echo 0 + return + fi + find "$path" -mindepth 1 -maxdepth 1 -type d -print | wc -l +} + +cleanup() { + podman_cmd kill -s KILL "$clone_name" >/dev/null 2>&1 || true + podman_cmd rm -f -t 0 "$clone_name" >/dev/null 2>&1 || true + podman_cmd kill -s KILL "$source_name" >/dev/null 2>&1 || true + podman_cmd rm -f -t 0 "$source_name" >/dev/null 2>&1 || true +} +trap cleanup EXIT + +runtime_before=$(count_dirs "$TFORK_RUNTIME_ROOT") +bundle_root=$(podman_cmd info --format '{{.Store.GraphRoot}}')/tfork-bundles +bundles_before=$(count_dirs "$bundle_root") + +podman_cmd run -d --name "$source_name" \ + --log-driver k8s-file \ + --security-opt seccomp=unconfined \ + --security-opt apparmor=unconfined \ + "$IMAGE" sh -c 'echo source-before-fork >/tmp/sentinel; exec tail -f /dev/null' >/dev/null +cgroups_before=$(find /sys/fs/cgroup -type d -name 'libpod-*' -print 2>/dev/null | wc -l) + +for stage in \ + before_freeze \ + after_freeze \ + after_filesystem \ + before_restore \ + after_restore \ + after_thaw \ + after_register_0 \ + before_commit +do + if env PODMAN_TFORK_FAULT_INJECT="$stage" \ + "$PODMAN" "${podman_global_args[@]}" container clone --live --tfork-overlay-btrfs \ + "$source_name" "$clone_name" >/tmp/tfork-fault.stdout 2>/tmp/tfork-fault.stderr + then + echo "fault stage $stage unexpectedly succeeded" >&2 + exit 1 + fi + if ! grep -Fq "injected tfork fault at $stage" /tmp/tfork-fault.stderr; then + echo "fault stage $stage was not reached; clone failed for another reason" >&2 + sed -n '1,120p' /tmp/tfork-fault.stderr >&2 + exit 1 + fi + podman_cmd exec "$source_name" sh -c \ + 'test "$(cat /tmp/sentinel)" = source-before-fork' + if podman_cmd container exists "$clone_name"; then + echo "fault stage $stage published clone $clone_name" >&2 + exit 1 + fi + if [[ $(count_dirs "$bundle_root") != "$bundles_before" ]]; then + echo "fault stage $stage leaked a graphroot bundle" >&2 + exit 1 + fi + if [[ $(count_dirs "$TFORK_RUNTIME_ROOT") != "$runtime_before" ]]; then + echo "fault stage $stage leaked a runtime publication" >&2 + exit 1 + fi + if [[ $(find /sys/fs/cgroup -type d -name 'libpod-*' -print 2>/dev/null | wc -l) != "$cgroups_before" ]]; then + echo "fault stage $stage leaked a cgroup" >&2 + exit 1 + fi + echo "PASS fault=$stage" +done + +podman_cmd container clone --live --tfork-overlay-btrfs \ + "$source_name" "$clone_name" >/dev/null +[[ $(podman_cmd exec "$clone_name" cat /tmp/sentinel) == source-before-fork ]] +podman_cmd exec "$clone_name" sh -c 'echo child-only >/tmp/sentinel' +[[ $(podman_cmd exec "$source_name" cat /tmp/sentinel) == source-before-fork ]] + +podman_cmd kill -s KILL "$clone_name" >/dev/null +podman_cmd rm -f -t 0 "$clone_name" >/dev/null +if [[ $(count_dirs "$TFORK_RUNTIME_ROOT") != "$runtime_before" ]]; then + echo "successful clone cleanup leaked a runtime publication" >&2 + exit 1 +fi + +echo "PASS normal-fork" diff --git a/podman/libpod/container_internal_linux.go b/podman/libpod/container_internal_linux.go index 29159584d..d87256fc7 100644 --- a/podman/libpod/container_internal_linux.go +++ b/podman/libpod/container_internal_linux.go @@ -528,6 +528,22 @@ func (c *Container) CleanupExternalCloneStorage() { return } + // Direct n-copy tfork uses a small runtime-only publication directory for + // attach/log plumbing. It is outside the graphroot bundle and therefore + // was not covered by the storage teardown below. + runtimeBundle := filepath.Clean(c.config.ExternalBundlePath) + runtimeRoot := filepath.Clean("/run/libpod/tfork") + if runtimeBundle != "" && runtimeBundle != "." && + strings.HasPrefix(runtimeBundle, runtimeRoot+string(os.PathSeparator)) { + if err := os.RemoveAll(runtimeBundle); err != nil && !errors.Is(err, fs.ErrNotExist) { + logrus.Debugf("tfork: clone %s rm runtime bundle %s: %v", c.ID(), runtimeBundle, err) + } + parent := filepath.Dir(runtimeBundle) + if entries, err := os.ReadDir(parent); err == nil && len(entries) == 0 { + _ = os.Remove(parent) + } + } + if hpid := c.config.TforkDumpdHolderPid; hpid > 0 { expectedStart := c.config.TforkDumpdHolderStartTime curStart, stErr := ReadProcStartTime(hpid) diff --git a/podman/libpod/oci_conmon_common.go b/podman/libpod/oci_conmon_common.go index 0b521a180..2871078c7 100644 --- a/podman/libpod/oci_conmon_common.go +++ b/podman/libpod/oci_conmon_common.go @@ -828,6 +828,18 @@ func (r *ConmonOCIRuntime) CheckpointContainer(ctr *Container, options Container func (r *ConmonOCIRuntime) CheckConmonRunning(ctr *Container) (bool, error) { if ctr.state.ConmonPID == 0 { + // Direct tfork clones deliberately use a lightweight exit watcher + // instead of conmon. Their init PID, not a missing conmon PID, is the + // authoritative liveness signal. + if ctr.config.ExternalSetup && ctr.state.PID > 0 { + if err := unix.Kill(ctr.state.PID, 0); err != nil { + if errors.Is(err, unix.ESRCH) { + return false, nil + } + return false, fmt.Errorf("pinging external clone pid %d: %w", ctr.state.PID, err) + } + return true, nil + } // If the container is running or paused, assume Conmon is // running. We didn't record Conmon PID on some old versions, so // that is likely what's going on... diff --git a/podman/libpod/runtime_ctr.go b/podman/libpod/runtime_ctr.go index fafaac15a..0cb13dbfd 100644 --- a/podman/libpod/runtime_ctr.go +++ b/podman/libpod/runtime_ctr.go @@ -150,6 +150,17 @@ func (r *Runtime) RegisterExternalContainer(ctx context.Context, rSpec *spec.Spe } ctr.valid = true + createdPaths := make([]string, 0, 2) + defer func() { + if retErr == nil { + return + } + for i := len(createdPaths) - 1; i >= 0; i-- { + if err := os.RemoveAll(createdPaths[i]); err != nil { + logrus.Errorf("Removing path for failed external container registration %s: %v", createdPaths[i], err) + } + } + }() if ctr.config.StaticDir == "" { sd := filepath.Join(r.config.Engine.StaticDir, "containers", ctr.ID(), "userdata") @@ -157,11 +168,13 @@ func (r *Runtime) RegisterExternalContainer(ctx context.Context, rSpec *spec.Spe return nil, fmt.Errorf("creating static dir: %w", err) } ctr.config.StaticDir = sd + createdPaths = append(createdPaths, filepath.Dir(sd)) } rd := filepath.Join(r.storageConfig.RunRoot, "containers", ctr.ID(), "userdata") if err := os.MkdirAll(rd, 0o700); err != nil { return nil, fmt.Errorf("creating run dir: %w", err) } + createdPaths = append(createdPaths, filepath.Dir(rd)) ctr.state.RunDir = rd if ctr.config.ConmonPidFile == "" { diff --git a/podman/pkg/domain/entities/containers.go b/podman/pkg/domain/entities/containers.go index 8295ffe98..89b83eb50 100644 --- a/podman/pkg/domain/entities/containers.go +++ b/podman/pkg/domain/entities/containers.go @@ -238,7 +238,15 @@ type RestoreOptions struct { type RestoreReport = types.RestoreReport type ContainerCreateReport struct { - Id string + Id string `json:"Id"` + TforkClones []TforkCloneMetadata `json:"TforkClones,omitempty"` +} + +type TforkCloneMetadata struct { + ID string `json:"id"` + Name string `json:"name"` + PID int `json:"pid"` + Rootfs string `json:"rootfs"` } // AttachOptions describes the cli and other values @@ -495,15 +503,19 @@ type ContainerCloneOptions struct { Run bool Force bool - Live bool - Copies int - Persistent string - WithPrevious bool - SharedUsr bool - TforkGhostLimit uint - TforkTCPClose bool - TforkFullMemcopy bool - TforkOverlayBtrfs bool + Live bool + Copies int + Persistent string + WithPrevious bool + SharedUsr bool + TforkGhostLimit uint + TforkTCPClose bool + TforkFullMemcopy bool + TforkNetworkLock string + TforkOverlayBtrfs bool + TforkMetadata bool + TforkInjectFiles []string + TforkInjectSourceFiles []string } // ContainerUpdateOptions containers options for updating an existing containers cgroup configuration diff --git a/podman/pkg/domain/infra/abi/container_tfork.go b/podman/pkg/domain/infra/abi/container_tfork.go index 8aa6926d5..331ea8db3 100644 --- a/podman/pkg/domain/infra/abi/container_tfork.go +++ b/podman/pkg/domain/infra/abi/container_tfork.go @@ -4,11 +4,14 @@ import ( "bufio" "context" "encoding/json" + "errors" "fmt" + "io" "net" "os" "os/exec" "path/filepath" + "strconv" "strings" "syscall" "time" @@ -16,17 +19,91 @@ import ( "github.com/containers/podman/v5/libpod" "github.com/containers/podman/v5/libpod/define" - "strconv" + "github.com/cyphar/filepath-securejoin/pathrs-lite" "github.com/containers/podman/v5/pkg/domain/entities" "github.com/containers/podman/v5/utils" spec "github.com/opencontainers/runtime-spec/specs-go" "github.com/sirupsen/logrus" "go.podman.io/common/libnetwork/types" + commonconfig "go.podman.io/common/pkg/config" "go.podman.io/storage/pkg/stringid" "golang.org/x/sys/unix" ) +const ( + tforkSourceFreezeTimeout = 10 * time.Second + tforkSourceThawTimeout = 10 * time.Second + tforkCloneReadyTimeout = 60 * time.Second + tforkCgroupPollInterval = 50 * time.Millisecond + tforkClonePollInterval = 200 * time.Millisecond +) + +type tforkSourceSyncMode string + +const ( + tforkSourceSyncFS tforkSourceSyncMode = "syncfs" + tforkSourceSyncGlobal tforkSourceSyncMode = "global" + tforkSourceSyncNone tforkSourceSyncMode = "none" +) + +func tforkSourceSyncModeFromEnv() (tforkSourceSyncMode, error) { + value := strings.ToLower(strings.TrimSpace(os.Getenv("PODMAN_TFORK_SYNC_MODE"))) + if value == "" { + return tforkSourceSyncFS, nil + } + mode := tforkSourceSyncMode(value) + switch mode { + case tforkSourceSyncFS, tforkSourceSyncGlobal, tforkSourceSyncNone: + return mode, nil + default: + return "", fmt.Errorf("invalid PODMAN_TFORK_SYNC_MODE=%q (must be syncfs, global, or none)", value) + } +} + +func tforkSyncSource(rootfs string) error { + mode, err := tforkSourceSyncModeFromEnv() + if err != nil { + return err + } + + switch mode { + case tforkSourceSyncNone: + logrus.Infof("tfork: source sync disabled by PODMAN_TFORK_SYNC_MODE=none") + return nil + case tforkSourceSyncGlobal: + if out, err := exec.Command("sync").CombinedOutput(); err != nil { + return fmt.Errorf("global sync: %s: %w", strings.TrimSpace(string(out)), err) + } + return nil + case tforkSourceSyncFS: + root, err := os.Open(rootfs) + if err != nil { + return fmt.Errorf("open source rootfs %s for syncfs: %w", rootfs, err) + } + defer root.Close() + if err := unix.Syncfs(int(root.Fd())); err != nil { + return fmt.Errorf("syncfs source rootfs %s: %w", rootfs, err) + } + return nil + default: + return fmt.Errorf("unsupported source sync mode %q", mode) + } +} + +func tforkCloneReadyTimeoutFromEnv() time.Duration { + value := strings.TrimSpace(os.Getenv("PODMAN_TFORK_CLONE_READY_TIMEOUT_SECS")) + if value == "" { + return tforkCloneReadyTimeout + } + seconds, err := strconv.Atoi(value) + if err != nil || seconds <= 0 { + logrus.Warnf("tfork: ignoring invalid PODMAN_TFORK_CLONE_READY_TIMEOUT_SECS=%q", value) + return tforkCloneReadyTimeout + } + return time.Duration(seconds) * time.Second +} + func (ic *ContainerEngine) containerCloneLive(ctx context.Context, opts entities.ContainerCloneOptions) (rep *entities.ContainerCreateReport, retErr error) { src, err := ic.Libpod.LookupContainer(opts.ID) if err != nil { @@ -40,11 +117,30 @@ func (ic *ContainerEngine) containerCloneLive(ctx context.Context, opts entities if state != define.ContainerStateRunning { return nil, fmt.Errorf("source %q is not running (state=%s); tfork requires a live source", src.ID(), state.String()) } + if manager := src.CgroupManager(); manager != commonconfig.CgroupfsCgroupsManager { + return nil, fmt.Errorf("tfork currently requires the cgroupfs cgroup manager (source uses %q); retry Podman with --cgroup-manager=cgroupfs", manager) + } copies := opts.Copies if copies <= 0 { copies = 1 } + requestedCopies := copies + fileInjections, err := tforkParseFileInjections(opts.TforkInjectFiles, requestedCopies) + if err != nil { + return nil, err + } + sourceFileInjections, err := tforkParseSourceFileInjections(opts.TforkInjectSourceFiles) + if err != nil { + return nil, err + } + useSingleCopyConmon := requestedCopies == 1 && os.Getenv("PODMAN_TFORK_SINGLE_COPY_CONMON") == "1" + // PODMAN_TFORK_SINGLE_COPY_DIRECT is a debugging escape hatch that skips + // the n-copy restore helper for single-copy experiments. Production paths + // keep the n-copy helper even for copies=1 so attach/status handling is + // consistent with multi-copy forks. + useSingleCopyDirect := requestedCopies == 1 && os.Getenv("PODMAN_TFORK_SINGLE_COPY_DIRECT") == "1" + useNcopyRestore := copies > 1 || (requestedCopies == 1 && !useSingleCopyConmon && !useSingleCopyDirect) var srcRootfs string if cfg := src.Config(); cfg != nil && cfg.ExternalSetup && cfg.Rootfs != "" { @@ -66,11 +162,29 @@ func (ic *ContainerEngine) containerCloneLive(ctx context.Context, opts entities if err := os.MkdirAll(bundleDir, 0o700); err != nil { return nil, fmt.Errorf("mkdir bundle: %w", err) } + txn := newTforkCloneTransaction(ctx, ic.Libpod, src, bundleDir, copies) + defer func() { + if retErr != nil { + txn.rollback(retErr) + } + }() snapRO := filepath.Join(bundleDir, "snap-ro") - if out, err := exec.Command("sync").CombinedOutput(); err != nil { - return nil, fmt.Errorf("sync: %s: %w", out, err) + if err := tforkInjectFault("before_freeze"); err != nil { + return nil, err + } + thawSource, err := tforkFreezeSourceCgroup(src, tforkSourceFreezeTimeout) + if err != nil { + return nil, fmt.Errorf("freeze source cgroup before tfork snapshot: %w", err) + } + txn.setSourceRestore(thawSource) + if err := tforkInjectFault("after_freeze"); err != nil { + return nil, err + } + + if err := tforkSyncSource(srcRootfs); err != nil { + return nil, err } recursive := false @@ -116,6 +230,7 @@ func (ic *ContainerEngine) containerCloneLive(ctx context.Context, opts entities cloneIDs := make([]string, copies) cloneRootfsList := make([]string, copies) cloneRootfsRel := make([]string, copies) + overlapSocketPurge := os.Getenv("PODMAN_TFORK_OVERLAP_SOCKET_PURGE") == "1" for i := 0; i < copies; i++ { cloneIDs[i] = stringid.GenerateRandomID() var rel string @@ -139,8 +254,10 @@ func (ic *ContainerEngine) containerCloneLive(ctx context.Context, opts entities return nil, fmt.Errorf("setup snap-subvol rootfs for copy %d: %w", i, err) } } - if err := tforkPurgeSockets(cloneRootfsList[i]); err != nil { - logrus.Warnf("tfork: purge sockets in %s: %v (proceeding)", cloneRootfsList[i], err) + if !overlapSocketPurge { + if err := tforkPurgeSockets(cloneRootfsList[i]); err != nil { + logrus.Warnf("tfork: purge sockets in %s: %v (proceeding)", cloneRootfsList[i], err) + } } if srcPID, perr := src.PID(); perr == nil && srcPID > 0 { @@ -157,6 +274,48 @@ func (ic *ContainerEngine) containerCloneLive(ctx context.Context, opts entities } } } + if len(sourceFileInjections) > 0 { + srcPID, err := src.PID() + if err != nil { + return nil, fmt.Errorf("read frozen source PID for file injection: %w", err) + } + for _, injection := range sourceFileInjections { + if err := tforkInjectFileIntoProcessRoot(srcPID, injection); err != nil { + return nil, fmt.Errorf("inject file into frozen source: %w", err) + } + } + } + var socketPurgeRead *os.File + var socketPurgeFinished chan struct{} + if overlapSocketPurge { + readEnd, writeEnd, err := os.Pipe() + if err != nil { + return nil, fmt.Errorf("create socket-purge barrier: %w", err) + } + socketPurgeRead = readEnd + socketPurgeFinished = make(chan struct{}) + manifestPath := filepath.Join(bundleDir, "socket-paths.manifest0") + go func() { + defer close(socketPurgeFinished) + defer writeEnd.Close() + if err := tforkPurgeSocketsWithManifest(cloneRootfsList, manifestPath); err != nil { + logrus.Warnf("tfork: manifest socket purge: %v (proceeding)", err) + } + if _, err := writeEnd.Write([]byte{1}); err != nil { + logrus.Warnf("tfork: signal socket-purge completion: %v", err) + } + }() + defer func() { + <-socketPurgeFinished + if socketPurgeRead != nil { + _ = socketPurgeRead.Close() + } + }() + } + txn.setCloneIDs(cloneIDs) + if err := tforkInjectFault("after_filesystem"); err != nil { + return nil, err + } if recursive && parentUpperFrozen != "" { if err := os.RemoveAll(parentUpperFrozen); err != nil { @@ -260,6 +419,7 @@ func (ic *ContainerEngine) containerCloneLive(ctx context.Context, opts entities if err := os.MkdirAll(imgDir, 0o700); err != nil { return nil, fmt.Errorf("mkdir img: %w", err) } + txn.setImageDir(imgDir) cloneCgroupPaths := make([]string, copies) if cfg := src.Config(); cfg != nil { @@ -270,8 +430,10 @@ func (ic *ContainerEngine) containerCloneLive(ctx context.Context, opts entities return nil, fmt.Errorf("mkdir clone cgroup %s: %w", cgFS, err) } cloneCgroupPaths[i] = cgRel + txn.setCgroupPaths(cloneCgroupPaths) } } + txn.setCgroupPaths(cloneCgroupPaths) srcStatePath := fmt.Sprintf("/run/crun/%s/status", src.ID()) crunArgs := []string{ @@ -280,7 +442,7 @@ func (ic *ContainerEngine) containerCloneLive(ctx context.Context, opts entities "--source-state", srcStatePath, "--image-path", imgDir, } - if copies == 1 { + if !useNcopyRestore { crunArgs = append(crunArgs, "--tfork-snap-root", cloneRootfsList[0]) crunArgs = append(crunArgs, "--tfork-snap-mount", "/") if cloneCgroupPaths[0] != "" { @@ -338,6 +500,15 @@ func (ic *ContainerEngine) containerCloneLive(ctx context.Context, opts entities default: return nil, fmt.Errorf("invalid --persistent value %q (must be \"async\" or \"sync\")", opts.Persistent) } + if opts.TforkNetworkLock == "" { + opts.TforkNetworkLock = "nftables" + } + switch opts.TforkNetworkLock { + case "iptables", "nftables": + crunArgs = append(crunArgs, "--network-lock", opts.TforkNetworkLock) + default: + return nil, fmt.Errorf("invalid --tfork-network-lock value %q (must be \"iptables\" or \"nftables\")", opts.TforkNetworkLock) + } dumpdHolderPid := 0 var dumpdHolderStartTime uint64 if opts.Persistent == "async" { @@ -384,6 +555,9 @@ func (ic *ContainerEngine) containerCloneLive(ctx context.Context, opts entities crunArgs = append(crunArgs, "--parent-path", parentImgDir) logrus.Infof("tfork: --with-previous chains off clone %s (imgDir=%s)", parentCloneID, parentImgDir) } + if err := tforkInjectFault("before_restore"); err != nil { + return nil, err + } cloneLogPaths := make([]string, copies) cloneConmonPids := make([]int, copies) @@ -400,9 +574,16 @@ func (ic *ContainerEngine) containerCloneLive(ctx context.Context, opts entities } }() hasTTY := len(ttySrcFds) > 0 - useConmon := copies == 1 + // Keep the legacy single-copy conmon bootstrap opt-in because it can fail + // before crun starts and report only `conmon reported pid=-1`. + useConmon := copies == 1 && useSingleCopyConmon if useConmon { + if socketPurgeRead != nil { + <-socketPurgeFinished + _ = socketPurgeRead.Close() + socketPurgeRead = nil + } conmonInheritFds := inheritFds if hasTTY { conmonInheritFds = filterOutTtyInheritFds(inheritFds) @@ -425,6 +606,7 @@ func (ic *ContainerEngine) containerCloneLive(ctx context.Context, opts entities ghostLimit: opts.TforkGhostLimit, tcpClose: opts.TforkTCPClose, fullMemcopy: opts.TforkFullMemcopy, + networkLock: opts.TforkNetworkLock, cgroupRoot: cloneCgroupPaths[0], dumpdHolderPid: dumpdHolderPid, }) @@ -433,9 +615,29 @@ func (ic *ContainerEngine) containerCloneLive(ctx context.Context, opts entities } cloneLogPaths[0] = logPath cloneConmonPids[0] = conmonPid + txn.trackPID(conmonPid, "conmon") logrus.Infof("tfork: clone %s up via conmon pid=%d; log=%s", cloneIDs[0], conmonPid, logPath) } else { directArgs := append([]string{}, crunArgs...) + eventReadiness := os.Getenv("PODMAN_TFORK_EVENT_READINESS") == "1" + var sourceDetachedRead *os.File + var sourceDetachedWrite *os.File + var sourceDetachedDone chan error + if eventReadiness { + var err error + sourceDetachedRead, sourceDetachedWrite, err = os.Pipe() + if err != nil { + return nil, fmt.Errorf("create source-detached event pipe: %w", err) + } + defer func() { + if sourceDetachedRead != nil { + _ = sourceDetachedRead.Close() + } + if sourceDetachedWrite != nil { + _ = sourceDetachedWrite.Close() + } + }() + } if dumpdHolderPid > 0 { directArgs = append(directArgs, fmt.Sprintf("--tfork-dumpd-parent=%d", dumpdHolderPid)) @@ -443,8 +645,8 @@ func (ic *ContainerEngine) containerCloneLive(ctx context.Context, opts entities var perCopyExtraFiles []*os.File var perCopyReadEnds []*os.File var perCopyArgs []string - skipTtySrcFds := copies > 1 && hasTTY - if copies > 1 { + skipTtySrcFds := useNcopyRestore && hasTTY + if useNcopyRestore { ifdsForPerCopy, stdioKeys := splitStdioInheritFds(inheritFds) inheritFds = ifdsForPerCopy extraFDBase := 3 @@ -477,6 +679,17 @@ func (ic *ContainerEngine) containerCloneLive(ctx context.Context, opts entities for _, pcArg := range perCopyArgs { directArgs = append(directArgs, pcArg) } + nextExtraFD := 3 + len(perCopyExtraFiles) + if hasTTY && !skipTtySrcFds { + nextExtraFD += len(ttySrcFds) + } + if socketPurgeRead != nil { + directArgs = append(directArgs, "--tfork-pre-restore-fd", strconv.Itoa(nextExtraFD)) + nextExtraFD++ + } + if sourceDetachedWrite != nil { + directArgs = append(directArgs, "--tfork-source-detached-fd", strconv.Itoa(nextExtraFD)) + } directArgs = append(directArgs, cloneIDs[0]) crunCmd := exec.Command(defaultCrunPath, directArgs...) crunCmd.Stdin = nil @@ -486,6 +699,12 @@ func (ic *ContainerEngine) containerCloneLive(ctx context.Context, opts entities if len(perCopyExtraFiles) > 0 { crunCmd.ExtraFiles = append(crunCmd.ExtraFiles, perCopyExtraFiles...) } + if socketPurgeRead != nil { + crunCmd.ExtraFiles = append(crunCmd.ExtraFiles, socketPurgeRead) + } + if sourceDetachedWrite != nil { + crunCmd.ExtraFiles = append(crunCmd.ExtraFiles, sourceDetachedWrite) + } logPath := filepath.Join(bundleDir, "crun-tfork.log") logF, err := os.Create(logPath) if err != nil { @@ -497,6 +716,25 @@ func (ic *ContainerEngine) containerCloneLive(ctx context.Context, opts entities logF.Close() return nil, fmt.Errorf("start crun tfork: %w", err) } + if socketPurgeRead != nil { + _ = socketPurgeRead.Close() + socketPurgeRead = nil + } + if sourceDetachedWrite != nil { + _ = sourceDetachedWrite.Close() + sourceDetachedWrite = nil + sourceDetachedDone = make(chan error, 1) + go func() { + var byte [1]byte + n, err := sourceDetachedRead.Read(byte[:]) + if err == nil && n != 1 { + err = fmt.Errorf("short source-detached event read: %d bytes", n) + } + sourceDetachedDone <- err + }() + } + txn.trackPID(crunCmd.Process.Pid, "crun-tfork") + crunDone := make(chan error, 1) var crunAborted bool defer func() { if !crunAborted && retErr != nil { @@ -507,7 +745,7 @@ func (ic *ContainerEngine) containerCloneLive(ctx context.Context, opts entities _ = f.Close() } go func() { - _ = crunCmd.Wait() + crunDone <- crunCmd.Wait() logF.Close() }() perCopyAttachSocks := make([]*os.File, copies) @@ -516,6 +754,7 @@ func (ic *ContainerEngine) containerCloneLive(ctx context.Context, opts entities shortBatch = shortBatch[:12] } runtimeAttachBase := filepath.Join("/run/libpod/tfork", shortBatch) + txn.setRuntimeBatchDir(runtimeAttachBase) for i := 0; i < copies; i++ { perCopyBundle := filepath.Join(runtimeAttachBase, fmt.Sprintf("%d", i)) if err := os.MkdirAll(perCopyBundle, 0o700); err != nil { @@ -538,8 +777,11 @@ func (ic *ContainerEngine) containerCloneLive(ctx context.Context, opts entities } else { cloneLogPath = filepath.Join(bundleDir, fmt.Sprintf("clone.%d.log", i)) } - if err := spawnTforkStdioHelper(readEnd, perCopyAttachSocks[i], cloneLogPath, hasTTY); err != nil { + helperPID, err := spawnTforkStdioHelper(readEnd, perCopyAttachSocks[i], cloneLogPath, hasTTY) + if err != nil { logrus.Warnf("tfork: per-copy %d stdio-helper spawn: %v", i, err) + } else { + txn.trackPID(helperPID, fmt.Sprintf("stdio-helper-%d", i)) } if perCopyAttachSocks[i] != nil { _ = perCopyAttachSocks[i].Close() @@ -549,57 +791,163 @@ func (ic *ContainerEngine) containerCloneLive(ctx context.Context, opts entities } pidFileFor := func(i int) string { - if copies == 1 { + if !useNcopyRestore { return filepath.Join(imgDir, "tfork.pid") } return filepath.Join(imgDir, fmt.Sprintf("tfork.pid.copy%d", i)) } statePath := fmt.Sprintf("/run/crun/%s/status", cloneIDs[0]) - needState := copies == 1 - deadline := time.Now().Add(60 * time.Second) + needState := !useNcopyRestore + cloneReadyTimeout := tforkCloneReadyTimeoutFromEnv() + deadline := time.Now().Add(cloneReadyTimeout) readyCopies := 0 stateReady := !needState - for readyCopies < copies || !stateReady { + crunExited := false + var crunErr error + if eventReadiness { + timer := time.NewTimer(cloneReadyTimeout) + defer timer.Stop() + sourceDetached := false + for !crunExited || !sourceDetached { + select { + case detachErr := <-sourceDetachedDone: + if detachErr != nil { + tforkAbortCrunCmd(crunCmd, src, bundleDir, copies) + crunAborted = true + return nil, fmt.Errorf("wait for tfork source-detached event: %w; see %s", detachErr, logPath) + } + sourceDetached = true + if err := txn.restoreSourceOnce(); err != nil { + tforkAbortCrunCmd(crunCmd, src, bundleDir, copies) + crunAborted = true + return nil, fmt.Errorf("early thaw at tfork source-detached event: %w", err) + } + logrus.Infof("tfork: source thawed at CRIU source-detached event") + case crunErr = <-crunDone: + crunExited = true + if crunErr != nil { + tforkAbortCrunCmd(crunCmd, src, bundleDir, copies) + crunAborted = true + return nil, fmt.Errorf("crun tfork failed before event readiness: %w; see %s", crunErr, logPath) + } + case <-timer.C: + tforkAbortCrunCmd(crunCmd, src, bundleDir, copies) + crunAborted = true + return nil, fmt.Errorf("timeout waiting %s for tfork source-detached and runtime-exit events; see %s", + cloneReadyTimeout, logPath) + } + } + readyCopies = 0 for i := 0; i < copies; i++ { if _, err := os.Stat(pidFileFor(i)); err == nil { readyCopies++ } } - if needState && !stateReady { - if _, err := os.Stat(statePath); err == nil { - stateReady = true - } + if needState { + _, err := os.Stat(statePath) + stateReady = err == nil } - if readyCopies >= copies && stateReady { - break + if readyCopies < copies || !stateReady { + return nil, fmt.Errorf("crun exited successfully but clone readiness artifacts are incomplete: pidfiles=%d/%d stateReady=%v; see %s", + readyCopies, copies, stateReady, logPath) } - if crunCmd.ProcessState != nil && crunCmd.ProcessState.Exited() && readyCopies < copies { - tforkAbortCrunCmd(crunCmd, src, bundleDir, copies) - crunAborted = true - return nil, fmt.Errorf("crun tfork exited before %d clones came up (got %d); see %s", - copies, readyCopies, logPath) + } else { + for readyCopies < copies || !stateReady { + if !crunExited { + select { + case crunErr = <-crunDone: + crunExited = true + default: + } + } + readyCopies = 0 + for i := 0; i < copies; i++ { + if _, err := os.Stat(pidFileFor(i)); err == nil { + readyCopies++ + } + } + if needState && !stateReady { + if _, err := os.Stat(statePath); err == nil { + stateReady = true + } + } + if readyCopies >= copies && stateReady { + break + } + if crunExited && readyCopies < copies { + tforkAbortCrunCmd(crunCmd, src, bundleDir, copies) + crunAborted = true + return nil, fmt.Errorf("crun tfork exited before %d clones came up (got %d); see %s", + copies, readyCopies, logPath) + } + if time.Now().After(deadline) { + tforkAbortCrunCmd(crunCmd, src, bundleDir, copies) + crunAborted = true + return nil, fmt.Errorf("timeout waiting for %d tfork.pid* files in %s (got %d, stateReady=%v); see %s", + copies, imgDir, readyCopies, stateReady, logPath) + } + time.Sleep(tforkClonePollInterval) } - if time.Now().After(deadline) { - tforkAbortCrunCmd(crunCmd, src, bundleDir, copies) - crunAborted = true - return nil, fmt.Errorf("timeout waiting for %d tfork.pid* files in %s (got %d, stateReady=%v); see %s", - copies, imgDir, readyCopies, stateReady, logPath) + if !crunExited { + select { + case crunErr = <-crunDone: + crunExited = true + case <-time.After(cloneReadyTimeout): + tforkAbortCrunCmd(crunCmd, src, bundleDir, copies) + crunAborted = true + return nil, fmt.Errorf("timeout waiting %s for crun tfork to finish after %d clones came up; see %s", + cloneReadyTimeout, copies, logPath) + } } - time.Sleep(200 * time.Millisecond) + } + if crunErr != nil { + tforkAbortCrunCmd(crunCmd, src, bundleDir, copies) + crunAborted = true + return nil, fmt.Errorf("crun tfork failed after %d clones came up: %w; see %s", + copies, crunErr, logPath) } logrus.Infof("tfork: batch %s up (N=%d); crun-tfork.log at %s", batchID, copies, logPath) } + if err := tforkInjectFault("after_restore"); err != nil { + return nil, err + } + + if err := txn.restoreSourceOnce(); err != nil { + logrus.Warnf("tfork: clones are up, but thawing source cgroup after restore failed: %v", err) + return nil, fmt.Errorf("thaw source cgroup after tfork restore: %w", err) + } + if err := tforkInjectFault("after_thaw"); err != nil { + return nil, err + } + srcPID, err := src.PID() + if err != nil { + return nil, fmt.Errorf("read source PID after restore: %w", err) + } + if err := tforkPIDRunning(srcPID); err != nil { + return nil, fmt.Errorf("source is not usable after tfork restore: %w", err) + } srcCfg := src.Config() if srcCfg == nil { return nil, fmt.Errorf("source %q: could not read libpod config", src.ID()) } + visibleCloneIDs := make([]string, 0, requestedCopies) + visibleClones := make([]entities.TforkCloneMetadata, 0, requestedCopies) for i, cloneID := range cloneIDs { - clonePID, err := readTforkClonePID(cloneID, imgDir, i, copies) + clonePID, err := readTforkClonePID(cloneID, imgDir, i, copies, useNcopyRestore) if err != nil { return nil, fmt.Errorf("read clone %d PID: %w", i, err) } + txn.trackPID(clonePID, fmt.Sprintf("clone-init-%d", i)) + if err := tforkPIDRunning(clonePID); err != nil { + return nil, fmt.Errorf("clone %d is not running before publication: %w", i, err) + } + for _, injection := range fileInjections[i] { + if err := tforkInjectFileIntoProcessRoot(clonePID, injection); err != nil { + return nil, fmt.Errorf("inject clone %d file %s: %w", i, injection.destination, err) + } + } cloneCfg, err := buildCloneContainerConfig(srcCfg, cloneID, cloneNames[i], cloneRootfsList[i], cloneSpecs[i]) if err != nil { return nil, fmt.Errorf("build clone %d config: %w", i, err) @@ -633,16 +981,17 @@ func (ic *ContainerEngine) containerCloneLive(ctx context.Context, opts entities if err != nil { return nil, fmt.Errorf("register clone %d (%s) in libpod state: %w", i, cloneID, err) } + txn.addRegistered(ctr) if opts.TforkOverlayBtrfs { tforkFreezeUpperForRollback(bundleDir, i, copies) } if cloneConmonPids[i] > 0 { if err := ctr.SetConmonPID(cloneConmonPids[i]); err != nil { - logrus.Warnf("tfork: clone %s SetConmonPID(%d): %v", cloneID, cloneConmonPids[i], err) + return nil, fmt.Errorf("clone %s SetConmonPID(%d): %w", cloneID, cloneConmonPids[i], err) } } if err := ic.Libpod.SetupExternalCloneNetwork(src, ctr, clonePID); err != nil { - logrus.Warnf("tfork: clone %s network setup failed (clone has no network): %v", cloneID, err) + return nil, fmt.Errorf("clone %s network setup: %w", cloneID, err) } if cmd := exec.Command("nsenter", "-t", strconv.Itoa(clonePID), "-n", "--", @@ -653,17 +1002,307 @@ func (ic *ContainerEngine) containerCloneLive(ctx context.Context, opts entities } } if err := ctr.MoveExternalCloneToOwnCgroup(clonePID); err != nil { - logrus.Warnf("tfork: clone %s F8 cgroup migration failed: %v (clone shares source's cgroup)", cloneID, err) + return nil, fmt.Errorf("clone %s cgroup migration: %w", cloneID, err) } if cloneConmonPids[i] == 0 { - if err := spawnTforkExitWatcher(ctx, ic.Libpod, ctr, clonePID); err != nil { - logrus.Warnf("tfork: clone %s exit-watcher spawn: %v (podman will fall back to F4.2 /proc liveness check)", cloneID, err) + watcherPID, err := spawnTforkExitWatcher(ctx, ic.Libpod, ctr, clonePID) + if err != nil { + return nil, fmt.Errorf("tfork: clone %s exit-watcher spawn: %w", cloneID, err) } + txn.trackPID(watcherPID, fmt.Sprintf("exit-watcher-%d", i)) + } + if err := tforkPIDRunning(clonePID); err != nil { + return nil, fmt.Errorf("clone %d died during publication: %w", i, err) + } + if err := tforkInjectFault(fmt.Sprintf("after_register_%d", i)); err != nil { + return nil, err } logrus.Infof("tfork: clone %s (%s) registered in libpod state, pid=%d", cloneID, cloneNames[i], clonePID) + visibleCloneIDs = append(visibleCloneIDs, cloneID) + visibleClones = append(visibleClones, entities.TforkCloneMetadata{ + ID: cloneID, + Name: cloneNames[i], + PID: clonePID, + Rootfs: cloneRootfsList[i], + }) + } + + if err := tforkInjectFault("before_commit"); err != nil { + return nil, err } + txn.commit() + return &entities.ContainerCreateReport{ + Id: strings.Join(visibleCloneIDs, "\n"), + TforkClones: visibleClones, + }, nil +} - return &entities.ContainerCreateReport{Id: strings.Join(cloneIDs, "\n")}, nil +type tforkFileInjection struct { + source string + destination string +} + +func tforkParseFileInjections(specs []string, copies int) (map[int][]tforkFileInjection, error) { + parsed := make(map[int][]tforkFileInjection) + for _, spec := range specs { + parts := strings.SplitN(spec, ":", 3) + if len(parts) != 3 { + return nil, fmt.Errorf("invalid --tfork-inject-file %q (expected COPY_INDEX:HOST_PATH:CONTAINER_PATH)", spec) + } + copyIndex, err := strconv.Atoi(parts[0]) + if err != nil || copyIndex < 0 || copyIndex >= copies { + return nil, fmt.Errorf("invalid --tfork-inject-file copy index %q for %d copies", parts[0], copies) + } + source := filepath.Clean(parts[1]) + destination := filepath.Clean(parts[2]) + if !filepath.IsAbs(source) || source == string(os.PathSeparator) { + return nil, fmt.Errorf("tfork injection source must be a non-root absolute path: %q", parts[1]) + } + if !filepath.IsAbs(destination) || destination == string(os.PathSeparator) { + return nil, fmt.Errorf("tfork injection destination must be a non-root absolute path: %q", parts[2]) + } + parsed[copyIndex] = append(parsed[copyIndex], tforkFileInjection{ + source: source, + destination: destination, + }) + } + return parsed, nil +} + +func tforkParseSourceFileInjections(specs []string) ([]tforkFileInjection, error) { + parsed := make([]tforkFileInjection, 0, len(specs)) + for _, spec := range specs { + parts := strings.SplitN(spec, ":", 2) + if len(parts) != 2 { + return nil, fmt.Errorf("invalid --tfork-inject-source-file %q (expected HOST_PATH:CONTAINER_PATH)", spec) + } + source := filepath.Clean(parts[0]) + destination := filepath.Clean(parts[1]) + if !filepath.IsAbs(source) || source == string(os.PathSeparator) { + return nil, fmt.Errorf("tfork source injection source must be a non-root absolute path: %q", parts[0]) + } + if !filepath.IsAbs(destination) || destination == string(os.PathSeparator) { + return nil, fmt.Errorf("tfork source injection destination must be a non-root absolute path: %q", parts[1]) + } + parsed = append(parsed, tforkFileInjection{source: source, destination: destination}) + } + return parsed, nil +} + +func tforkInjectFileIntoProcessRoot(pid int, injection tforkFileInjection) error { + const maximumInjectionSize = 1 << 20 + sourceFD, err := unix.Open(injection.source, unix.O_RDONLY|unix.O_CLOEXEC|unix.O_NOFOLLOW, 0) + if err != nil { + return fmt.Errorf("open source %s: %w", injection.source, err) + } + source := os.NewFile(uintptr(sourceFD), injection.source) + defer source.Close() + info, err := source.Stat() + if err != nil { + return fmt.Errorf("stat source %s: %w", injection.source, err) + } + if !info.Mode().IsRegular() || info.Size() > maximumInjectionSize { + return fmt.Errorf("source must be a regular file no larger than %d bytes", maximumInjectionSize) + } + + root := filepath.Join("/proc", strconv.Itoa(pid), "root") + parent, err := pathrs.OpenInRoot(root, filepath.Dir(injection.destination)) + if err != nil { + return fmt.Errorf("open destination parent: %w", err) + } + defer parent.Close() + destinationFD, err := unix.Openat( + int(parent.Fd()), + filepath.Base(injection.destination), + unix.O_WRONLY|unix.O_CREAT|unix.O_TRUNC|unix.O_CLOEXEC|unix.O_NOFOLLOW, + 0o600, + ) + if err != nil { + return fmt.Errorf("open destination: %w", err) + } + destination := os.NewFile(uintptr(destinationFD), injection.destination) + defer destination.Close() + if err := destination.Chmod(0o600); err != nil { + return fmt.Errorf("chmod destination: %w", err) + } + if _, err := io.Copy(destination, source); err != nil { + return fmt.Errorf("copy payload: %w", err) + } + return nil +} + +type tforkCgroupFreezer struct { + root string + statePath string + eventsPath string + version string +} + +func tforkFreezeSourceCgroup(src *libpod.Container, timeout time.Duration) (func() error, error) { + if src == nil { + return nil, fmt.Errorf("source container is nil") + } + cgPath, err := src.CgroupPath() + if err != nil { + return nil, fmt.Errorf("read source cgroup path: %w", err) + } + cgPath = strings.TrimPrefix(filepath.Clean(cgPath), string(os.PathSeparator)) + if cgPath == "" || cgPath == "." { + return nil, fmt.Errorf("source cgroup path is empty") + } + freezer, err := tforkSourceCgroupFreezer(cgPath) + if err != nil { + return nil, err + } + + originalFrozen, err := freezer.frozen() + if err != nil { + return nil, err + } + if err := freezer.freeze(); err != nil { + return nil, err + } + if err := freezer.waitFrozen(true, timeout); err != nil { + if !originalFrozen { + _ = freezer.thaw() + } + return nil, err + } + logrus.Infof("tfork: froze source cgroup for snapshot consistency: %s (%s)", freezer.root, freezer.version) + + thaw := func() error { + if originalFrozen { + return freezer.freeze() + } + if err := freezer.thaw(); err != nil { + return err + } + if err := freezer.waitFrozen(false, tforkSourceThawTimeout); err != nil { + return err + } + logrus.Infof("tfork: thawed source cgroup after clone restore: %s (%s)", freezer.root, freezer.version) + return nil + } + return thaw, nil +} + +func tforkSourceCgroupFreezer(cgPath string) (*tforkCgroupFreezer, error) { + v2Root := filepath.Join("/sys/fs/cgroup", cgPath) + v2FreezePath := filepath.Join(v2Root, "cgroup.freeze") + v2EventsPath := filepath.Join(v2Root, "cgroup.events") + if _, err := os.Stat(v2FreezePath); err == nil { + return &tforkCgroupFreezer{ + root: v2Root, + statePath: v2FreezePath, + eventsPath: v2EventsPath, + version: "cgroup v2", + }, nil + } else if !os.IsNotExist(err) { + return nil, fmt.Errorf("stat %s: %w", v2FreezePath, err) + } + + v1Root := filepath.Join("/sys/fs/cgroup/freezer", cgPath) + v1StatePath := filepath.Join(v1Root, "freezer.state") + if _, err := os.Stat(v1StatePath); err == nil { + return &tforkCgroupFreezer{ + root: v1Root, + statePath: v1StatePath, + version: "cgroup v1 freezer", + }, nil + } else if !os.IsNotExist(err) { + return nil, fmt.Errorf("stat %s: %w", v1StatePath, err) + } + + return nil, fmt.Errorf("source cgroup freezer not found for %q; expected cgroup v2 %s or cgroup v1 %s", + cgPath, v2FreezePath, v1StatePath) +} + +func (f *tforkCgroupFreezer) frozen() (bool, error) { + data, err := os.ReadFile(f.statePath) + if err != nil { + return false, fmt.Errorf("read %s: %w", f.statePath, err) + } + frozen, ok := f.parseFrozen(data) + if !ok { + return false, fmt.Errorf("could not parse frozen state from %s", f.statePath) + } + return frozen, nil +} + +func (f *tforkCgroupFreezer) freeze() error { + value := []byte("1") + if f.version == "cgroup v1 freezer" { + value = []byte("FROZEN") + } + if err := os.WriteFile(f.statePath, value, 0o644); err != nil { + return fmt.Errorf("write %s=%s: %w", f.statePath, value, err) + } + return nil +} + +func (f *tforkCgroupFreezer) thaw() error { + value := []byte("0") + if f.version == "cgroup v1 freezer" { + value = []byte("THAWED") + } + if err := os.WriteFile(f.statePath, value, 0o644); err != nil { + return fmt.Errorf("write %s=%s: %w", f.statePath, value, err) + } + return nil +} + +func (f *tforkCgroupFreezer) waitFrozen(wantFrozen bool, timeout time.Duration) error { + deadline := time.Now().Add(timeout) + for { + path := f.statePath + if f.eventsPath != "" { + path = f.eventsPath + } + data, err := os.ReadFile(path) + if err != nil { + return fmt.Errorf("read %s: %w", path, err) + } + frozen, ok := f.parseFrozen(data) + if ok && frozen == wantFrozen { + return nil + } + if time.Now().After(deadline) { + return fmt.Errorf("timeout waiting %s for %s to report frozen=%t", timeout, path, wantFrozen) + } + time.Sleep(tforkCgroupPollInterval) + } +} + +func (f *tforkCgroupFreezer) parseFrozen(data []byte) (bool, bool) { + if f.version == "cgroup v1 freezer" { + switch strings.TrimSpace(string(data)) { + case "FROZEN": + return true, true + case "THAWED": + return false, true + } + return false, false + } + + switch strings.TrimSpace(string(data)) { + case "0": + return false, true + case "1": + return true, true + } + + for _, line := range strings.Split(string(data), "\n") { + fields := strings.Fields(line) + if len(fields) == 2 && fields[0] == "frozen" { + switch fields[1] { + case "0": + return false, true + case "1": + return true, true + } + } + } + return false, false } func spawnTforkDumpdHolder() (int, uint64, error) { @@ -689,23 +1328,40 @@ func spawnTforkDumpdHolder() (int, uint64, error) { func tforkAbortCrunCmd(crunCmd *exec.Cmd, src *libpod.Container, bundleDir string, copies int) { if crunCmd != nil && crunCmd.Process != nil { + protected := make(map[int]bool) + if src != nil { + if srcPID, err := src.PID(); err == nil && srcPID > 0 { + protected[srcPID] = true + for _, pid := range tforkCollectDescendants(srcPID) { + protected[pid] = true + } + } + } toKill := tforkCollectDescendants(crunCmd.Process.Pid) - toKill = append(toKill, crunCmd.Process.Pid) - for _, pid := range toKill { - _ = syscall.Kill(pid, syscall.SIGKILL) + // Give CRIU's service and restore helpers a chance to unwind ptrace, + // parasite, namespace, and cgyard state before killing their parent. + // Never signal a source-tree PID even if transient reparenting makes it + // appear below the runtime. + for i := len(toKill) - 1; i >= 0; i-- { + if !protected[toKill[i]] { + _ = syscall.Kill(toKill[i], syscall.SIGTERM) + } } - done := make(chan struct{}) - go func() { - _, _ = crunCmd.Process.Wait() - close(done) - }() - select { - case <-done: - case <-time.After(3 * time.Second): - logrus.Warnf("tfork: crun-tfork didn't exit in 3s after SIGKILL — source may still be ptraced") + if !tforkWaitPIDGone(crunCmd.Process.Pid, time.Second) { + for i := len(toKill) - 1; i >= 0; i-- { + if !protected[toKill[i]] { + _ = syscall.Kill(toKill[i], syscall.SIGKILL) + } + } + _ = syscall.Kill(crunCmd.Process.Pid, syscall.SIGKILL) + if !tforkWaitPIDGone(crunCmd.Process.Pid, 3*time.Second) { + logrus.Warnf("tfork: crun-tfork didn't exit after TERM/KILL escalation — source may still be ptraced") + } } for _, pid := range tforkCollectDescendants(crunCmd.Process.Pid) { - _ = syscall.Kill(pid, syscall.SIGKILL) + if !protected[pid] { + _ = syscall.Kill(pid, syscall.SIGKILL) + } } } if src != nil { @@ -725,6 +1381,19 @@ func tforkAbortCrunCmd(crunCmd *exec.Cmd, src *libpod.Container, bundleDir strin } } +func tforkWaitPIDGone(pid int, timeout time.Duration) bool { + deadline := time.Now().Add(timeout) + for { + if err := syscall.Kill(pid, 0); errors.Is(err, syscall.ESRCH) { + return true + } + if time.Now().After(deadline) { + return false + } + time.Sleep(20 * time.Millisecond) + } +} + func tforkCollectDescendants(root int) []int { entries, err := os.ReadDir("/proc") if err != nil { @@ -863,9 +1532,9 @@ while True: return nil } -func spawnTforkStdioHelper(readEnd *os.File, attachSock *os.File, logPath string, tty bool) error { +func spawnTforkStdioHelper(readEnd *os.File, attachSock *os.File, logPath string, tty bool) (int, error) { if err := os.MkdirAll(filepath.Dir(logPath), 0o755); err != nil { - return fmt.Errorf("mkdir %s: %w", filepath.Dir(logPath), err) + return 0, fmt.Errorf("mkdir %s: %w", filepath.Dir(logPath), err) } pyCode := `import os, sys, asyncio, datetime, socket, io, traceback LOG_PATH = os.environ["LOG_PATH"] @@ -985,13 +1654,15 @@ except Exception: env = append(env, fmt.Sprintf("ATTACH_FD=%d", attachFD)) cmd.Env = env if err := cmd.Start(); err != nil { - return fmt.Errorf("start stdio-helper: %w", err) + return 0, fmt.Errorf("start stdio-helper: %w", err) } + pid := 0 if cmd.Process != nil { + pid = cmd.Process.Pid _ = cmd.Process.Release() } logrus.Debugf("tfork: spawned stdio-helper for read-fd → %s (attach=%v, tty=%v)", logPath, attachSock != nil, tty) - return nil + return pid, nil } func boolToInt(b bool) int { @@ -1025,10 +1696,10 @@ func allocPerCopyAttachSocket(path string) (*os.File, error) { return f, nil } -func spawnTforkExitWatcher(ctx context.Context, rt *libpod.Runtime, ctr *libpod.Container, clonePID int) error { +func spawnTforkExitWatcher(ctx context.Context, rt *libpod.Runtime, ctr *libpod.Container, clonePID int) (int, error) { exitDir := "/run/libpod/exits" if err := os.MkdirAll(exitDir, 0o755); err != nil { - return fmt.Errorf("mkdir %s: %w", exitDir, err) + return 0, fmt.Errorf("mkdir %s: %w", exitDir, err) } exitFile := filepath.Join(exitDir, ctr.ID()) script := fmt.Sprintf(`while [ -d /proc/%d ]; do sleep 0.2; done; tmp=%s.tmp; printf 137 > "$tmp" && mv "$tmp" %s`, clonePID, exitFile, exitFile) @@ -1037,16 +1708,18 @@ func spawnTforkExitWatcher(ctx context.Context, rt *libpod.Runtime, ctr *libpod. cmd.Stdout = nil cmd.Stderr = nil if err := cmd.Start(); err != nil { - return fmt.Errorf("start exit-watcher: %w", err) + return 0, fmt.Errorf("start exit-watcher: %w", err) } + pid := 0 if cmd.Process != nil { + pid = cmd.Process.Pid _ = cmd.Process.Release() } logrus.Debugf("tfork: spawned exit-watcher for clone %s (PID %d) → %s", ctr.ID(), clonePID, exitFile) - return nil + return pid, nil } -func readTforkClonePID(cloneID string, imgDir string, copyIdx, copies int) (int, error) { +func readTforkClonePID(cloneID string, imgDir string, copyIdx, copies int, useNcopyRestore bool) (int, error) { statePath := fmt.Sprintf("/run/crun/%s/status", cloneID) if data, err := os.ReadFile(statePath); err == nil { var st struct { @@ -1057,7 +1730,7 @@ func readTforkClonePID(cloneID string, imgDir string, copyIdx, copies int) (int, } } var pidFile string - if copies == 1 { + if !useNcopyRestore { pidFile = filepath.Join(imgDir, "tfork.pid") } else { pidFile = filepath.Join(imgDir, fmt.Sprintf("tfork.pid.copy%d", copyIdx)) @@ -1070,7 +1743,7 @@ func readTforkClonePID(cloneID string, imgDir string, copyIdx, copies int) (int, if err != nil { return 0, err } - if copies == 1 { + if !useNcopyRestore { return rcPID, nil } initPID, err := readFirstChildPID(rcPID) @@ -1193,6 +1866,7 @@ type conmonForTforkOpts struct { ghostLimit uint tcpClose bool fullMemcopy bool + networkLock string cgroupRoot string dumpdHolderPid int } @@ -1418,6 +2092,10 @@ func spawnConmonForTfork(ctx context.Context, opts conmonForTforkOpts) (int, err if opts.fullMemcopy { tforkRuntimeOpts = append(tforkRuntimeOpts, "--tfork-full-memcopy") } + if opts.networkLock != "" { + tforkRuntimeOpts = append(tforkRuntimeOpts, + fmt.Sprintf("--network-lock=%s", opts.networkLock)) + } if opts.cgroupRoot != "" { tforkRuntimeOpts = append(tforkRuntimeOpts, fmt.Sprintf("--cgroup-root=%s", opts.cgroupRoot)) @@ -1750,6 +2428,41 @@ func tforkPurgeSockets(rootfs string) error { return nil } +func tforkPurgeSocketsWithManifest(rootfsList []string, manifestPath string) error { + if len(rootfsList) == 0 { + return fmt.Errorf("socket purge requires at least one clone rootfs") + } + started := time.Now() + cmd := exec.Command("find", rootfsList[0], "-mindepth", "1", "-type", "s", "-printf", "%P\\0") + out, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("find socket manifest in %s: %s: %w", rootfsList[0], strings.TrimSpace(string(out)), err) + } + if err := os.WriteFile(manifestPath, out, 0o600); err != nil { + return fmt.Errorf("write socket manifest %s: %w", manifestPath, err) + } + + count := 0 + for _, rel := range strings.Split(string(out), "\x00") { + if rel == "" { + continue + } + clean := filepath.Clean(rel) + if filepath.IsAbs(clean) || clean == ".." || strings.HasPrefix(clean, ".."+string(os.PathSeparator)) { + return fmt.Errorf("unsafe socket path %q in %s", rel, manifestPath) + } + count++ + for _, rootfs := range rootfsList { + if err := os.Remove(filepath.Join(rootfs, clean)); err != nil && !os.IsNotExist(err) { + return fmt.Errorf("remove socket %s from %s: %w", clean, rootfs, err) + } + } + } + logrus.Infof("tfork: socket manifest found %d path(s), applied to %d clone rootfs(es) in %s", + count, len(rootfsList), time.Since(started)) + return nil +} + func tforkBuildSkipMnts(srcPID int) []string { out := make([]string, 0, len(tforkPodmanSkipMnts)+8) seen := make(map[string]struct{}, len(tforkPodmanSkipMnts)) diff --git a/podman/pkg/domain/infra/abi/container_tfork_sync_test.go b/podman/pkg/domain/infra/abi/container_tfork_sync_test.go new file mode 100644 index 000000000..60cff2728 --- /dev/null +++ b/podman/pkg/domain/infra/abi/container_tfork_sync_test.go @@ -0,0 +1,37 @@ +package abi + +import "testing" + +func TestTforkSourceSyncModeFromEnv(t *testing.T) { + tests := []struct { + name string + value string + want tforkSourceSyncMode + wantErr bool + }{ + {name: "default", value: "", want: tforkSourceSyncFS}, + {name: "syncfs", value: "syncfs", want: tforkSourceSyncFS}, + {name: "case and whitespace", value: " Global\n", want: tforkSourceSyncGlobal}, + {name: "none", value: "none", want: tforkSourceSyncNone}, + {name: "invalid", value: "source", wantErr: true}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Setenv("PODMAN_TFORK_SYNC_MODE", test.value) + got, err := tforkSourceSyncModeFromEnv() + if test.wantErr { + if err == nil { + t.Fatalf("expected an error, got mode %q", got) + } + return + } + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if got != test.want { + t.Fatalf("mode = %q, want %q", got, test.want) + } + }) + } +} diff --git a/podman/pkg/domain/infra/abi/container_tfork_transaction.go b/podman/pkg/domain/infra/abi/container_tfork_transaction.go new file mode 100644 index 000000000..5497d5db9 --- /dev/null +++ b/podman/pkg/domain/infra/abi/container_tfork_transaction.go @@ -0,0 +1,288 @@ +//go:build !remote + +package abi + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "strconv" + "strings" + "syscall" + "time" + + "github.com/containers/podman/v5/libpod" + "github.com/sirupsen/logrus" + "golang.org/x/sys/unix" +) + +const tforkRollbackWait = 3 * time.Second + +type tforkTrackedPID struct { + pid int + startTime uint64 + role string +} + +// tforkCloneTransaction owns every host-side object created before a clone is +// published. Ownership is transferred to libpod only by commit. Until then, +// every return path converges on rollback. +type tforkCloneTransaction struct { + ctx context.Context + rt *libpod.Runtime + src *libpod.Container + + bundleDir string + runtimeBatchDir string + imgDir string + copies int + cloneIDs []string + cgroupPaths []string + + restoreSource func() error + sourceRestored bool + registered []*libpod.Container + pids []tforkTrackedPID + committed bool + rolledBack bool +} + +func newTforkCloneTransaction(ctx context.Context, rt *libpod.Runtime, src *libpod.Container, bundleDir string, copies int) *tforkCloneTransaction { + return &tforkCloneTransaction{ + ctx: ctx, + rt: rt, + src: src, + bundleDir: bundleDir, + copies: copies, + } +} + +func (t *tforkCloneTransaction) setCloneIDs(ids []string) { + t.cloneIDs = append([]string(nil), ids...) +} + +func (t *tforkCloneTransaction) setCgroupPaths(paths []string) { + t.cgroupPaths = append([]string(nil), paths...) +} + +func (t *tforkCloneTransaction) setRuntimeBatchDir(path string) { + t.runtimeBatchDir = path +} + +func (t *tforkCloneTransaction) setImageDir(path string) { + t.imgDir = path +} + +func (t *tforkCloneTransaction) setSourceRestore(restore func() error) { + t.restoreSource = restore +} + +func (t *tforkCloneTransaction) restoreSourceOnce() error { + if t.sourceRestored || t.restoreSource == nil { + return nil + } + if err := t.restoreSource(); err != nil { + return err + } + t.sourceRestored = true + return nil +} + +func (t *tforkCloneTransaction) trackPID(pid int, role string) { + if pid <= 0 { + return + } + for _, tracked := range t.pids { + if tracked.pid == pid { + return + } + } + startTime, err := libpod.ReadProcStartTime(pid) + if err != nil { + logrus.Debugf("tfork: transaction cannot record %s pid=%d start time: %v", role, pid, err) + return + } + t.pids = append(t.pids, tforkTrackedPID{pid: pid, startTime: startTime, role: role}) +} + +func (t *tforkCloneTransaction) addRegistered(ctr *libpod.Container) { + if ctr != nil { + t.registered = append(t.registered, ctr) + } +} + +func (t *tforkCloneTransaction) commit() { + t.committed = true +} + +func (t *tforkCloneTransaction) rollback(cause error) { + if t == nil || t.committed || t.rolledBack { + return + } + t.rolledBack = true + logrus.Warnf("tfork: rolling back unpublished clone transaction: %v", cause) + + // A registered external clone knows how to tear down its network, state, + // cgroup, and per-copy storage. Remove in reverse publication order. + zero := uint(0) + for i := len(t.registered) - 1; i >= 0; i-- { + ctr := t.registered[i] + if err := t.rt.RemoveContainer(context.WithoutCancel(t.ctx), ctr, true, false, &zero); err != nil { + logrus.Warnf("tfork: rollback remove registered clone %s: %v", ctr.ID(), err) + } + } + + t.trackRestorePIDs() + t.killCloneCgroups() + for i := len(t.pids) - 1; i >= 0; i-- { + t.killTrackedPID(t.pids[i]) + } + + if err := t.restoreSourceOnce(); err != nil { + logrus.Warnf("tfork: rollback restore source cgroup: %v", err) + } + + t.removeCloneCgroups() + for _, id := range t.cloneIDs { + _ = os.Remove(filepath.Join("/run/libpod/exits", id)) + _ = os.RemoveAll(filepath.Join("/run/crun", id)) + } + if t.runtimeBatchDir != "" { + if err := os.RemoveAll(t.runtimeBatchDir); err != nil { + logrus.Warnf("tfork: rollback remove runtime publication %s: %v", t.runtimeBatchDir, err) + } + tforkRemoveEmptyParent(filepath.Dir(t.runtimeBatchDir)) + } + if t.bundleDir != "" { + tforkBestEffortBundleReap(t.bundleDir, t.copies) + } +} + +func (t *tforkCloneTransaction) trackRestorePIDs() { + if t.imgDir == "" { + return + } + for i := 0; i < t.copies; i++ { + for _, path := range []string{ + filepath.Join(t.imgDir, "tfork.pid"), + filepath.Join(t.imgDir, fmt.Sprintf("tfork.pid.copy%d", i)), + } { + data, err := os.ReadFile(path) + if err != nil { + continue + } + pid, err := parsePIDBytes(data, path) + if err == nil { + t.trackPID(pid, "restore-child") + if child, err := readFirstChildPID(pid); err == nil { + t.trackPID(child, "clone-init") + } + } + } + } +} + +func (t *tforkCloneTransaction) killCloneCgroups() { + for _, rel := range t.cgroupPaths { + root := tforkCgroupFSPath(rel) + if root == "" { + continue + } + killPath := filepath.Join(root, "cgroup.kill") + if err := os.WriteFile(killPath, []byte("1"), 0o644); err != nil && !errors.Is(err, os.ErrNotExist) { + logrus.Debugf("tfork: rollback write %s: %v", killPath, err) + } + } +} + +func (t *tforkCloneTransaction) removeCloneCgroups() { + deadline := time.Now().Add(tforkRollbackWait) + for _, rel := range t.cgroupPaths { + root := tforkCgroupFSPath(rel) + if root == "" { + continue + } + for { + err := os.Remove(root) + if err == nil || errors.Is(err, os.ErrNotExist) { + break + } + if time.Now().After(deadline) { + logrus.Warnf("tfork: rollback cgroup remains at %s: %v", root, err) + break + } + time.Sleep(25 * time.Millisecond) + } + } +} + +func tforkCgroupFSPath(rel string) string { + clean := strings.TrimPrefix(filepath.Clean(rel), string(os.PathSeparator)) + if clean == "" || clean == "." || strings.HasPrefix(clean, "..") { + return "" + } + return filepath.Join("/sys/fs/cgroup", clean) +} + +func (t *tforkCloneTransaction) killTrackedPID(tracked tforkTrackedPID) { + current, err := libpod.ReadProcStartTime(tracked.pid) + if errors.Is(err, os.ErrNotExist) || errors.Is(err, unix.ESRCH) { + return + } + if err != nil { + logrus.Warnf("tfork: rollback cannot verify %s pid=%d: %v; refusing unsafe kill", tracked.role, tracked.pid, err) + return + } + if current != tracked.startTime { + logrus.Warnf("tfork: rollback %s pid=%d was recycled; refusing kill", tracked.role, tracked.pid) + return + } + descendants := tforkCollectDescendants(tracked.pid) + for i := len(descendants) - 1; i >= 0; i-- { + _ = syscall.Kill(descendants[i], syscall.SIGKILL) + } + _ = syscall.Kill(tracked.pid, syscall.SIGKILL) +} + +func tforkRemoveEmptyParent(path string) { + if path == "" { + return + } + entries, err := os.ReadDir(path) + if err == nil && len(entries) == 0 { + _ = os.Remove(path) + } +} + +func tforkPIDRunning(pid int) error { + if pid <= 0 { + return fmt.Errorf("invalid pid %d", pid) + } + data, err := os.ReadFile(filepath.Join("/proc", strconv.Itoa(pid), "stat")) + if err != nil { + return err + } + rp := strings.LastIndexByte(string(data), ')') + if rp < 0 || rp+2 >= len(data) { + return fmt.Errorf("malformed /proc/%d/stat", pid) + } + if data[rp+2] == 'Z' { + return fmt.Errorf("pid %d is a zombie", pid) + } + return nil +} + +func tforkInjectFault(stage string) error { + want := strings.TrimSpace(os.Getenv("PODMAN_TFORK_FAULT_INJECT")) + if want == "" { + return nil + } + for _, candidate := range strings.Split(want, ",") { + if strings.TrimSpace(candidate) == stage { + return fmt.Errorf("injected tfork fault at %s", stage) + } + } + return nil +} diff --git a/podman/pkg/domain/infra/abi/container_tfork_transaction_test.go b/podman/pkg/domain/infra/abi/container_tfork_transaction_test.go new file mode 100644 index 000000000..225929fa1 --- /dev/null +++ b/podman/pkg/domain/infra/abi/container_tfork_transaction_test.go @@ -0,0 +1,111 @@ +//go:build !remote + +package abi + +import ( + "errors" + "os" + "path/filepath" + "testing" +) + +func TestTforkInjectFault(t *testing.T) { + t.Setenv("PODMAN_TFORK_FAULT_INJECT", "after_freeze, before_commit") + for _, stage := range []string{"after_freeze", "before_commit"} { + if err := tforkInjectFault(stage); err == nil { + t.Fatalf("expected injected fault at %s", stage) + } + } + if err := tforkInjectFault("after_restore"); err != nil { + t.Fatalf("unexpected fault at unselected stage: %v", err) + } +} + +func TestTforkCgroupFSPath(t *testing.T) { + for _, unsafe := range []string{"", ".", "/", "..", "../escape"} { + if got := tforkCgroupFSPath(unsafe); got != "" { + t.Errorf("tforkCgroupFSPath(%q) = %q; want empty", unsafe, got) + } + } + const rel = "machine.slice/libpod-test" + if got, want := tforkCgroupFSPath(rel), "/sys/fs/cgroup/"+rel; got != want { + t.Fatalf("tforkCgroupFSPath(%q) = %q; want %q", rel, got, want) + } +} + +func TestTforkPIDRunning(t *testing.T) { + if err := tforkPIDRunning(os.Getpid()); err != nil { + t.Fatalf("current process should be running: %v", err) + } + if err := tforkPIDRunning(-1); err == nil { + t.Fatal("negative pid unexpectedly reported running") + } +} + +func TestTforkParseFileInjections(t *testing.T) { + parsed, err := tforkParseFileInjections([]string{ + "0:/tmp/source-0:/tmp/context.json", + "1:/tmp/source-1:/run/context.json", + }, 2) + if err != nil { + t.Fatal(err) + } + if got := parsed[1][0].destination; got != "/run/context.json" { + t.Fatalf("destination = %q", got) + } + for _, spec := range []string{ + "missing-fields", + "2:/tmp/source:/tmp/context", + "0:relative:/tmp/context", + "0:/tmp/source:relative", + } { + if _, err := tforkParseFileInjections([]string{spec}, 2); err == nil { + t.Fatalf("expected %q to fail", spec) + } + } +} + +func TestTforkParseSourceFileInjections(t *testing.T) { + parsed, err := tforkParseSourceFileInjections([]string{ + "/tmp/source.json:/tmp/gensee-run-context.json", + }) + if err != nil { + t.Fatal(err) + } + if len(parsed) != 1 || parsed[0].source != "/tmp/source.json" || parsed[0].destination != "/tmp/gensee-run-context.json" { + t.Fatalf("unexpected source file injections: %#v", parsed) + } + if _, err := tforkParseSourceFileInjections([]string{"relative:/tmp/context"}); err == nil { + t.Fatal("expected relative source path to fail") + } +} + +func TestTforkTransactionRollbackIsIdempotent(t *testing.T) { + temp := t.TempDir() + bundle := filepath.Join(temp, "graph", "tfork-bundles", "batch") + runtimeBatch := filepath.Join(temp, "run", "tfork", "batch") + for _, path := range []string{bundle, runtimeBatch} { + if err := os.MkdirAll(path, 0o700); err != nil { + t.Fatal(err) + } + } + + restoreCalls := 0 + txn := newTforkCloneTransaction(t.Context(), nil, nil, bundle, 1) + txn.setRuntimeBatchDir(runtimeBatch) + txn.setSourceRestore(func() error { + restoreCalls++ + return nil + }) + txn.rollback(errors.New("test")) + txn.rollback(errors.New("test again")) + + if restoreCalls != 1 { + t.Fatalf("source restore called %d times; want exactly once", restoreCalls) + } + for _, path := range []string{bundle, runtimeBatch} { + if _, err := os.Stat(path); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("rollback left %s: %v", path, err) + } + } +} diff --git a/ubuntu-img/Dockerfile b/ubuntu-img/Dockerfile index 235528ac9..782c9193c 100644 --- a/ubuntu-img/Dockerfile +++ b/ubuntu-img/Dockerfile @@ -119,6 +119,7 @@ RUN \ libreoffice-style-breeze \ libreoffice-writer \ thunderbird \ + tmux \ ubuntu-wallpapers \ ubuntu-wallpapers-jammy \ vlc && \ diff --git a/ubuntu-img/docker-compose.yml b/ubuntu-img/docker-compose.yml index cf5768548..3dc4c708a 100644 --- a/ubuntu-img/docker-compose.yml +++ b/ubuntu-img/docker-compose.yml @@ -1,6 +1,6 @@ services: webtop: - image: ${IMAGE:-ghcr.io/wuklab/webtop:ubuntu-kde} + image: ${IMAGE:-localhost/gensee-tclone-webtop:tmux} container_name: webtop security_opt: - seccomp=unconfined