diff --git a/README.md b/README.md index 1fa656e37..6a5001f8e 100644 --- a/README.md +++ b/README.md @@ -210,6 +210,8 @@ Please refer to the [DEVELOPMENT](docs/developer_guide.md) [An Example of Flash Checkpoint.](examples/pytorch/fcp_demo.py) +[MoE In-Place Restart Prototypes.](examples/in_place_restart/README.md) + [Train a PyTorch Model on Kubernetes.](docs/tutorial/torch_elasticjob_on_k8s.md) [Train a GPT Model on Kubernetes.](docs/tutorial/torch_nanogpt.md) diff --git a/examples/in_place_restart/Dockerfile b/examples/in_place_restart/Dockerfile new file mode 100644 index 000000000..147f57033 --- /dev/null +++ b/examples/in_place_restart/Dockerfile @@ -0,0 +1,51 @@ +# Copyright 2026 The DLRover Authors. All rights reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# NGC 26.09 contains PyTorch 2.14.0a0 and NCCL 2.31. That NCCL release is +# sufficient for the nccl2 reconfigure API, which requires NCCL 2.28 or newer. +# +# To test with NCCL 2.32 instead, rebuild with: +# --build-arg BASE_IMAGE=nvcr.io/nvidia/pytorch:26.10-py3 +# NGC 26.10 uses PyTorch 2.15.0a0, so rerun the recovery test matrix first. +ARG BASE_IMAGE=nvcr.io/nvidia/pytorch:26.09-py3 +FROM ${BASE_IMAGE} + +ENV DEBIAN_FRONTEND=noninteractive \ + PIP_NO_CACHE_DIR=1 \ + PIP_CONSTRAINT=/etc/pip/constraint.txt \ + PYTHONUNBUFFERED=1 \ + PYTHONPATH=/workspace/dlrover \ + TORCH_NCCL_ASYNC_ERROR_HANDLING=0 \ + TORCH_NCCL_BLOCKING_WAIT=0 + +WORKDIR /workspace/dlrover + +# Do not install examples/in_place_restart/requirements.txt here. The NGC +# image contains a purpose-built PyTorch build which must not be overwritten +# by a generic PyPI wheel. +COPY LICENSE README.md setup.py ./ +COPY dlrover ./dlrover +COPY examples/in_place_restart ./examples/in_place_restart + +RUN python -m pip install ".[k8s]" \ + && python -c "import torch, torch.distributed as dist; \ +assert (int(torch.__version__.split('.')[0]), \ +int(torch.__version__.split('.')[1])) >= (2, 14); \ +assert dist.is_backend_available('nccl2'); \ +assert hasattr(dist, '_reconfigure'); \ +assert hasattr(dist, '_get_reconfigure_handle'); \ +assert torch.cuda.nccl.version() >= (2, 28, 0); \ +print('PyTorch', torch.__version__); \ +print('NCCL', torch.cuda.nccl.version())" + +CMD ["python", "examples/in_place_restart/dp4/run_in_place_demo.py"] diff --git a/examples/in_place_restart/README.md b/examples/in_place_restart/README.md new file mode 100644 index 000000000..8fe17bec1 --- /dev/null +++ b/examples/in_place_restart/README.md @@ -0,0 +1,453 @@ +# MoE In-Place Restart Demos + +This example starts with a small replicated-expert MoE model trained with +DDP. It then demonstrates the first healthy-rank recovery prototype: + +- Four CUDA processes use a reconfigurable PyTorch `nccl2` process group. +- Rank 2 exits at a configured training step. +- The local supervisor leaves ranks 0, 1, and 3 running. +- Only rank 2 is started again. +- All four ranks reconfigure the same default process group. +- Every rank restores the same committed checkpoint. +- The DDP wrapper is rebuilt around the existing raw model and optimizer. +- Training resumes and reaches the configured final step. + +The experts are replicated on every rank. The only distributed communication +is DDP over the default process group. This deliberately avoids TP, PP, EP, +and expert all-to-all recovery in the first prototype. + +## Requirements + +- Linux with four GPUs for the DDP demo, eight GPUs on one node for the + single-node hybrid demo, or four GPUs on each of two nodes for the + multi-node hybrid demo +- PyTorch 2.14 or newer +- The experimental `nccl2` distributed backend +- NCCL 2.28 or newer; NCCL 2.32 is recommended + +PyTorch 2.14 marks `nccl2` and process-group reconfiguration as experimental. +Pin and test the exact PyTorch build used by the cluster. + +The local supervisor disables the legacy asynchronous error processes so that +`nccl2` can revoke the failed communicator and return control to the training +process. + +```bash +python -m pip install -r examples/in_place_restart/requirements.txt +``` + +## Build a Docker image + +The recommended base image is `nvcr.io/nvidia/pytorch:26.09-py3`. NVIDIA's +26.09 release contains PyTorch `2.14.0a0`, CUDA 13.4.1, and NCCL 2.31. NCCL +2.31 is sufficient for PyTorch's `nccl2` reconfigure implementation, which +requires NCCL 2.28 or newer. + +The host must have a driver compatible with CUDA 13.4 and the NVIDIA Container +Toolkit installed. If the NGC registry requires authentication, use an NGC API +key: + +```bash +docker login nvcr.io +# Username: $oauthtoken +# Password: +``` + +Build the image from the repository root: + +```bash +docker build \ + -f examples/in_place_restart/Dockerfile \ + -t dlrover-inplace-moe:26.09 \ + . +``` + +The build installs DLRover into the NGC PyTorch environment and verifies at +image-build time that: + +- PyTorch is the expected 2.14 build; +- the `nccl2` backend is available; +- `dist._reconfigure()` and reconfigure handles are exposed; +- the linked NCCL version is at least 2.28. + +The image does not run: + +```text +pip install -r examples/in_place_restart/requirements.txt +``` + +That command could replace NVIDIA's optimized PyTorch build with a generic +PyPI wheel. The NGC base image already contains the required framework. + +To require NCCL 2.32 explicitly, build against NGC 26.10: + +```bash +docker build \ + -f examples/in_place_restart/Dockerfile \ + --build-arg BASE_IMAGE=nvcr.io/nvidia/pytorch:26.10-py3 \ + -t dlrover-inplace-moe:nccl232 \ + . +``` + +NGC 26.10 contains NCCL 2.32.3 but PyTorch `2.15.0a0`, so rerun the complete +recovery test before treating that image as supported. + +Run the four-rank DDP demonstration with all four GPUs: + +```bash +mkdir -p /tmp/dlrover-moe-artifacts +docker run --rm \ + --gpus '"device=0,1,2,3"' \ + --shm-size=16g \ + -v /tmp/dlrover-moe-artifacts:/artifacts \ + dlrover-inplace-moe:26.09 \ + python examples/in_place_restart/dp4/run_in_place_demo.py \ + --work-dir /artifacts +``` + +Before running the recovery demo, execute the image preflight: + +```bash +docker run --rm \ + --gpus '"device=0,1,2,3"' \ + --shm-size=16g \ + dlrover-inplace-moe:26.09 \ + python examples/in_place_restart/verify_image.py +``` + +The preflight checks the PyTorch and NCCL versions, `nccl2` backend, +reconfigure APIs, visible GPUs, `/dev/shm`, DLRover installation, and an actual +four-rank initial reconfiguration followed by an all-reduce. + +Use `--ipc=host` instead of `--shm-size=16g` when host IPC is permitted and +the cluster policy expects that configuration. The example starts all four +workers inside this one container; it does not launch a separate container per +rank. + +For Kubernetes, run the local demonstration in one Pod that requests all four +GPUs: + +```yaml +resources: + limits: + nvidia.com/gpu: 4 +``` + +For DLRover jobs, override the container command with the appropriate +`dlrover-run` invocation. The image includes both the DLRover package and this +example directory. + +Use a digest rather than only the moving `26.09-py3` tag when promoting an +image to a shared cluster, for example: + +```text +nvcr.io/nvidia/pytorch@sha256: +``` + +## Four-GPU DDP demo + +From the DLRover repository root: + +```bash +python examples/in_place_restart/dp4/run_in_place_demo.py +``` + +The default sequence is: + +1. Train ranks 0-3. +2. Commit checkpoints at steps 0, 10, 20, and later checkpoints. +3. Exit rank 2 at step 23. +4. Start a replacement for rank 2 with the same logical rank. +5. Reconfigure the `nccl2` default group. +6. Restore the step-20 checkpoint on all ranks. +7. Continue and finish 40 steps. + +A temporary work directory is printed at the end. It contains: + +```text +checkpoints/ + checkpoint-00000020.pt + latest.json +recovery/ + command.json +rendezvous/ + store +status/ + rank_0.json + rank_1.json + rank_2.json + rank_3.json +``` + +The final status verifies that: + +- ranks 0, 1, and 3 retained their original PIDs; +- rank 2 has its replacement PID; +- all ranks completed at least one recovery; +- all ranks reached the final training step. + +The supervisor prints the process replacement and continuity explicitly: + +```text +[supervisor] initial processes: rank 0=pid 101, rank 1=pid 102, +rank 2=pid 103, rank 3=pid 104 +[supervisor] FAILED rank 2: old_pid=103, exit_code=17 +[supervisor] healthy processes still running: +rank 0=pid 101, rank 1=pid 102, rank 3=pid 104 +[supervisor] REPLACED rank 2: old_pid=103, new_pid=205 +[supervisor] RECOVERED rank 0: pid=101, epoch=1, step=20 +[supervisor] RECOVERED rank 1: pid=102, epoch=1, step=20 +[supervisor] RECOVERED rank 2: pid=205, epoch=1, step=20 +[supervisor] RECOVERED rank 3: pid=104, epoch=1, step=20 +``` + +The final summary marks ranks 0, 1, and 3 as `CONTINUED` and rank 2 as +`REPLACED`. + +Use a persistent work directory to inspect the artifacts: + +```bash +python examples/in_place_restart/dp4/run_in_place_demo.py \ + --work-dir /tmp/dlrover-moe-recovery +``` + +The work directory must be empty for a new demonstration. + +## Checkpoint-only baseline + +The training script can be started by `dlrover-run` without in-place +reconfiguration. This is useful for checking deterministic data generation +and checkpoint restore before testing process-group recovery. + +```bash +mkdir -p /tmp/dlrover-moe-baseline +dlrover-run --standalone --nproc_per_node=4 \ + examples/in_place_restart/dp4/train.py \ + --backend nccl \ + --work-dir /tmp/dlrover-moe-baseline \ + --steps 40 \ + --checkpoint-interval 10 +``` + +Do not use the local supervisor with `--backend nccl`. In-place recovery +requires the `nccl2` backend and `enable_reconfigure=True`. + +## How recovery works + +The worker performs these phases: + +```text +RUNNING + -> stop old-epoch training + -> detach the DDP wrapper while retaining raw model parameters + -> revoke in-flight nccl2 work + -> exchange opaque reconfigure handles + -> dist._reconfigure() on the same ProcessGroup object + -> restore checkpoint on the same raw model and optimizer + -> rebuild DDP around the same parameters + -> wait for every rank + -> RESUME +``` + +The ordered reconfigure handle list assigns ranks by position. The replacement +rank's handle is therefore placed at logical rank 2, preserving the original +rank mapping. + +Healthy ranks retain: + +- process PID and Python runtime; +- CUDA context; +- raw model parameter tensors; +- optimizer object; +- dataset and training configuration. + +The demonstration rebuilds the DDP wrapper after recovery. This clears any +partially completed DDP reducer state and is safer for the first prototype. +It does not rebuild the model or process. + +## Eight-GPU TP2 x PP2 x DP2 demo + +The [hybrid worker](tp2_pp2_dp2/hybrid_moe_train.py) adds a TP/PP/DP topology: + +```text +World size = 8 +TP = 2 +PP = 2 +DP = 2 +EP = 1 +``` + +The rank layout is: + +```text +rank = DP * 4 + PP_stage * 2 + TP_rank +``` + +The example creates and reconfigures: + +```text +TP groups: [0,1] [2,3] [4,5] [6,7] +PP groups: [0,2] [1,3] [4,6] [5,7] +DP groups: [0,4] [1,5] [2,6] [3,7] +``` + +Stage 0 owns a TP-sharded embedding. Stage 1 owns TP-sharded MoE experts and +a TP-sharded output head. `EP=1` means there is no additional expert partition; +each DP replica contains the full expert set, while each expert's weights are +split across the TP group. The gradient for the replicated router is reduced +across the TP group, while each stage synchronizes its local shard across its +DP group. + +Run the eight-GPU demo: + +```bash +ARTIFACT_DIR=$(mktemp -d /tmp/dlrover-hybrid.XXXXXX) + +python examples/in_place_restart/tp2_pp2_dp2/run_in_place_demo.py \ + --work-dir "$ARTIFACT_DIR" \ + --steps 40 \ + --checkpoint-interval 10 \ + --fail-rank 3 \ + --fail-at-step 23 +``` + +Rank 3 is `TP1-PP1-DP0`. Its TP, PP, DP, and default groups are all exercised +during recovery. The healthy ranks retain their PIDs, rank 3 receives a new +PID, all eight ranks restore their topology-specific checkpoint shards, and +training resumes from the previous committed checkpoint. + +The same supervisor accepts `--recovery-mode restart-all` for a full +eight-process restart baseline. It terminates all healthy processes, starts +eight fresh workers on a fresh `FileStore` generation, and resumes every rank +from the committed checkpoint. The comparison launcher runs both modes: + +```bash +COMPARE_DIR=$(mktemp -d /tmp/dlrover-hybrid-compare.XXXXXX) + +python examples/in_place_restart/tp2_pp2_dp2/run_recovery_comparison.py \ + --work-dir "$COMPARE_DIR" \ + --steps 40 \ + --checkpoint-interval 10 \ + --fail-rank 3 \ + --fail-at-step 23 +``` + +It reports loss differences after the checkpoint, failure-to-resume time, +process restart counts, and the time saved by in-place recovery. The +comparison artifacts are described in the +[eight-GPU README](tp2_pp2_dp2/README.md). + +The eight-rank image check is: + +```bash +docker run --rm \ + --gpus '"device=0,1,2,3,4,5,6,7"' \ + --shm-size=32g \ + dlrover-inplace-moe:26.09 \ + python examples/in_place_restart/verify_image.py --world-size 8 +``` + +Run the eight-rank recovery demo in the image: + +```bash +ARTIFACT_DIR=$(mktemp -d /tmp/dlrover-hybrid.XXXXXX) + +docker run --rm \ + --gpus '"device=0,1,2,3,4,5,6,7"' \ + --shm-size=32g \ + -v "$ARTIFACT_DIR:/artifacts" \ + dlrover-inplace-moe:26.09 \ + python examples/in_place_restart/tp2_pp2_dp2/run_in_place_demo.py \ + --work-dir /artifacts +``` + +## Two-node TP2 x PP2 x DP2 demo + +The [two-node hybrid worker](tp2_pp2_dp2_2nodes/run_node_agent.py) uses the +same model, topology, checkpoint format, and reconfiguration code as the +eight-GPU single-node demo. It changes only the process and control-plane +layout: + +```text +node 0: ranks 0,1,2,3 +node 1: ranks 4,5,6,7 +``` + +Both nodes launch four workers with `LOCAL_RANK=0..3`. The default failure is +rank 5 (`TP1-PP0-DP1`). Rank 5's TP and PP peers are on node 1, and its DP +peer rank 1 is on node 0. Only node 1 starts rank 5's replacement. All other +workers retain their original process and recover in place. + +The two agents coordinate through one shared work directory. That directory +must have the same absolute path on both nodes and must support +`torch.distributed.FileStore`. It also contains the checkpoint shards, +recovery command, and final status files. + +Start node 0 first: + +```bash +export RUN_DIR=/nas/shared/dlrover/two-node-run-001 +mkdir -p "$RUN_DIR" + +bash examples/in_place_restart/tp2_pp2_dp2_2nodes/run_node0.sh \ + --work-dir "$RUN_DIR" \ + --fail-rank 5 \ + --fail-at-step 23 +``` + +Start node 1 while node 0 is waiting: + +```bash +bash examples/in_place_restart/tp2_pp2_dp2_2nodes/run_node1.sh \ + --work-dir "$RUN_DIR" \ + --fail-rank 5 \ + --fail-at-step 23 +``` + +Pass `--nccl-socket-ifname` or `--nccl-ib-hca` when the cluster does not +auto-select the correct cross-node network. Set `NCCL_DEBUG=INFO` for the +first validation run. See the [two-node README](tp2_pp2_dp2_2nodes/README.md) +for the complete launch, container, topology, recovery, and verification +details. + +Run `verify_image.py --world-size 4` on each node before the two-node job. +That check validates the local four-GPU runtime; the job itself validates +cross-node NCCL and shared-directory access. + +## DLRover integration + +The local supervisor in each job directory models the control-plane work that +DLRover must perform: + +- detect the failed logical rank; +- preserve healthy worker processes; +- start only the failed rank's replacement process; +- select a globally committed checkpoint; +- allocate a monotonic recovery epoch and fresh 64-bit reconfigure UUID; +- publish a `RECOVER` command out of band; +- wait for all ranks to become ready before releasing training. + +The file-based command and status protocol keeps the example self-contained. +Production integration should replace it with a DLRover master-to-worker +control channel, such as a per-process Unix socket or an existing agent RPC. + +The standalone demo uses a `FileStore`. A production multi-node deployment +must use a control-plane-owned rendezvous that does not disappear when a +worker, including rank 0, fails. + +## Known limitations + +- `nccl2`, `_supports_reconfigure`, `_get_reconfigure_handle`, and + `_reconfigure` are experimental PyTorch APIs. +- The four-rank demo uses only the default group; the eight-rank demo uses + the default, TP, PP, and DP groups. +- The two-node demo uses a shared `FileStore` for control and rendezvous. A + production deployment should replace it with a DLRover-owned rendezvous + that survives control-plane or worker failures. +- The TP2 x PP2 x DP2 demo uses a toy pipeline schedule with one microbatch. +- The local supervisor injects one failure and does not handle a second rank + failure during recovery. +- Experts are replicated; this is not expert-parallel training. +- CUDA graphs and third-party NCCL communicators are not covered. +- The local supervisor and file-based protocol are for the example only. +- Heartbeat detection lives in DLRover and is outside this demo's scope. diff --git a/examples/in_place_restart/dp4/README.md b/examples/in_place_restart/dp4/README.md new file mode 100644 index 000000000..93e476d49 --- /dev/null +++ b/examples/in_place_restart/dp4/README.md @@ -0,0 +1,31 @@ +# Four-GPU DDP MoE Recovery + +This is the original replicated-expert MoE demo: + +```text +World size = 4 +DP = 4 +TP = 1 +PP = 1 +EP = 1 +``` + +It uses one reconfigurable default `nccl2` process group. The default demo +fails rank 2 at step 23, restarts only rank 2, restores checkpoint step 20, +and continues to step 40. + +From the repository root: + +```bash +ARTIFACT_DIR=$(mktemp -d /tmp/dlrover-dp4.XXXXXX) + +python examples/in_place_restart/dp4/run_in_place_demo.py \ + --work-dir "$ARTIFACT_DIR" +``` + +Files: + +- `train.py`: checkpoint-only DLRover-compatible entry point +- `moe_train.py`: training and recovery implementation +- `run_in_place_demo.py`: four-rank local supervisor +- `run.sh`: convenience launcher diff --git a/examples/in_place_restart/dp4/moe_train.py b/examples/in_place_restart/dp4/moe_train.py new file mode 100755 index 000000000..1483be995 --- /dev/null +++ b/examples/in_place_restart/dp4/moe_train.py @@ -0,0 +1,1049 @@ +#!/usr/bin/env python3 +# Copyright 2026 The DLRover Authors. All rights reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Four-rank MoE DDP training with optional in-place rank recovery. + +The MoE experts in this example are replicated on every rank. The only +distributed communication is DDP over the default process group. This keeps +the example focused on recovering that group without rebuilding the model or +optimizer on healthy ranks. +""" + +import argparse +import gc +import os +import sys +import time +from dataclasses import dataclass +from datetime import timedelta +from pathlib import Path +from typing import Any, Dict, Optional, Tuple + +import torch +import torch.distributed as dist +import torch.nn as nn +import torch.nn.functional as F +import torch.optim as optim +from torch.nn.parallel import DistributedDataParallel as DDP +from torch.utils.data import DataLoader, Dataset +from torch.utils.data.distributed import DistributedSampler + +EXAMPLE_ROOT = Path(__file__).resolve().parent.parent +if str(EXAMPLE_ROOT) not in sys.path: + sys.path.insert(0, str(EXAMPLE_ROOT)) + +from recovery_protocol import ( + RecoveryCommand, + atomic_write_json, + fresh_reconfigure_uuid, + load_recovery_command, + wait_for_recovery_command, +) + + +BATCH_SIZE = 32 +SEQ_LEN = 64 +VOCAB_SIZE = 1_000 +EMBED_DIM = 128 +NUM_EXPERTS = 4 +TOP_K = 1 +MODEL_SEED = 1234 +DATA_SEED = 5678 +CHECKPOINT_FORMAT_VERSION = 1 +TOPOLOGY = "dp4-experts-replicated" + + +@dataclass(frozen=True) +class SlowPlan: + rank: int + mode: str + delay_ms: float + extra_cycles: int + start_step: int + end_step: int + + def active(self, rank: int, step: int) -> bool: + return ( + self.rank >= 0 + and rank == self.rank + and self.start_step <= step < self.end_step + ) + + +class RecoveryRequested(Exception): + def __init__(self, command: RecoveryCommand) -> None: + super().__init__( + f"recovery epoch {command.recovery_epoch} requested" + ) + self.command = command + + +@dataclass +class TrainingRuntime: + rank: int + local_rank: int + world_size: int + device: torch.device + work_dir: Path + store: Any + process_group: Any + dataset: "LazyRandomDataset" + raw_model: nn.Module + optimizer: optim.Optimizer + model: Optional[DDP] = None + global_step: int = 0 + epoch: int = 0 + batch_index: int = 0 + recovery_epoch: int = 0 + recovery_count: int = 0 + + +class FeedForwardExpert(nn.Module): + def __init__(self, input_dim: int, hidden_dim: int) -> None: + super().__init__() + self.net = nn.Sequential( + nn.Linear(input_dim, hidden_dim), + nn.ReLU(), + nn.Linear(hidden_dim, input_dim), + ) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + return self.net(x) + + +class MoELayer(nn.Module): + def __init__(self, input_dim: int, num_experts: int, top_k: int) -> None: + super().__init__() + self.num_experts = num_experts + self.top_k = top_k + self.gate = nn.Linear(input_dim, num_experts) + self.experts = nn.ModuleList( + [ + FeedForwardExpert(input_dim, input_dim * 4) + for _ in range(num_experts) + ] + ) + + def forward(self, x: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: + batch_size, seq_len, embed_dim = x.shape + flat_x = x.reshape(-1, embed_dim) + probs = F.softmax(self.gate(flat_x), dim=-1) + weights, selected = torch.topk(probs, self.top_k, dim=-1) + if self.top_k > 1: + weights = weights / weights.sum(dim=-1, keepdim=True) + + output = torch.zeros_like(flat_x) + for expert_id, expert in enumerate(self.experts): + selected_for_expert = selected == expert_id + expert_weight = (weights * selected_for_expert).sum( + dim=-1, keepdim=True + ) + mask = expert_weight.squeeze(-1) > 0 + if mask.any(): + output[mask] += ( + expert(flat_x[mask]) * expert_weight[mask] + ) + + top1 = torch.argmax(probs, dim=-1) + density = ( + torch.bincount(top1, minlength=self.num_experts).float() + / top1.numel() + ) + aux_loss = ( + (self.num_experts * density) + * (self.num_experts * probs.mean(dim=0)) + ).mean() + return output.reshape(batch_size, seq_len, embed_dim), aux_loss + + +class ToyMoETransformer(nn.Module): + def __init__(self) -> None: + super().__init__() + self.embed = nn.Embedding(VOCAB_SIZE, EMBED_DIM) + self.moe = MoELayer(EMBED_DIM, NUM_EXPERTS, TOP_K) + self.norm = nn.LayerNorm(EMBED_DIM) + self.head = nn.Linear(EMBED_DIM, VOCAB_SIZE) + + def forward( + self, tokens: torch.Tensor + ) -> Tuple[torch.Tensor, torch.Tensor]: + hidden = self.embed(tokens) + moe_output, aux_loss = self.moe(hidden) + hidden = hidden + moe_output + return self.head(self.norm(hidden)), aux_loss + + +class LazyRandomDataset(Dataset): + """Deterministic synthetic data so recovery can reproduce a data cursor.""" + + def __init__( + self, + sequence_length: int, + length: int = 10_000, + seed: int = DATA_SEED, + ) -> None: + self.sequence_length = sequence_length + self.length = length + self.seed = seed + + def __getitem__(self, index: int) -> Tuple[torch.Tensor, torch.Tensor]: + generator = torch.Generator() + generator.manual_seed(self.seed + int(index)) + tokens = torch.randint( + 0, + VOCAB_SIZE, + (self.sequence_length,), + generator=generator, + ) + return tokens, tokens + + def __len__(self) -> int: + return self.length + + +def _log(rank: int, message: str) -> None: + print(f"[rank {rank} pid {os.getpid()}] {message}", flush=True) + + +def _write_status( + runtime: TrainingRuntime, + phase: str, + **extra: Any, +) -> None: + status = { + "rank": runtime.rank, + "pid": os.getpid(), + "phase": phase, + "global_step": runtime.global_step, + "epoch": runtime.epoch, + "batch_index": runtime.batch_index, + "recovery_epoch": runtime.recovery_epoch, + "recovery_count": runtime.recovery_count, + "timestamp_unix_ms": int(time.time() * 1000), + } + status.update(extra) + atomic_write_json( + runtime.work_dir / "status" / f"rank_{runtime.rank}.json", + status, + ) + + +def inject_slowdown( + plan: SlowPlan, + rank: int, + step: int, + device: int, +) -> None: + if plan.rank < 0: + return + + is_slow_rank = plan.active(rank, step) + if plan.mode == "cpu-sleep": + if is_slow_rank and plan.delay_ms: + torch.cuda.synchronize(device) + time.sleep(plan.delay_ms / 1_000.0) + return + + if plan.mode != "cuda-sleep": + return + if not hasattr(torch.cuda, "_sleep"): + raise RuntimeError( + "this PyTorch build does not expose torch.cuda._sleep" + ) + + cycles = max(1, plan.extra_cycles if is_slow_rank else 1) + torch.cuda._sleep(cycles) + + +def estimated_training_flops_per_token() -> int: + gate_flops = 2 * EMBED_DIM * NUM_EXPERTS + expert_hidden_dim = EMBED_DIM * 4 + expert_flops = 4 * EMBED_DIM * expert_hidden_dim + head_flops = 2 * EMBED_DIM * VOCAB_SIZE + forward_flops = gate_flops + expert_flops + head_flops + return 3 * forward_flops + + +def print_performance_metrics( + global_step: int, + total_steps: int, + loss_value: float, + interval_steps: int, + interval_seconds: float, +) -> None: + interval_ms = interval_seconds * 1_000.0 + elapsed_ms_per_iteration = interval_ms / interval_steps + tokens_per_gpu = interval_steps * BATCH_SIZE * SEQ_LEN + tokens_per_second_per_gpu = tokens_per_gpu / max( + 1e-9, interval_seconds + ) + tflops_per_gpu = ( + tokens_per_second_per_gpu + * estimated_training_flops_per_token() + / 1e12 + ) + timestamp = time.strftime("%Y-%m-%d %H:%M:%S") + print( + f"[{timestamp}] " + f"iteration {global_step}/{total_steps} " + f"elapsed time per iteration (ms): " + f"{elapsed_ms_per_iteration:.3f} | " + f"throughput per GPU (TFLOP/s/GPU): {tflops_per_gpu:.6f} | " + f"tokens/s/GPU: {tokens_per_second_per_gpu:.3f} | " + f"loss: {loss_value:.6f}", + flush=True, + ) + + +def _require_reconfigure_api() -> None: + required = ( + "_get_reconfigure_handle", + "_reconfigure", + "_supports_reconfigure", + "get_backend_impl", + ) + missing = [name for name in required if not hasattr(dist, name)] + if missing: + raise RuntimeError( + "in-place recovery requires PyTorch 2.14 or newer; " + f"missing torch.distributed APIs: {missing}" + ) + if not dist.is_backend_available("nccl2"): + raise RuntimeError( + "in-place recovery requires the experimental nccl2 backend" + ) + + +def _get_default_process_group() -> Any: + getter = getattr(dist, "_get_default_group", None) + if getter is None: + getter = dist.distributed_c10d._get_default_group + return getter() + + +def _setup_distributed( + args: argparse.Namespace, +) -> Tuple[int, int, int, torch.device, Any, Any]: + rank = int(os.environ["RANK"]) + local_rank = int(os.environ["LOCAL_RANK"]) + world_size = int(os.environ["WORLD_SIZE"]) + if world_size != 4: + raise ValueError( + "this example requires exactly four ranks; " + f"torchrun reported WORLD_SIZE={world_size}" + ) + if not torch.cuda.is_available(): + raise RuntimeError("this example requires four CUDA devices") + if torch.cuda.device_count() < world_size: + raise RuntimeError( + f"requires {world_size} CUDA devices, found " + f"{torch.cuda.device_count()}" + ) + + torch.cuda.set_device(local_rank) + device = torch.device("cuda", local_rank) + + work_dir = Path(args.work_dir) + rendezvous_dir = work_dir / "rendezvous" + rendezvous_dir.mkdir(parents=True, exist_ok=True) + store = dist.FileStore(str(rendezvous_dir / "store"), world_size) + + timeout = timedelta(seconds=args.dist_timeout_seconds) + if args.backend == "nccl2": + _require_reconfigure_api() + dist.init_process_group( + backend=args.backend, + store=store, + rank=rank, + world_size=world_size, + timeout=timeout, + enable_reconfigure=True, + ) + else: + dist.init_process_group( + backend=args.backend, + store=store, + rank=rank, + world_size=world_size, + timeout=timeout, + ) + + return ( + rank, + local_rank, + world_size, + device, + store, + _get_default_process_group(), + ) + + +def _reconfigure_group( + store: Any, + process_group: Any, + rank: int, + world_size: int, + attempt_id: str, + reconfigure_uuid: int, + timeout_seconds: float, +) -> None: + if not dist._supports_reconfigure(process_group): + raise RuntimeError( + "the default process group does not support reconfigure" + ) + + attempt_store = dist.PrefixStore( + f"atorch-in-place/{attempt_id}", store + ) + handle = dist._get_reconfigure_handle(process_group) + attempt_store.set(f"handle/{rank}", handle) + handle_keys = [ + f"handle/{handle_rank}" for handle_rank in range(world_size) + ] + attempt_store.wait( + handle_keys, + timedelta(seconds=timeout_seconds), + ) + handles = [ + attempt_store.get(key).decode("utf-8") for key in handle_keys + ] + + _log( + rank, + f"reconfiguring process group for {attempt_id}; " + f"rank ordering={list(range(world_size))}", + ) + work = dist._reconfigure( + uuid=reconfigure_uuid, + handles=handles, + group=process_group, + timeout=timedelta(seconds=timeout_seconds), + ) + work.wait() + + +def _bootstrap_reconfigurable_group( + args: argparse.Namespace, + store: Any, + process_group: Any, + rank: int, + world_size: int, +) -> None: + uuid_key = "atorch-in-place/bootstrap-uuid" + if rank == 0: + store.set(uuid_key, str(fresh_reconfigure_uuid())) + store.wait( + [uuid_key], + timedelta(seconds=args.recovery_timeout_seconds), + ) + reconfigure_uuid = int(store.get(uuid_key).decode("utf-8")) + + _reconfigure_group( + store=store, + process_group=process_group, + rank=rank, + world_size=world_size, + attempt_id="bootstrap", + reconfigure_uuid=reconfigure_uuid, + timeout_seconds=args.recovery_timeout_seconds, + ) + tensor = torch.ones(1, device=torch.cuda.current_device()) + dist.all_reduce(tensor) + if tensor.item() != world_size: + raise RuntimeError("initial process-group validation failed") + + +def _build_model( + device: torch.device, +) -> Tuple[nn.Module, optim.Optimizer]: + torch.manual_seed(MODEL_SEED) + model = ToyMoETransformer().to(device) + optimizer = optim.Adam(model.parameters(), lr=1e-3) + return model, optimizer + + +def _wrap_ddp( + raw_model: nn.Module, + local_rank: int, +) -> DDP: + return DDP( + raw_model, + device_ids=[local_rank], + output_device=local_rank, + find_unused_parameters=True, + ) + + +def _checkpoint_payload( + runtime: TrainingRuntime, + args: argparse.Namespace, +) -> Dict[str, Any]: + return { + "format_version": CHECKPOINT_FORMAT_VERSION, + "checkpoint_step": runtime.global_step, + "model": runtime.raw_model.state_dict(), + "optimizer": runtime.optimizer.state_dict(), + "cursor": { + "global_step": runtime.global_step, + "epoch": runtime.epoch, + "batch_index": runtime.batch_index, + }, + "topology": TOPOLOGY, + "world_size": runtime.world_size, + "training_config": { + "batch_size": BATCH_SIZE, + "sequence_length": SEQ_LEN, + "vocab_size": VOCAB_SIZE, + "embed_dim": EMBED_DIM, + "num_experts": NUM_EXPERTS, + "top_k": TOP_K, + "steps": args.steps, + }, + } + + +def _save_checkpoint( + runtime: TrainingRuntime, + args: argparse.Namespace, +) -> Optional[Path]: + if runtime.rank != 0: + return None + + checkpoint_dir = runtime.work_dir / "checkpoints" + checkpoint_dir.mkdir(parents=True, exist_ok=True) + checkpoint_path = ( + checkpoint_dir / f"checkpoint-{runtime.global_step:08d}.pt" + ) + temporary_path = checkpoint_path.with_suffix(".pt.tmp") + + torch.cuda.synchronize(runtime.local_rank) + payload = _checkpoint_payload(runtime, args) + with temporary_path.open("wb") as stream: + torch.save(payload, stream) + stream.flush() + os.fsync(stream.fileno()) + os.replace(str(temporary_path), str(checkpoint_path)) + + manifest = { + "checkpoint_id": f"moe-step-{runtime.global_step}", + "checkpoint_step": runtime.global_step, + "checkpoint_path": str(checkpoint_path.resolve()), + "committed_unix_ms": int(time.time() * 1000), + "topology": TOPOLOGY, + "world_size": runtime.world_size, + } + atomic_write_json(checkpoint_dir / "latest.json", manifest) + _log(runtime.rank, f"committed checkpoint at step {runtime.global_step}") + return checkpoint_path + + +def _load_checkpoint( + runtime: TrainingRuntime, + command: RecoveryCommand, +) -> None: + checkpoint_path = Path(command.checkpoint_path) + checkpoint = torch.load( + checkpoint_path, + map_location=runtime.device, + weights_only=True, + ) + + if checkpoint.get("format_version") != CHECKPOINT_FORMAT_VERSION: + raise ValueError("unsupported checkpoint format") + if checkpoint.get("topology") != TOPOLOGY: + raise ValueError("checkpoint topology does not match this job") + if checkpoint.get("world_size") != runtime.world_size: + raise ValueError("checkpoint world size does not match this job") + + cursor = checkpoint["cursor"] + if cursor["global_step"] != command.checkpoint_step: + raise ValueError( + "recovery command checkpoint step does not match checkpoint" + ) + + runtime.raw_model.load_state_dict(checkpoint["model"], strict=True) + runtime.optimizer.load_state_dict(checkpoint["optimizer"]) + runtime.global_step = cursor["global_step"] + runtime.epoch = cursor["epoch"] + runtime.batch_index = cursor["batch_index"] + _log( + runtime.rank, + f"restored checkpoint step {runtime.global_step}, " + f"epoch {runtime.epoch}, batch {runtime.batch_index}", + ) + + +def _make_loader( + runtime: TrainingRuntime, +) -> Tuple[DataLoader, DistributedSampler]: + sampler = DistributedSampler( + runtime.dataset, + num_replicas=runtime.world_size, + rank=runtime.rank, + shuffle=True, + seed=DATA_SEED, + ) + sampler.set_epoch(runtime.epoch) + loader = DataLoader( + runtime.dataset, + batch_size=BATCH_SIZE, + sampler=sampler, + num_workers=0, + pin_memory=True, + ) + return loader, sampler + + +def _abort_reconfigurable_group(runtime: TrainingRuntime) -> None: + backend = dist.get_backend_impl( + group=runtime.process_group, + device=runtime.device, + ) + # In nccl2 reconfigure mode, abort revokes in-flight work and keeps the + # process-group shell available for reconfigure. + backend.abort() + + +def _detach_ddp(runtime: TrainingRuntime) -> None: + if runtime.model is None: + return + runtime.raw_model = runtime.model.module + runtime.model = None + gc.collect() + + +def _wait_for_all_ranks( + runtime: TrainingRuntime, + attempt_id: str, + timeout_seconds: float, +) -> None: + ready_store = dist.PrefixStore( + f"atorch-in-place/{attempt_id}", runtime.store + ) + ready_store.set(f"ready/{runtime.rank}", "1") + ready_keys = [ + f"ready/{ready_rank}" for ready_rank in range(runtime.world_size) + ] + ready_store.wait( + ready_keys, + timedelta(seconds=timeout_seconds), + ) + + +def _recover( + runtime: TrainingRuntime, + args: argparse.Namespace, + command: RecoveryCommand, + replacement: bool, +) -> None: + if args.backend != "nccl2": + raise RuntimeError( + "in-place recovery requires --backend nccl2" + ) + if command.world_size != runtime.world_size: + raise ValueError("recovery command world size does not match") + if replacement and command.failed_rank != runtime.rank: + raise ValueError( + "replacement process does not match the failed logical rank" + ) + remaining_seconds = ( + command.deadline_unix_ms / 1000.0 - time.time() + ) + if remaining_seconds <= 0: + raise TimeoutError( + f"recovery epoch {command.recovery_epoch} expired" + ) + recovery_timeout = min( + args.recovery_timeout_seconds, + remaining_seconds, + ) + + _write_status( + runtime, + "quiescing", + recovery_epoch=command.recovery_epoch, + ) + if not replacement: + _abort_reconfigurable_group(runtime) + _detach_ddp(runtime) + runtime.optimizer.zero_grad(set_to_none=True) + _write_status( + runtime, + "quiesced", + recovery_epoch=command.recovery_epoch, + ) + + _reconfigure_group( + store=runtime.store, + process_group=runtime.process_group, + rank=runtime.rank, + world_size=runtime.world_size, + attempt_id=command.attempt_id, + reconfigure_uuid=command.reconfigure_uuid, + timeout_seconds=recovery_timeout, + ) + validation = torch.ones(1, device=runtime.device) + dist.all_reduce(validation) + if validation.item() != runtime.world_size: + raise RuntimeError("reconfigured process-group validation failed") + _write_status( + runtime, + "comm_ready", + recovery_epoch=command.recovery_epoch, + ) + + _load_checkpoint(runtime, command) + runtime.model = _wrap_ddp(runtime.raw_model, runtime.local_rank) + runtime.model.train() + runtime.optimizer.zero_grad(set_to_none=True) + runtime.recovery_epoch = command.recovery_epoch + runtime.recovery_count += 1 + + _wait_for_all_ranks( + runtime, + command.attempt_id, + recovery_timeout, + ) + dist.barrier() + _write_status(runtime, "running") + _log( + runtime.rank, + f"in-place recovery complete at step {runtime.global_step}; " + f"recovery epoch={runtime.recovery_epoch}", + ) + + +def _is_comm_error(error: Exception) -> bool: + name = type(error).__name__.lower() + message = str(error).lower() + return ( + "backend" in name + or "nccl" in message + or "collective" in message + or "timed out" in message + ) + + +def _run_training_segment( + runtime: TrainingRuntime, + args: argparse.Namespace, +) -> None: + assert runtime.model is not None + criterion = nn.CrossEntropyLoss() + plan = SlowPlan( + rank=args.slow_rank, + mode=args.slow_mode, + delay_ms=args.slow_delay_ms, + extra_cycles=args.slow_extra_cycles, + start_step=args.slow_start_step, + end_step=args.slow_end_step, + ) + recovery_command_path = Path(args.recovery_command) + + interval_steps = 0 + interval_start = time.perf_counter() + + while runtime.global_step < args.steps: + loader, _ = _make_loader(runtime) + batches_processed = 0 + + for batch_index, (inputs, targets) in enumerate(loader): + if runtime.global_step >= args.steps: + break + if batch_index < runtime.batch_index: + continue + + command = load_recovery_command(recovery_command_path) + if ( + command is not None + and command.recovery_epoch > runtime.recovery_epoch + ): + raise RecoveryRequested(command) + + if ( + runtime.rank == args.fail_rank + and runtime.global_step == args.fail_at_step + ): + _write_status( + runtime, + "injected_failure", + injected_failure=True, + ) + _log( + runtime.rank, + f"injecting process failure at step " + f"{runtime.global_step}", + ) + os._exit(17) + + inputs = inputs.to(runtime.device, non_blocking=True) + targets = targets.to(runtime.device, non_blocking=True) + inject_slowdown( + plan, + runtime.rank, + runtime.global_step, + runtime.local_rank, + ) + + runtime.optimizer.zero_grad(set_to_none=True) + logits, aux_loss = runtime.model(inputs) + loss = criterion( + logits.reshape(-1, VOCAB_SIZE), + targets.reshape(-1), + ) + (loss + 0.01 * aux_loss).backward() + runtime.optimizer.step() + + runtime.global_step += 1 + runtime.batch_index = batch_index + 1 + if runtime.batch_index >= len(loader): + runtime.epoch += 1 + runtime.batch_index = 0 + batches_processed += 1 + interval_steps += 1 + + if ( + args.checkpoint_interval > 0 + and runtime.global_step % args.checkpoint_interval == 0 + ): + _save_checkpoint(runtime, args) + + should_log = ( + interval_steps >= args.log_every + or runtime.global_step >= args.steps + ) + if runtime.rank == 0 and should_log: + torch.cuda.synchronize(runtime.local_rank) + interval_seconds = time.perf_counter() - interval_start + print_performance_metrics( + global_step=runtime.global_step, + total_steps=args.steps, + loss_value=loss.item(), + interval_steps=interval_steps, + interval_seconds=interval_seconds, + ) + interval_start = time.perf_counter() + interval_steps = 0 + + _write_status(runtime, "running") + + if runtime.global_step >= args.steps: + break + + if runtime.global_step >= args.steps: + break + if batches_processed == 0: + raise RuntimeError( + "dataloader made no progress while resuming" + ) + + +def train(args: argparse.Namespace) -> None: + ( + rank, + local_rank, + world_size, + device, + store, + process_group, + ) = _setup_distributed(args) + + if args.backend == "nccl2" and not args.replacement: + _bootstrap_reconfigurable_group( + args, + store, + process_group, + rank, + world_size, + ) + + dataset = LazyRandomDataset(SEQ_LEN) + raw_model, optimizer = _build_model(device) + runtime = TrainingRuntime( + rank=rank, + local_rank=local_rank, + world_size=world_size, + device=device, + work_dir=Path(args.work_dir), + store=store, + process_group=process_group, + dataset=dataset, + raw_model=raw_model, + optimizer=optimizer, + ) + _write_status(runtime, "initializing") + + try: + if args.replacement: + command = wait_for_recovery_command( + Path(args.recovery_command), + timeout_seconds=args.recovery_timeout_seconds, + ) + _recover(runtime, args, command, replacement=True) + else: + runtime.model = _wrap_ddp(runtime.raw_model, runtime.local_rank) + runtime.model.train() + if args.checkpoint_interval > 0: + _save_checkpoint(runtime, args) + _write_status(runtime, "running") + + while runtime.global_step < args.steps: + try: + _run_training_segment(runtime, args) + except RecoveryRequested as error: + _recover( + runtime, + args, + error.command, + replacement=False, + ) + except Exception as error: + if not ( + args.backend == "nccl2" + and _is_comm_error(error) + ): + raise + _log( + runtime.rank, + f"recoverable communication failure: {error}", + ) + command = wait_for_recovery_command( + Path(args.recovery_command), + timeout_seconds=args.recovery_timeout_seconds, + minimum_epoch=runtime.recovery_epoch, + ) + _recover( + runtime, + args, + command, + replacement=False, + ) + + dist.barrier() + if ( + args.checkpoint_interval > 0 + and runtime.global_step % args.checkpoint_interval != 0 + ): + _save_checkpoint(runtime, args) + _write_status(runtime, "finished") + _log( + runtime.rank, + f"finished {runtime.global_step} steps; " + f"recoveries={runtime.recovery_count}", + ) + except Exception as error: + _write_status( + runtime, + "failed", + error=repr(error), + ) + raise + finally: + if dist.is_initialized(): + dist.destroy_process_group() + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--work-dir", type=str, required=True) + parser.add_argument("--steps", type=int, default=40) + parser.add_argument("--log-every", type=int, default=5) + parser.add_argument("--checkpoint-interval", type=int, default=10) + parser.add_argument( + "--backend", + choices=("nccl2", "nccl"), + default="nccl2", + help="nccl2 enables in-place process-group reconfigure", + ) + parser.add_argument( + "--recovery-command", + type=str, + default="", + help="JSON command published by DLRover or the local demo supervisor", + ) + parser.add_argument("--replacement", action="store_true") + parser.add_argument("--dist-timeout-seconds", type=int, default=20) + parser.add_argument( + "--recovery-timeout-seconds", type=float, default=180.0 + ) + parser.add_argument("--fail-rank", type=int, default=-1) + parser.add_argument("--fail-at-step", type=int, default=-1) + parser.add_argument( + "--slow-rank", + type=int, + default=-1, + help="global rank to slow; -1 disables injection", + ) + parser.add_argument( + "--slow-mode", + choices=("cuda-sleep", "cpu-sleep"), + default="cuda-sleep", + ) + parser.add_argument("--slow-delay-ms", type=float, default=25.0) + parser.add_argument("--slow-extra-cycles", type=int, default=10_000_000) + parser.add_argument("--slow-start-step", type=int, default=60) + parser.add_argument("--slow-end-step", type=int, default=900) + return parser + + +def _validate_parser(args: argparse.Namespace) -> None: + if args.steps <= 0: + raise SystemExit("--steps must be positive") + if args.log_every <= 0: + raise SystemExit("--log-every must be positive") + if args.checkpoint_interval < 0: + raise SystemExit("--checkpoint-interval must be nonnegative") + if args.fail_rank < -1 or args.fail_rank >= 4: + raise SystemExit("--fail-rank must be -1, 0, 1, 2, or 3") + if (args.fail_rank == -1) != (args.fail_at_step == -1): + raise SystemExit( + "--fail-rank and --fail-at-step must be set together" + ) + if args.fail_at_step < -1: + raise SystemExit("--fail-at-step must be -1 or nonnegative") + if args.fail_rank >= 0 and args.fail_at_step >= args.steps: + raise SystemExit("--fail-at-step must be less than --steps") + if args.fail_rank >= 0 and args.checkpoint_interval <= 0: + raise SystemExit( + "failure injection requires a positive checkpoint interval" + ) + if args.slow_rank < -1 or args.slow_rank >= 4: + raise SystemExit("--slow-rank must be -1, 0, 1, 2, or 3") + if args.slow_delay_ms < 0 or args.slow_extra_cycles < 1: + raise SystemExit("slowdown values must be nonnegative/positive") + if args.slow_start_step < 0 or args.slow_end_step <= args.slow_start_step: + raise SystemExit( + "slow step range must satisfy 0 <= start < end" + ) + if args.replacement and not args.recovery_command: + raise SystemExit( + "--replacement requires --recovery-command" + ) + if args.replacement and args.backend != "nccl2": + raise SystemExit("--replacement requires --backend nccl2") + + +def main() -> None: + args = build_parser().parse_args() + _validate_parser(args) + work_dir = Path(args.work_dir) + work_dir.mkdir(parents=True, exist_ok=True) + if not args.recovery_command: + args.recovery_command = str( + work_dir / "recovery" / "command.json" + ) + train(args) + + +if __name__ == "__main__": + main() diff --git a/examples/in_place_restart/dp4/run.sh b/examples/in_place_restart/dp4/run.sh new file mode 100755 index 000000000..809755238 --- /dev/null +++ b/examples/in_place_restart/dp4/run.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +# Copyright 2026 The DLRover Authors. All rights reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +python "${SCRIPT_DIR}/run_in_place_demo.py" "$@" diff --git a/examples/in_place_restart/dp4/run_in_place_demo.py b/examples/in_place_restart/dp4/run_in_place_demo.py new file mode 100755 index 000000000..b58e64d78 --- /dev/null +++ b/examples/in_place_restart/dp4/run_in_place_demo.py @@ -0,0 +1,424 @@ +#!/usr/bin/env python3 +# Copyright 2026 The DLRover Authors. All rights reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Launch a local four-rank in-place recovery demonstration. + +The supervisor models the DLRover control-plane responsibilities needed by +this example: it preserves healthy workers, restarts only the failed logical +rank, publishes a committed checkpoint, and supplies a fresh recovery epoch +and reconfigure UUID. +""" + +import argparse +import os +import socket +import subprocess +import sys +import tempfile +import time +from pathlib import Path +from typing import Any, Dict + +EXAMPLE_ROOT = Path(__file__).resolve().parent.parent +if str(EXAMPLE_ROOT) not in sys.path: + sys.path.insert(0, str(EXAMPLE_ROOT)) + +from recovery_protocol import ( + RecoveryCommand, + atomic_write_json, + fresh_reconfigure_uuid, + read_json, +) + + +WORLD_SIZE = 4 + + +def _wait_for_file(path: Path, timeout_seconds: float) -> Dict[str, Any]: + deadline = time.monotonic() + timeout_seconds + while time.monotonic() < deadline: + value = read_json(path) + if value is not None: + return value + time.sleep(0.2) + raise TimeoutError(f"timed out waiting for {path}") + + +def _find_free_port() -> int: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: + sock.bind(("127.0.0.1", 0)) + return int(sock.getsockname()[1]) + + +def _launch_worker( + rank: int, + args: argparse.Namespace, + work_dir: Path, + recovery_command_path: Path, + replacement: bool, +) -> subprocess.Popen: + env = os.environ.copy() + env.update( + { + "RANK": str(rank), + "LOCAL_RANK": str(rank), + "WORLD_SIZE": str(WORLD_SIZE), + "MASTER_ADDR": "127.0.0.1", + "MASTER_PORT": str(_find_free_port()), + "PYTHONUNBUFFERED": "1", + "OMP_NUM_THREADS": env.get("OMP_NUM_THREADS", "1"), + # nccl2 owns failure handling in reconfigure mode. Leaving legacy + # process-abort handling enabled would obscure the demonstration. + "TORCH_NCCL_ASYNC_ERROR_HANDLING": "0", + "TORCH_NCCL_BLOCKING_WAIT": "0", + } + ) + script_dir = Path(__file__).resolve().parent + worker_script = script_dir / "moe_train.py" + command = [ + sys.executable, + str(worker_script), + "--work-dir", + str(work_dir), + "--steps", + str(args.steps), + "--log-every", + str(args.log_every), + "--checkpoint-interval", + str(args.checkpoint_interval), + "--backend", + "nccl2", + "--recovery-command", + str(recovery_command_path), + "--dist-timeout-seconds", + str(args.dist_timeout_seconds), + "--recovery-timeout-seconds", + str(args.recovery_timeout_seconds), + ] + if replacement: + command.append("--replacement") + else: + command.extend( + [ + "--fail-rank", + str(args.fail_rank), + "--fail-at-step", + str(args.fail_at_step), + ] + ) + return subprocess.Popen( + command, + cwd=str(script_dir), + env=env, + ) + + +def _cleanup_workers( + processes: Dict[int, subprocess.Popen], +) -> None: + for process in processes.values(): + if process.poll() is None: + process.terminate() + for process in processes.values(): + try: + process.wait(timeout=10) + except subprocess.TimeoutExpired: + process.kill() + process.wait(timeout=10) + + +def _print_summary( + work_dir: Path, + initial_pids: Dict[int, int], + final_pids: Dict[int, int], + recovery_started: bool, +) -> None: + print("") + print("In-place restart demo summary") + print(f" work_dir: {work_dir}") + print(f" recovery_started: {recovery_started}") + for rank in range(WORLD_SIZE): + status = read_json(work_dir / "status" / f"rank_{rank}.json") + status = status or {} + if initial_pids[rank] != final_pids[rank]: + print( + f" rank {rank}: REPLACED " + f"{initial_pids[rank]} -> {final_pids[rank]}, " + f"step={status.get('global_step')}, " + f"recoveries={status.get('recovery_count')}" + ) + else: + print( + f" rank {rank}: CONTINUED with pid {final_pids[rank]}, " + f"step={status.get('global_step')}, " + f"recoveries={status.get('recovery_count')}" + ) + + +def run(args: argparse.Namespace) -> int: + if args.fail_rank < 0: + raise ValueError("this demo requires failure injection") + if args.fail_at_step >= args.steps: + raise ValueError("--fail-at-step must be less than --steps") + if args.checkpoint_interval <= 0: + raise ValueError("--checkpoint-interval must be positive") + + if args.work_dir: + work_dir = Path(args.work_dir).resolve() + work_dir.mkdir(parents=True, exist_ok=True) + if any(work_dir.iterdir()): + raise ValueError( + f"work directory is not empty: {work_dir}; " + "use a fresh directory" + ) + else: + work_dir = Path( + tempfile.mkdtemp(prefix="dlrover-in-place-moe-") + ).resolve() + + recovery_dir = work_dir / "recovery" + recovery_dir.mkdir(parents=True, exist_ok=True) + recovery_command_path = recovery_dir / "command.json" + + processes: Dict[int, subprocess.Popen] = {} + initial_pids: Dict[int, int] = {} + final_pids: Dict[int, int] = {} + try: + for rank in range(WORLD_SIZE): + process = _launch_worker( + rank, + args, + work_dir, + recovery_command_path, + replacement=False, + ) + processes[rank] = process + initial_pids[rank] = process.pid + print( + "[supervisor] initial processes: " + + ", ".join( + f"rank {rank}=pid {initial_pids[rank]}" + for rank in range(WORLD_SIZE) + ) + ) + + deadline = time.monotonic() + args.job_timeout_seconds + recovery_started = False + replacement_pid = None + reported_recovery = set() + + while True: + alive = [ + rank + for rank, process in processes.items() + if process.poll() is None + ] + if not alive: + break + if time.monotonic() >= deadline: + raise TimeoutError("in-place recovery demo timed out") + + failed_processes = { + rank: process + for rank, process in processes.items() + if process.poll() is not None + } + for rank, process in failed_processes.items(): + return_code = process.returncode + if ( + rank == args.fail_rank + and not recovery_started + and return_code != 0 + ): + healthy_pids = { + healthy_rank: healthy_process.pid + for healthy_rank, healthy_process in ( + processes.items() + ) + if healthy_rank != rank + and healthy_process.poll() is None + } + latest = _wait_for_file( + work_dir / "checkpoints" / "latest.json", + timeout_seconds=args.recovery_timeout_seconds, + ) + command = RecoveryCommand( + recovery_epoch=1, + attempt_id=( + f"attempt-{int(time.time() * 1000)}-{rank}" + ), + failed_rank=rank, + checkpoint_path=latest["checkpoint_path"], + checkpoint_step=int(latest["checkpoint_step"]), + reconfigure_uuid=fresh_reconfigure_uuid(), + world_size=WORLD_SIZE, + deadline_unix_ms=int( + ( + time.time() + + args.recovery_timeout_seconds + ) + * 1000 + ), + ) + atomic_write_json( + recovery_command_path, + command.to_dict(), + ) + print( + f"[supervisor] FAILED rank {rank}: " + f"old_pid={process.pid}, exit_code={return_code}" + ) + print( + "[supervisor] healthy processes still running: " + + ", ".join( + f"rank {healthy_rank}=pid {pid}" + for healthy_rank, pid in sorted( + healthy_pids.items() + ) + ) + ) + replacement = _launch_worker( + rank, + args, + work_dir, + recovery_command_path, + replacement=True, + ) + processes[rank] = replacement + replacement_pid = replacement.pid + recovery_started = True + print( + f"[supervisor] REPLACED rank {rank}: " + f"old_pid={process.pid}, " + f"new_pid={replacement.pid}" + ) + elif not recovery_started and return_code != 0: + raise RuntimeError( + f"rank {rank} failed before recovery with " + f"exit code {return_code}" + ) + elif ( + recovery_started + and rank != args.fail_rank + and return_code != 0 + ): + raise RuntimeError( + f"healthy rank {rank} failed during recovery with " + f"exit code {return_code}" + ) + elif ( + recovery_started + and rank == args.fail_rank + and return_code != 0 + ): + raise RuntimeError( + f"replacement rank {rank} failed during recovery " + f"with exit code {return_code}" + ) + + for rank in range(WORLD_SIZE): + if rank in reported_recovery: + continue + status = read_json( + work_dir / "status" / f"rank_{rank}.json" + ) + if ( + status is not None + and int(status.get("recovery_count", 0)) > 0 + and status.get("phase") == "running" + ): + print( + f"[supervisor] RECOVERED rank {rank}: " + f"pid={status['pid']}, " + f"epoch={status['recovery_epoch']}, " + f"step={status['global_step']}" + ) + reported_recovery.add(rank) + + time.sleep(0.2) + + for rank, process in processes.items(): + final_pids[rank] = process.pid + if process.returncode != 0: + raise RuntimeError( + f"rank {rank} exited with code {process.returncode}" + ) + + for rank in range(WORLD_SIZE): + status = read_json( + work_dir / "status" / f"rank_{rank}.json" + ) + if status is None: + raise RuntimeError(f"rank {rank} did not publish status") + if status.get("phase") != "finished": + raise RuntimeError( + f"rank {rank} did not finish: {status}" + ) + if int(status.get("recovery_count", 0)) < 1: + raise RuntimeError( + f"rank {rank} did not complete recovery" + ) + if rank != args.fail_rank: + if int(status["pid"]) != initial_pids[rank]: + raise RuntimeError( + f"healthy rank {rank} changed PID" + ) + elif replacement_pid is None or int(status["pid"]) != ( + replacement_pid + ): + raise RuntimeError( + f"failed rank {rank} was not replaced correctly" + ) + + _print_summary( + work_dir, + initial_pids, + final_pids, + recovery_started, + ) + return 0 + finally: + _cleanup_workers(processes) + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--work-dir", type=str, default="") + parser.add_argument("--steps", type=int, default=40) + parser.add_argument("--log-every", type=int, default=5) + parser.add_argument("--checkpoint-interval", type=int, default=10) + parser.add_argument("--fail-rank", type=int, default=2) + parser.add_argument("--fail-at-step", type=int, default=23) + parser.add_argument("--dist-timeout-seconds", type=int, default=20) + parser.add_argument( + "--recovery-timeout-seconds", type=float, default=180.0 + ) + parser.add_argument( + "--job-timeout-seconds", type=float, default=900.0 + ) + return parser + + +def main() -> None: + args = build_parser().parse_args() + if args.fail_rank < 0 or args.fail_rank >= WORLD_SIZE: + raise SystemExit("--fail-rank must be between 0 and 3") + try: + raise SystemExit(run(args)) + except (OSError, RuntimeError, TimeoutError, ValueError) as error: + print(f"in-place restart demo failed: {error}", file=sys.stderr) + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/examples/in_place_restart/dp4/train.py b/examples/in_place_restart/dp4/train.py new file mode 100755 index 000000000..926ad46ba --- /dev/null +++ b/examples/in_place_restart/dp4/train.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python3 +# Copyright 2026 The DLRover Authors. All rights reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Compatibility entry point for the four-rank MoE example.""" + +from moe_train import main + + +if __name__ == "__main__": + main() diff --git a/examples/in_place_restart/recovery_protocol.py b/examples/in_place_restart/recovery_protocol.py new file mode 100644 index 000000000..a59913584 --- /dev/null +++ b/examples/in_place_restart/recovery_protocol.py @@ -0,0 +1,123 @@ +# Copyright 2026 The DLRover Authors. All rights reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Small file-based control protocol used by the in-place restart demo.""" + +import json +import os +import secrets +import tempfile +import time +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import Any, Dict, Optional + + +@dataclass(frozen=True) +class RecoveryCommand: + """A request to roll back and reconfigure one recovery epoch.""" + + recovery_epoch: int + attempt_id: str + failed_rank: int + checkpoint_path: str + checkpoint_step: int + reconfigure_uuid: int + world_size: int + deadline_unix_ms: int + + def to_dict(self) -> Dict[str, Any]: + return asdict(self) + + @classmethod + def from_dict(cls, value: Dict[str, Any]) -> "RecoveryCommand": + command = cls(**value) + if command.recovery_epoch <= 0: + raise ValueError("recovery_epoch must be positive") + if not command.attempt_id: + raise ValueError("attempt_id must not be empty") + if command.checkpoint_step < 0: + raise ValueError("checkpoint_step must be nonnegative") + if command.reconfigure_uuid <= 0: + raise ValueError("reconfigure_uuid must be positive") + return command + + +def fresh_reconfigure_uuid() -> int: + """Return a positive signed-64-bit UUID for PyTorch reconfigure.""" + + return secrets.randbits(62) + 1 + + +def atomic_write_json(path: Path, value: Dict[str, Any]) -> None: + """Atomically replace a JSON file after syncing it to disk.""" + + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + fd, temporary_name = tempfile.mkstemp( + prefix=f".{path.name}.", + dir=str(path.parent), + text=True, + ) + temporary_path = Path(temporary_name) + try: + with os.fdopen(fd, "w", encoding="utf-8") as stream: + json.dump(value, stream, indent=2, sort_keys=True) + stream.write("\n") + stream.flush() + os.fsync(stream.fileno()) + os.replace(str(temporary_path), str(path)) + finally: + if temporary_path.exists(): + temporary_path.unlink() + + +def read_json(path: Path) -> Optional[Dict[str, Any]]: + """Read a complete JSON file, returning None while it is unavailable.""" + + try: + with Path(path).open("r", encoding="utf-8") as stream: + return json.load(stream) + except (FileNotFoundError, json.JSONDecodeError): + return None + + +def load_recovery_command(path: Path) -> Optional[RecoveryCommand]: + value = read_json(path) + if value is None: + return None + return RecoveryCommand.from_dict(value) + + +def wait_for_recovery_command( + path: Path, + timeout_seconds: float, + minimum_epoch: int = 0, +) -> RecoveryCommand: + """Wait for a recovery command newer than ``minimum_epoch``.""" + + deadline = time.monotonic() + timeout_seconds + last_error: Optional[Exception] = None + while time.monotonic() < deadline: + try: + command = load_recovery_command(path) + except (TypeError, ValueError) as error: + last_error = error + command = None + if command is not None and command.recovery_epoch > minimum_epoch: + return command + time.sleep(0.2) + + detail = f": {last_error}" if last_error is not None else "" + raise TimeoutError( + f"timed out waiting for recovery command at {path}{detail}" + ) diff --git a/examples/in_place_restart/requirements.txt b/examples/in_place_restart/requirements.txt new file mode 100644 index 000000000..e5918e6d1 --- /dev/null +++ b/examples/in_place_restart/requirements.txt @@ -0,0 +1,2 @@ +# PyTorch 2.14 provides the experimental nccl2 reconfigure APIs. +torch>=2.14.0 diff --git a/examples/in_place_restart/tp2_pp2_dp2/README.md b/examples/in_place_restart/tp2_pp2_dp2/README.md new file mode 100644 index 000000000..ce5e665e0 --- /dev/null +++ b/examples/in_place_restart/tp2_pp2_dp2/README.md @@ -0,0 +1,91 @@ +# Eight-GPU TP2 x PP2 x DP2 MoE Recovery + +This demo uses: + +```text +World size = 8 +TP = 2 +PP = 2 +DP = 2 +EP = 1 +``` + +Process groups: + +```text +TP: [0,1] [2,3] [4,5] [6,7] +PP: [0,2] [1,3] [4,6] [5,7] +DP: [0,4] [1,5] [2,6] [3,7] +``` + +Rank 3 is `TP1-PP1-DP0`. It fails at step 23, is replaced with a new process, +and all eight ranks reconfigure the default, TP, PP, and DP groups before +restoring checkpoint step 20. The same trainer also supports a restart-all +baseline that replaces every rank. + +From the repository root: + +```bash +ARTIFACT_DIR=$(mktemp -d /tmp/dlrover-hybrid.XXXXXX) + +python examples/in_place_restart/tp2_pp2_dp2/run_in_place_demo.py \ + --work-dir "$ARTIFACT_DIR" +``` + +Run the restart-all baseline with the same failure point: + +```bash +RESTART_DIR=$(mktemp -d /tmp/dlrover-hybrid-restart.XXXXXX) + +python examples/in_place_restart/tp2_pp2_dp2/run_in_place_demo.py \ + --work-dir "$RESTART_DIR" \ + --recovery-mode restart-all +``` + +Use `--recovery-mode in-place` (the default) to run only the in-place case. +For a single run that executes both modes and compares them, use: + +```bash +COMPARE_DIR=$(mktemp -d /tmp/dlrover-hybrid-compare.XXXXXX) + +python examples/in_place_restart/tp2_pp2_dp2/run_recovery_comparison.py \ + --work-dir "$COMPARE_DIR" \ + --steps 40 \ + --checkpoint-interval 10 \ + --fail-rank 3 \ + --fail-at-step 23 +``` + +The comparison run writes: + +```text +in_place/metrics/loss_rank_2.jsonl +in_place/metrics/summary.json +restart_all/metrics/loss_rank_2.jsonl +restart_all/metrics/summary.json +comparison.json +``` + +It reports the maximum absolute and relative loss difference after the +restored checkpoint, failure-to-resume time for both modes, the number of +process restarts, and the time saved by in-place recovery. Losses are +compared after the checkpoint step because steps 21 and 22 are replayed after +recovery. + +For timing measurements, run the comparison several times with fresh +directories and alternate `--run-order in-place-first` with +`--run-order restart-all-first`. The small demo is sensitive to process-start +and filesystem-cache effects, so one timing sample is not enough for a +production performance claim. + +For the same topology on two nodes with four GPUs per node, use +`../tp2_pp2_dp2_2nodes/README.md`. That variant keeps this training +implementation and replaces the single-node supervisor with two node agents. + +Files: + +- `train.py`: checkpoint-only DLRover-compatible entry point +- `hybrid_moe_train.py`: TP/PP/DP training and recovery implementation +- `run_in_place_demo.py`: eight-rank local supervisor +- `run_recovery_comparison.py`: in-place versus restart-all comparison +- `run.sh`: convenience launcher diff --git a/examples/in_place_restart/tp2_pp2_dp2/hybrid_moe_train.py b/examples/in_place_restart/tp2_pp2_dp2/hybrid_moe_train.py new file mode 100755 index 000000000..51ef00c68 --- /dev/null +++ b/examples/in_place_restart/tp2_pp2_dp2/hybrid_moe_train.py @@ -0,0 +1,1567 @@ +#!/usr/bin/env python3 +# Copyright 2026 The DLRover Authors. All rights reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Eight-rank TP2 x PP2 x DP2 MoE recovery example. + +EP=1 means there is no expert partition across additional expert-parallel +ranks. Stage 0 owns a TP-sharded embedding. Stage 1 owns TP-sharded MoE +experts and a TP-sharded output head. Each stage uses its own data-parallel +group. The same worker runs on one node with eight GPUs or on two nodes with +four GPUs each; the launcher supplies the global and local rank mapping. +""" + +import argparse +import gc +import json +import os +import sys +import threading +import time +from dataclasses import dataclass, field +from datetime import timedelta +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple + +import torch +import torch.distributed as dist +import torch.nn as nn +import torch.nn.functional as F +import torch.optim as optim +from torch.nn.parallel import DistributedDataParallel as DDP +from torch.utils.data import DataLoader, Dataset +from torch.utils.data.distributed import DistributedSampler + +EXAMPLE_ROOT = Path(__file__).resolve().parent.parent +if str(EXAMPLE_ROOT) not in sys.path: + sys.path.insert(0, str(EXAMPLE_ROOT)) + +from recovery_protocol import ( + RecoveryCommand, + atomic_write_json, + fresh_reconfigure_uuid, + load_recovery_command, + read_json, + wait_for_recovery_command, +) + + +WORLD_SIZE = 8 +TP_SIZE = 2 +PP_SIZE = 2 +DP_SIZE = 2 +EP_SIZE = 1 + +BATCH_SIZE = 8 +SEQ_LEN = 32 +VOCAB_SIZE = 1_000 +EMBED_DIM = 128 +EXPERT_HIDDEN_DIM = 512 +NUM_EXPERTS = 4 +TOP_K = 1 +MODEL_SEED = 2026 +DATA_SEED = 2027 +CHECKPOINT_FORMAT_VERSION = 1 +TOPOLOGY_ID = "tp2-pp2-dp2-ep1" + + +@dataclass(frozen=True) +class ParallelTopology: + tp: int = TP_SIZE + pp: int = PP_SIZE + dp: int = DP_SIZE + ep: int = EP_SIZE + world_size: int = WORLD_SIZE + + def coordinates(self, rank: int) -> Tuple[int, int, int]: + if rank < 0 or rank >= self.world_size: + raise ValueError(f"invalid global rank {rank}") + tp_rank = rank % self.tp + pp_rank = (rank // self.tp) % self.pp + dp_rank = rank // (self.tp * self.pp) + return tp_rank, pp_rank, dp_rank + + def rank(self, tp_rank: int, pp_rank: int, dp_rank: int) -> int: + return ( + dp_rank * self.tp * self.pp + + pp_rank * self.tp + + tp_rank + ) + + def tp_groups(self) -> List[Tuple[int, ...]]: + return [ + tuple( + self.rank(tp_rank, pp_rank, dp_rank) + for tp_rank in range(self.tp) + ) + for dp_rank in range(self.dp) + for pp_rank in range(self.pp) + ] + + def pp_groups(self) -> List[Tuple[int, ...]]: + return [ + tuple( + self.rank(tp_rank, pp_rank, dp_rank) + for pp_rank in range(self.pp) + ) + for dp_rank in range(self.dp) + for tp_rank in range(self.tp) + ] + + def dp_groups(self) -> List[Tuple[int, ...]]: + return [ + tuple( + self.rank(tp_rank, pp_rank, dp_rank) + for dp_rank in range(self.dp) + ) + for pp_rank in range(self.pp) + for tp_rank in range(self.tp) + ] + + +@dataclass(frozen=True) +class GroupSpec: + group_id: str + category: str + members: Tuple[int, ...] + index: int + + +@dataclass +class HybridRuntime: + rank: int + local_rank: int + device: torch.device + topology: ParallelTopology + tp_rank: int + pp_rank: int + dp_rank: int + work_dir: Path + store: Any + default_group: Any + group_specs: List[GroupSpec] + local_groups: Dict[str, Any] + stage_module: nn.Module + optimizer: optim.Optimizer + model: Optional[DDP] = None + abort_lock: Any = field( + default_factory=threading.Lock, + repr=False, + ) + aborted_epoch: int = 0 + global_step: int = 0 + epoch: int = 0 + batch_index: int = 0 + recovery_epoch: int = 0 + recovery_count: int = 0 + process_generation: int = 0 + restart_count: int = 0 + recovery_ready_unix_ms: int = 0 + + @property + def tp_group(self) -> Any: + return self._group("tp") + + @property + def pp_group(self) -> Any: + return self._group("pp") + + @property + def dp_group(self) -> Any: + return self._group("dp") + + def _group(self, category: str) -> Any: + for group_id, group in self.local_groups.items(): + if group_id.startswith(f"{category}-"): + return group + raise RuntimeError(f"local {category} group is unavailable") + + +class RecoveryRequested(Exception): + def __init__(self, command: RecoveryCommand) -> None: + super().__init__( + f"recovery epoch {command.recovery_epoch} requested" + ) + self.command = command + + +class _AllReduceForward(torch.autograd.Function): + @staticmethod + def forward( + ctx: Any, + tensor: torch.Tensor, + group: Any, + ) -> torch.Tensor: + del ctx + output = tensor.contiguous().clone() + dist.all_reduce(output, group=group) + return output + + @staticmethod + def backward( + ctx: Any, + grad_output: torch.Tensor, + ) -> Tuple[torch.Tensor, Any]: + del ctx + return grad_output, None + + +class _AllGatherLastDim(torch.autograd.Function): + @staticmethod + def forward( + ctx: Any, + tensor: torch.Tensor, + group: Any, + ) -> torch.Tensor: + tensor = tensor.contiguous() + world_size = dist.get_world_size(group) + gathered = [torch.empty_like(tensor) for _ in range(world_size)] + dist.all_gather(gathered, tensor, group=group) + ctx.group = group + ctx.world_size = world_size + ctx.rank = dist.get_rank(group) + return torch.cat(gathered, dim=-1) + + @staticmethod + def backward( + ctx: Any, + grad_output: torch.Tensor, + ) -> Tuple[torch.Tensor, Any]: + del ctx.group + chunks = grad_output.contiguous().chunk( + ctx.world_size, + dim=-1, + ) + return chunks[ctx.rank], None + + +class ParallelEmbeddingStage(nn.Module): + def __init__( + self, + device: torch.device, + tp_rank: int, + tp_group: Any, + ) -> None: + super().__init__() + self.tp_group = tp_group + full_weight = _seeded_tensor( + (VOCAB_SIZE, EMBED_DIM), + MODEL_SEED, + scale=0.02, + ) + local_width = EMBED_DIM // TP_SIZE + start = tp_rank * local_width + local_weight = full_weight[:, start : start + local_width] + self.weight = nn.Parameter(local_weight.to(device)) + + def forward(self, tokens: torch.Tensor) -> torch.Tensor: + local_hidden = F.embedding(tokens, self.weight) + return _AllGatherLastDim.apply(local_hidden, self.tp_group) + + +class TensorParallelExpert(nn.Module): + def __init__( + self, + device: torch.device, + tp_rank: int, + seed: int, + ) -> None: + super().__init__() + local_hidden = EXPERT_HIDDEN_DIM // TP_SIZE + self.up = nn.Linear( + EMBED_DIM, + local_hidden, + bias=False, + device=device, + ) + self.down = nn.Linear( + local_hidden, + EMBED_DIM, + bias=False, + device=device, + ) + + full_up = _seeded_tensor( + (EXPERT_HIDDEN_DIM, EMBED_DIM), + seed, + scale=0.02, + ) + start = tp_rank * local_hidden + self.up.weight.data.copy_( + full_up[start : start + local_hidden].to(device) + ) + + full_down = _seeded_tensor( + (EMBED_DIM, EXPERT_HIDDEN_DIM), + seed + 1, + scale=0.02, + ) + self.down.weight.data.copy_( + full_down[:, start : start + local_hidden].to(device) + ) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + return self.down(F.relu(self.up(x))) + + +class ParallelMoEStage(nn.Module): + def __init__( + self, + device: torch.device, + tp_rank: int, + tp_group: Any, + ) -> None: + super().__init__() + self.tp_group = tp_group + self.num_experts = NUM_EXPERTS + self.top_k = TOP_K + + gate_weight = _seeded_tensor( + (NUM_EXPERTS, EMBED_DIM), + MODEL_SEED + 100, + scale=0.02, + ) + self.gate = nn.Linear( + EMBED_DIM, + NUM_EXPERTS, + bias=False, + device=device, + ) + self.gate.weight.data.copy_(gate_weight.to(device)) + + self.experts = nn.ModuleList( + [ + TensorParallelExpert( + device, + tp_rank, + MODEL_SEED + 200 + expert_id * 10, + ) + for expert_id in range(NUM_EXPERTS) + ] + ) + + full_head = _seeded_tensor( + (VOCAB_SIZE, EMBED_DIM), + MODEL_SEED + 500, + scale=0.02, + ) + local_vocab = VOCAB_SIZE // TP_SIZE + start = tp_rank * local_vocab + self.head_weight = nn.Parameter( + full_head[start : start + local_vocab].to(device) + ) + self.head_bias = nn.Parameter( + torch.zeros(local_vocab, device=device) + ) + + def forward( + self, + x: torch.Tensor, + ) -> Tuple[torch.Tensor, torch.Tensor]: + batch_size, seq_len, embed_dim = x.shape + flat_x = x.reshape(-1, embed_dim) + probs = F.softmax(self.gate(flat_x), dim=-1) + weights, selected = torch.topk( + probs, + self.top_k, + dim=-1, + ) + + local_output = torch.zeros_like(flat_x) + for expert_id, expert in enumerate(self.experts): + selected_for_expert = selected == expert_id + expert_weight = ( + weights * selected_for_expert + ).sum(dim=-1, keepdim=True) + mask = expert_weight.squeeze(-1) > 0 + if mask.any(): + local_output[mask] += ( + expert(flat_x[mask]) * expert_weight[mask] + ) + + full_output = _AllReduceForward.apply( + local_output, + self.tp_group, + ) + local_logits = F.linear( + full_output, + self.head_weight, + self.head_bias, + ) + logits = _AllGatherLastDim.apply( + local_logits, + self.tp_group, + ) + logits = logits.reshape(batch_size, seq_len, VOCAB_SIZE) + + top1 = torch.argmax(probs, dim=-1) + density = ( + torch.bincount(top1, minlength=NUM_EXPERTS).float() + / top1.numel() + ) + aux_metric = ( + (NUM_EXPERTS * density) + * (NUM_EXPERTS * probs.mean(dim=0)) + ).mean() + return logits, aux_metric.detach() + + +class LazyRandomDataset(Dataset): + def __init__( + self, + sequence_length: int, + length: int = 10_000, + seed: int = DATA_SEED, + ) -> None: + self.sequence_length = sequence_length + self.length = length + self.seed = seed + + def __getitem__(self, index: int) -> Tuple[torch.Tensor, torch.Tensor]: + generator = torch.Generator() + generator.manual_seed(self.seed + int(index)) + tokens = torch.randint( + 0, + VOCAB_SIZE, + (self.sequence_length,), + generator=generator, + ) + return tokens, tokens + + def __len__(self) -> int: + return self.length + + +def _seeded_tensor( + shape: Tuple[int, ...], + seed: int, + scale: float, +) -> torch.Tensor: + generator = torch.Generator() + generator.manual_seed(seed) + return torch.randn(shape, generator=generator) * scale + + +def _log(rank: int, message: str) -> None: + node_rank = int(os.getenv("NODE_RANK", "0")) + local_rank = int(os.getenv("LOCAL_RANK", str(rank))) + print( + f"[rank {rank} node {node_rank} local_rank {local_rank} " + f"pid {os.getpid()}] {message}", + flush=True, + ) + + +def _group_specs(topology: ParallelTopology) -> List[GroupSpec]: + specs = [ + GroupSpec( + group_id="default", + category="default", + members=tuple(range(topology.world_size)), + index=0, + ) + ] + for category, groups in ( + ("tp", topology.tp_groups()), + ("pp", topology.pp_groups()), + ("dp", topology.dp_groups()), + ): + for index, members in enumerate(groups, start=1): + specs.append( + GroupSpec( + group_id=f"{category}-{index - 1}", + category=category, + members=members, + index=len(specs), + ) + ) + return specs + + +def _get_default_group() -> Any: + getter = getattr(dist, "_get_default_group", None) + if getter is None: + getter = dist.distributed_c10d._get_default_group + return getter() + + +def _require_reconfigure_api() -> None: + required = ( + "_supports_reconfigure", + "_get_reconfigure_handle", + "_reconfigure", + "get_backend_impl", + ) + missing = [name for name in required if not hasattr(dist, name)] + if missing: + raise RuntimeError( + f"PyTorch reconfigure APIs are missing: {missing}" + ) + if not dist.is_backend_available("nccl2"): + raise RuntimeError("the nccl2 backend is unavailable") + + +def _new_reconfigurable_group( + members: Tuple[int, ...], + timeout: timedelta, +) -> Any: + options = dist.ProcessGroupNCCL.Options() + options.enable_reconfigure = True + return dist.new_group( + ranks=list(members), + backend="nccl2", + pg_options=options, + sort_ranks=True, + timeout=timeout, + ) + + +def _setup_distributed( + args: argparse.Namespace, +) -> Tuple[ + int, + int, + torch.device, + Any, + Any, + List[GroupSpec], + Dict[str, Any], +]: + if int(os.environ["WORLD_SIZE"]) != WORLD_SIZE: + raise ValueError( + f"this demo requires WORLD_SIZE={WORLD_SIZE}" + ) + rank = int(os.environ["RANK"]) + local_rank = int(os.environ["LOCAL_RANK"]) + if rank < 0 or rank >= WORLD_SIZE: + raise ValueError(f"RANK must be in [0, {WORLD_SIZE})") + local_world_size = int( + os.getenv("LOCAL_WORLD_SIZE", str(WORLD_SIZE)) + ) + if local_world_size <= 0 or WORLD_SIZE % local_world_size != 0: + raise ValueError( + "LOCAL_WORLD_SIZE must be a positive divisor of WORLD_SIZE" + ) + expected_node_rank = rank // local_world_size + expected_node_count = WORLD_SIZE // local_world_size + node_count = int( + os.getenv("NODE_WORLD_SIZE", str(expected_node_count)) + ) + if node_count != expected_node_count: + raise ValueError( + f"WORLD_SIZE={WORLD_SIZE} and " + f"LOCAL_WORLD_SIZE={local_world_size} require " + f"NODE_WORLD_SIZE={expected_node_count}, not {node_count}" + ) + node_rank = int( + os.getenv("NODE_RANK", str(expected_node_rank)) + ) + expected_local_rank = rank % local_world_size + if node_rank != expected_node_rank: + raise ValueError( + f"RANK={rank} belongs to node {expected_node_rank}, " + f"not NODE_RANK={node_rank}" + ) + if local_rank != expected_local_rank: + raise ValueError( + f"RANK={rank} requires LOCAL_RANK={expected_local_rank}, " + f"not {local_rank}" + ) + torch.cuda.set_device(local_rank) + device = torch.device("cuda", local_rank) + + topology = ParallelTopology() + work_dir = Path(args.work_dir) + if args.store_path: + store_path = Path(args.store_path) + else: + store_path = work_dir / "rendezvous" / "store" + store_path.parent.mkdir(parents=True, exist_ok=True) + store = dist.FileStore(str(store_path), WORLD_SIZE) + + _require_reconfigure_api() + dist.init_process_group( + backend="nccl2", + store=store, + rank=rank, + world_size=WORLD_SIZE, + timeout=timedelta(seconds=args.dist_timeout_seconds), + enable_reconfigure=True, + ) + default_group = _get_default_group() + + specs = _group_specs(topology) + local_groups: Dict[str, Any] = {} + for spec in specs: + if spec.category == "default": + continue + group = _new_reconfigurable_group( + spec.members, + timedelta(seconds=args.dist_timeout_seconds), + ) + if rank in spec.members: + local_groups[spec.group_id] = group + + local_groups["default"] = default_group + return ( + rank, + local_rank, + device, + store, + default_group, + specs, + local_groups, + ) + + +def _store_barrier( + store: Any, + attempt_id: str, + phase: str, + rank: int, + timeout_seconds: float, +) -> None: + barrier_store = dist.PrefixStore( + f"atorch-hybrid/{attempt_id}", + store, + ) + barrier_store.set(f"{phase}/done/{rank}", "1") + keys = [ + f"{phase}/done/{peer}" for peer in range(WORLD_SIZE) + ] + barrier_store.wait( + keys, + timedelta(seconds=timeout_seconds), + ) + + +def _reconfigure_group( + runtime: HybridRuntime, + spec: GroupSpec, + attempt_id: str, + base_uuid: int, + timeout_seconds: float, +) -> None: + if runtime.rank not in spec.members: + return + group = runtime.local_groups[spec.group_id] + if not dist._supports_reconfigure(group): + raise RuntimeError( + f"process group {spec.group_id} cannot be reconfigured" + ) + + group_store = dist.PrefixStore( + f"atorch-hybrid/{attempt_id}/handles/{spec.group_id}", + runtime.store, + ) + group_store.set( + f"rank/{runtime.rank}", + dist._get_reconfigure_handle(group), + ) + handle_keys = [ + f"rank/{member}" for member in spec.members + ] + group_store.wait( + handle_keys, + timedelta(seconds=timeout_seconds), + ) + handles = [ + group_store.get(key).decode("utf-8") + for key in handle_keys + ] + + work = dist._reconfigure( + uuid=base_uuid + spec.index, + handles=handles, + group=group, + timeout=timedelta(seconds=timeout_seconds), + ) + work.wait() + _log( + runtime.rank, + f"reconfigured {spec.group_id} members={list(spec.members)}", + ) + + +def _reconfigure_topology( + runtime: HybridRuntime, + attempt_id: str, + base_uuid: int, + timeout_seconds: float, +) -> None: + by_category: Dict[str, List[GroupSpec]] = {} + for spec in runtime.group_specs: + by_category.setdefault(spec.category, []).append(spec) + + for category in ("default", "tp", "pp", "dp"): + for spec in by_category[category]: + _reconfigure_group( + runtime, + spec, + attempt_id, + base_uuid, + timeout_seconds, + ) + _store_barrier( + runtime.store, + attempt_id, + f"reconfigured/{category}", + runtime.rank, + timeout_seconds, + ) + + +def _validate_local_groups(runtime: HybridRuntime) -> None: + default_tensor = torch.ones(1, device=runtime.device) + dist.all_reduce(default_tensor) + if default_tensor.item() != WORLD_SIZE: + raise RuntimeError("default process group validation failed") + + tp_tensor = torch.ones(1, device=runtime.device) + dist.all_reduce(tp_tensor, group=runtime.tp_group) + if tp_tensor.item() != TP_SIZE: + raise RuntimeError("TP process group validation failed") + + dp_tensor = torch.ones(1, device=runtime.device) + dist.all_reduce(dp_tensor, group=runtime.dp_group) + if dp_tensor.item() != DP_SIZE: + raise RuntimeError("DP process group validation failed") + + pp_tensor = torch.ones(1, device=runtime.device) + pp_rank = dist.get_rank(group=runtime.pp_group) + if pp_rank == 0: + dist.send( + pp_tensor, + group=runtime.pp_group, + group_dst=1, + ) + else: + dist.recv( + pp_tensor, + group=runtime.pp_group, + group_src=0, + ) + if pp_tensor.item() != 1: + raise RuntimeError("PP process group validation failed") + + _log(runtime.rank, "TP/PP/DP group validation passed") + + +def _bootstrap_topology( + runtime: HybridRuntime, + timeout_seconds: float, +) -> None: + uuid_key = "atorch-hybrid/bootstrap-uuid" + if runtime.rank == 0: + runtime.store.set(uuid_key, str(fresh_reconfigure_uuid())) + runtime.store.wait( + [uuid_key], + timedelta(seconds=timeout_seconds), + ) + base_uuid = int(runtime.store.get(uuid_key).decode("utf-8")) + _reconfigure_topology( + runtime, + "bootstrap", + base_uuid, + timeout_seconds, + ) + _validate_local_groups(runtime) + + +def _find_local_group( + local_groups: Dict[str, Any], + category: str, +) -> Any: + for group_id, group in local_groups.items(): + if group_id.startswith(f"{category}-"): + return group + raise RuntimeError(f"local {category} group is unavailable") + + +def _wrap_ddp(runtime: HybridRuntime) -> DDP: + return DDP( + runtime.stage_module, + device_ids=[runtime.local_rank], + output_device=runtime.local_rank, + process_group=runtime.dp_group, + find_unused_parameters=True, + ) + + +def _write_status( + runtime: HybridRuntime, + phase: str, + **extra: Any, +) -> None: + local_world_size = int( + os.getenv("LOCAL_WORLD_SIZE", str(WORLD_SIZE)) + ) + status = { + "rank": runtime.rank, + "pid": os.getpid(), + "local_rank": runtime.local_rank, + "local_world_size": local_world_size, + "node_rank": runtime.rank // local_world_size, + "phase": phase, + "global_step": runtime.global_step, + "epoch": runtime.epoch, + "batch_index": runtime.batch_index, + "tp_rank": runtime.tp_rank, + "pp_rank": runtime.pp_rank, + "dp_rank": runtime.dp_rank, + "recovery_epoch": runtime.recovery_epoch, + "recovery_count": runtime.recovery_count, + "process_generation": runtime.process_generation, + "restart_count": runtime.restart_count, + "recovery_ready_unix_ms": runtime.recovery_ready_unix_ms, + "timestamp_unix_ms": int(time.time() * 1000), + } + status.update(extra) + atomic_write_json( + runtime.work_dir + / "status" + / f"rank_{runtime.rank}.json", + status, + ) + + +def _append_loss_metric( + runtime: HybridRuntime, + loss: torch.Tensor, + router_metric: torch.Tensor, +) -> None: + metrics_dir = runtime.work_dir / "metrics" + metrics_dir.mkdir(parents=True, exist_ok=True) + path = metrics_dir / f"loss_rank_{runtime.rank}.jsonl" + record = { + "rank": runtime.rank, + "pid": os.getpid(), + "global_step": runtime.global_step, + "loss": float(loss.detach().item()), + "router_metric": float(router_metric.detach().item()), + "process_generation": runtime.process_generation, + "recovery_epoch": runtime.recovery_epoch, + "recovery_count": runtime.recovery_count, + "restart_count": runtime.restart_count, + "timestamp_unix_ms": int(time.time() * 1000), + } + with path.open("a", encoding="utf-8") as stream: + stream.write(json.dumps(record, sort_keys=True)) + stream.write("\n") + stream.flush() + os.fsync(stream.fileno()) + + +def _save_checkpoint( + runtime: HybridRuntime, + args: argparse.Namespace, +) -> None: + checkpoint_root = runtime.work_dir / "checkpoints" + checkpoint_dir = ( + checkpoint_root / f"checkpoint-{runtime.global_step:08d}" + ) + checkpoint_dir.mkdir(parents=True, exist_ok=True) + + torch.cuda.synchronize(runtime.local_rank) + rank_path = checkpoint_dir / f"rank_{runtime.rank}.pt" + temporary_path = rank_path.with_suffix(".pt.tmp") + payload = { + "format_version": CHECKPOINT_FORMAT_VERSION, + "topology_id": TOPOLOGY_ID, + "rank": runtime.rank, + "tp_rank": runtime.tp_rank, + "pp_rank": runtime.pp_rank, + "dp_rank": runtime.dp_rank, + "model": runtime.stage_module.state_dict(), + "optimizer": runtime.optimizer.state_dict(), + "cursor": { + "global_step": runtime.global_step, + "epoch": runtime.epoch, + "batch_index": runtime.batch_index, + }, + } + with temporary_path.open("wb") as stream: + torch.save(payload, stream) + stream.flush() + os.fsync(stream.fileno()) + os.replace(str(temporary_path), str(rank_path)) + + barrier_id = ( + f"checkpoint-{runtime.recovery_epoch}-" + f"{runtime.global_step}" + ) + _store_barrier( + runtime.store, + barrier_id, + "written", + runtime.rank, + args.recovery_timeout_seconds, + ) + if runtime.rank == 0: + manifest_path = checkpoint_root / "latest.json" + manifest = { + "checkpoint_id": f"hybrid-moe-step-{runtime.global_step}", + "checkpoint_step": runtime.global_step, + "checkpoint_path": str(manifest_path.resolve()), + "topology_id": TOPOLOGY_ID, + "world_size": WORLD_SIZE, + "rank_paths": { + str(rank): str( + ( + checkpoint_dir / f"rank_{rank}.pt" + ).resolve() + ) + for rank in range(WORLD_SIZE) + }, + } + atomic_write_json(manifest_path, manifest) + _log( + runtime.rank, + f"committed hybrid checkpoint at step " + f"{runtime.global_step}", + ) + + +def _load_checkpoint( + runtime: HybridRuntime, + checkpoint_path: str, + checkpoint_step: int, +) -> None: + manifest = read_json(Path(checkpoint_path)) + if manifest is None: + raise ValueError( + "cannot read checkpoint manifest: " + f"{checkpoint_path}" + ) + if manifest.get("topology_id") != TOPOLOGY_ID: + raise ValueError("checkpoint topology does not match") + if manifest.get("world_size") != WORLD_SIZE: + raise ValueError("checkpoint world size does not match") + if manifest.get("checkpoint_step") != checkpoint_step: + raise ValueError("checkpoint step does not match") + + rank_path = Path(manifest["rank_paths"][str(runtime.rank)]) + checkpoint = torch.load( + rank_path, + map_location=runtime.device, + weights_only=True, + ) + if checkpoint.get("rank") != runtime.rank: + raise ValueError("checkpoint rank does not match") + if checkpoint.get("tp_rank") != runtime.tp_rank: + raise ValueError("checkpoint TP rank does not match") + if checkpoint.get("pp_rank") != runtime.pp_rank: + raise ValueError("checkpoint PP rank does not match") + if checkpoint.get("dp_rank") != runtime.dp_rank: + raise ValueError("checkpoint DP rank does not match") + + cursor = checkpoint["cursor"] + runtime.stage_module.load_state_dict( + checkpoint["model"], + strict=True, + ) + runtime.optimizer.load_state_dict(checkpoint["optimizer"]) + runtime.global_step = cursor["global_step"] + runtime.epoch = cursor["epoch"] + runtime.batch_index = cursor["batch_index"] + _log( + runtime.rank, + f"restored local shard at step {runtime.global_step}, " + f"epoch {runtime.epoch}, batch {runtime.batch_index}", + ) + + +def _abort_local_groups( + runtime: HybridRuntime, + recovery_epoch: int = 0, +) -> None: + with runtime.abort_lock: + if ( + recovery_epoch > 0 + and recovery_epoch <= runtime.aborted_epoch + ): + return + for group_id, group in runtime.local_groups.items(): + backend = dist.get_backend_impl( + group=group, + device=runtime.device, + ) + try: + backend.abort() + _log(runtime.rank, f"revoked {group_id}") + except Exception as error: + _log( + runtime.rank, + f"failed to revoke {group_id}: {error}", + ) + runtime.aborted_epoch = recovery_epoch + + +def _start_recovery_watcher( + runtime: HybridRuntime, + args: argparse.Namespace, +) -> Tuple[threading.Event, threading.Thread]: + stop_event = threading.Event() + command_path = Path(args.recovery_command) + + def watch() -> None: + seen_epoch = runtime.recovery_epoch + while not stop_event.wait(0.2): + try: + command = load_recovery_command(command_path) + except (TypeError, ValueError): + continue + if ( + command is not None + and command.recovery_epoch > seen_epoch + ): + _log( + runtime.rank, + f"control watcher observed recovery epoch " + f"{command.recovery_epoch}; revoking local groups", + ) + _abort_local_groups( + runtime, + command.recovery_epoch, + ) + seen_epoch = command.recovery_epoch + + thread = threading.Thread( + target=watch, + name=f"recovery-watcher-rank-{runtime.rank}", + daemon=True, + ) + thread.start() + return stop_event, thread + + +def _detach_ddp(runtime: HybridRuntime) -> None: + if runtime.model is None: + return + runtime.stage_module = runtime.model.module + runtime.model = None + gc.collect() + + +def _wait_for_all_ranks( + runtime: HybridRuntime, + attempt_id: str, + timeout_seconds: float, +) -> None: + ready_store = dist.PrefixStore( + f"atorch-hybrid/{attempt_id}", + runtime.store, + ) + ready_store.set(f"ready/{runtime.rank}", "1") + ready_keys = [ + f"ready/{rank}" for rank in range(WORLD_SIZE) + ] + ready_store.wait( + ready_keys, + timedelta(seconds=timeout_seconds), + ) + + +def _recover( + runtime: HybridRuntime, + args: argparse.Namespace, + command: RecoveryCommand, + replacement: bool, +) -> None: + if command.world_size != WORLD_SIZE: + raise ValueError("recovery world size does not match") + if replacement and command.failed_rank != runtime.rank: + raise ValueError("replacement rank does not match failure") + remaining = command.deadline_unix_ms / 1000.0 - time.time() + if remaining <= 0: + raise TimeoutError("recovery deadline expired") + timeout_seconds = min( + args.recovery_timeout_seconds, + remaining, + ) + _log( + runtime.rank, + f"entering recovery epoch {command.recovery_epoch}; " + f"replacement={replacement}", + ) + + _write_status( + runtime, + "quiescing", + recovery_epoch=command.recovery_epoch, + ) + if not replacement: + _abort_local_groups(runtime, command.recovery_epoch) + _detach_ddp(runtime) + runtime.optimizer.zero_grad(set_to_none=True) + _write_status( + runtime, + "quiesced", + recovery_epoch=command.recovery_epoch, + ) + _log( + runtime.rank, + "local training is quiesced; reconfiguring topology", + ) + + _reconfigure_topology( + runtime, + command.attempt_id, + command.reconfigure_uuid, + timeout_seconds, + ) + _validate_local_groups(runtime) + _write_status( + runtime, + "comm_ready", + recovery_epoch=command.recovery_epoch, + ) + + _load_checkpoint( + runtime, + command.checkpoint_path, + command.checkpoint_step, + ) + runtime.model = _wrap_ddp(runtime) + runtime.model.train() + runtime.optimizer.zero_grad(set_to_none=True) + runtime.recovery_epoch = command.recovery_epoch + runtime.recovery_count += 1 + + _wait_for_all_ranks( + runtime, + command.attempt_id, + timeout_seconds, + ) + dist.barrier() + runtime.recovery_ready_unix_ms = int(time.time() * 1000) + _write_status(runtime, "running") + _log( + runtime.rank, + f"hybrid recovery complete at step {runtime.global_step}; " + f"epoch={runtime.recovery_epoch}", + ) + + +def _make_loader( + runtime: HybridRuntime, + dataset: Dataset, +) -> DataLoader: + sampler = DistributedSampler( + dataset, + num_replicas=DP_SIZE, + rank=runtime.dp_rank, + shuffle=True, + seed=DATA_SEED, + ) + sampler.set_epoch(runtime.epoch) + return DataLoader( + dataset, + batch_size=BATCH_SIZE, + sampler=sampler, + num_workers=0, + pin_memory=True, + ) + + +def _sync_tp_gradients( + runtime: HybridRuntime, + hidden: torch.Tensor, +) -> torch.Tensor: + if runtime.pp_rank != 1: + raise RuntimeError("only the PP last stage can sync gradients") + if hidden.grad is None: + raise RuntimeError("pipeline input gradient is missing") + + hidden_grad = hidden.grad.contiguous() + dist.all_reduce(hidden_grad, group=runtime.tp_group) + + gate_grad = runtime.stage_module.gate.weight.grad + if gate_grad is None: + raise RuntimeError("MoE gate gradient is missing") + dist.all_reduce(gate_grad, group=runtime.tp_group) + return hidden_grad + + +def _pipeline_step( + runtime: HybridRuntime, + inputs: torch.Tensor, + targets: torch.Tensor, +) -> Tuple[Optional[torch.Tensor], Optional[torch.Tensor]]: + assert runtime.model is not None + runtime.optimizer.zero_grad(set_to_none=True) + + if runtime.pp_rank == 0: + hidden = runtime.model(inputs) + dist.send( + hidden.detach().contiguous(), + group=runtime.pp_group, + group_dst=1, + ) + + hidden_grad = torch.empty_like(hidden) + with torch.no_grad(): + dist.recv( + hidden_grad, + group=runtime.pp_group, + group_src=1, + ) + hidden.backward(hidden_grad) + runtime.optimizer.step() + return None, None + + hidden = torch.empty( + (BATCH_SIZE, SEQ_LEN, EMBED_DIM), + device=runtime.device, + ) + with torch.no_grad(): + dist.recv( + hidden, + group=runtime.pp_group, + group_src=0, + ) + hidden.requires_grad_(True) + + logits, aux_metric = runtime.model(hidden) + loss = F.cross_entropy( + logits.reshape(-1, VOCAB_SIZE), + targets.reshape(-1), + ) + loss.backward() + hidden_grad = _sync_tp_gradients(runtime, hidden) + dist.send( + hidden_grad, + group=runtime.pp_group, + group_dst=0, + ) + runtime.optimizer.step() + return loss, aux_metric + + +def _is_comm_error(error: Exception) -> bool: + name = type(error).__name__.lower() + message = str(error).lower() + return ( + "backend" in name + or "nccl" in message + or "collective" in message + or "timed out" in message + or "socket" in message + ) + + +def _run_training_segment( + runtime: HybridRuntime, + args: argparse.Namespace, +) -> None: + dataset = LazyRandomDataset(SEQ_LEN) + command_path = Path(args.recovery_command) + interval_steps = 0 + interval_start = time.perf_counter() + + while runtime.global_step < args.steps: + loader = _make_loader(runtime, dataset) + batches_processed = 0 + for batch_index, (inputs, targets) in enumerate(loader): + if runtime.global_step >= args.steps: + break + if batch_index < runtime.batch_index: + continue + + command = load_recovery_command(command_path) + if ( + command is not None + and command.recovery_epoch > runtime.recovery_epoch + ): + raise RecoveryRequested(command) + + if ( + runtime.rank == args.fail_rank + and runtime.global_step == args.fail_at_step + ): + _write_status( + runtime, + "injected_failure", + injected_failure=True, + ) + _log( + runtime.rank, + f"injecting failure at step " + f"{runtime.global_step}", + ) + os._exit(17) + + inputs = inputs.to(runtime.device) + targets = targets.to(runtime.device) + loss, aux_metric = _pipeline_step( + runtime, + inputs, + targets, + ) + + runtime.global_step += 1 + runtime.batch_index = batch_index + 1 + if runtime.batch_index >= len(loader): + runtime.epoch += 1 + runtime.batch_index = 0 + batches_processed += 1 + interval_steps += 1 + + if ( + args.checkpoint_interval > 0 + and runtime.global_step + % args.checkpoint_interval + == 0 + ): + _save_checkpoint(runtime, args) + + is_logger_rank = ( + runtime.pp_rank == 1 + and runtime.tp_rank == 0 + and runtime.dp_rank == 0 + ) + if is_logger_rank and loss is not None: + assert aux_metric is not None + _append_loss_metric( + runtime, + loss, + aux_metric, + ) + should_log = ( + interval_steps >= args.log_every + or runtime.global_step >= args.steps + ) + if is_logger_rank and should_log and loss is not None: + torch.cuda.synchronize(runtime.local_rank) + interval_seconds = time.perf_counter() - interval_start + elapsed_ms = ( + interval_seconds * 1000 / interval_steps + ) + print( + f"iteration {runtime.global_step}/{args.steps} " + f"elapsed_ms={elapsed_ms:.3f} " + f"loss={loss.item():.6f} " + f"router_metric={float(aux_metric):.6f}", + flush=True, + ) + interval_start = time.perf_counter() + interval_steps = 0 + + _write_status(runtime, "running") + if runtime.global_step >= args.steps: + break + + if runtime.global_step >= args.steps: + break + if batches_processed == 0: + raise RuntimeError("dataloader made no progress") + + +def train(args: argparse.Namespace) -> None: + ( + rank, + local_rank, + device, + store, + default_group, + specs, + local_groups, + ) = _setup_distributed(args) + topology = ParallelTopology() + tp_rank, pp_rank, dp_rank = topology.coordinates(rank) + + tp_group = _find_local_group(local_groups, "tp") + if pp_rank == 0: + stage_module: nn.Module = ParallelEmbeddingStage( + device, + tp_rank, + tp_group, + ) + else: + stage_module = ParallelMoEStage( + device, + tp_rank, + tp_group, + ) + optimizer = optim.Adam(stage_module.parameters(), lr=1e-3) + runtime = HybridRuntime( + rank=rank, + local_rank=local_rank, + device=device, + topology=topology, + tp_rank=tp_rank, + pp_rank=pp_rank, + dp_rank=dp_rank, + work_dir=Path(args.work_dir), + store=store, + default_group=default_group, + group_specs=specs, + local_groups=local_groups, + stage_module=stage_module, + optimizer=optimizer, + process_generation=args.process_generation, + ) + _write_status(runtime, "initializing") + watcher_stop = None + watcher_thread = None + + try: + if args.replacement: + command = wait_for_recovery_command( + Path(args.recovery_command), + args.recovery_timeout_seconds, + ) + _recover(runtime, args, command, replacement=True) + elif args.resume_checkpoint: + _bootstrap_topology( + runtime, + args.recovery_timeout_seconds, + ) + manifest = read_json(Path(args.resume_checkpoint)) + if manifest is None: + raise ValueError( + "cannot read resume checkpoint manifest: " + f"{args.resume_checkpoint}" + ) + checkpoint_step = int( + manifest.get("checkpoint_step", -1) + ) + if checkpoint_step < 0: + raise ValueError( + "resume checkpoint step is invalid" + ) + _load_checkpoint( + runtime, + args.resume_checkpoint, + checkpoint_step, + ) + runtime.restart_count = 1 + runtime.model = _wrap_ddp(runtime) + runtime.model.train() + runtime.recovery_ready_unix_ms = int(time.time() * 1000) + _write_status( + runtime, + "running", + resumed_from_checkpoint=args.resume_checkpoint, + ) + else: + _bootstrap_topology( + runtime, + args.recovery_timeout_seconds, + ) + runtime.model = _wrap_ddp(runtime) + runtime.model.train() + _save_checkpoint(runtime, args) + _write_status(runtime, "running") + watcher_stop, watcher_thread = ( + _start_recovery_watcher(runtime, args) + ) + + while runtime.global_step < args.steps: + try: + _run_training_segment(runtime, args) + except RecoveryRequested as error: + _recover( + runtime, + args, + error.command, + replacement=False, + ) + except Exception as error: + if not _is_comm_error(error): + raise + _log( + runtime.rank, + f"recoverable communication failure: {error}", + ) + command = wait_for_recovery_command( + Path(args.recovery_command), + args.recovery_timeout_seconds, + minimum_epoch=runtime.recovery_epoch, + ) + _recover( + runtime, + args, + command, + replacement=False, + ) + + dist.barrier() + if ( + args.checkpoint_interval > 0 + and runtime.global_step % args.checkpoint_interval != 0 + ): + _save_checkpoint(runtime, args) + _write_status(runtime, "finished") + _log( + runtime.rank, + f"finished {runtime.global_step} steps; " + f"recoveries={runtime.recovery_count}; " + f"restarts={runtime.restart_count}", + ) + except Exception as error: + _write_status(runtime, "failed", error=repr(error)) + raise + finally: + if watcher_stop is not None: + watcher_stop.set() + if watcher_thread is not None: + watcher_thread.join(timeout=2) + if dist.is_initialized(): + dist.destroy_process_group() + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--work-dir", type=str, required=True) + parser.add_argument("--steps", type=int, default=40) + parser.add_argument("--log-every", type=int, default=5) + parser.add_argument("--checkpoint-interval", type=int, default=10) + parser.add_argument("--recovery-command", type=str, default="") + parser.add_argument("--replacement", action="store_true") + parser.add_argument("--resume-checkpoint", type=str, default="") + parser.add_argument("--store-path", type=str, default="") + parser.add_argument("--process-generation", type=int, default=0) + parser.add_argument("--dist-timeout-seconds", type=int, default=30) + parser.add_argument( + "--recovery-timeout-seconds", + type=float, + default=240.0, + ) + parser.add_argument("--fail-rank", type=int, default=-1) + parser.add_argument("--fail-at-step", type=int, default=-1) + return parser + + +def main() -> None: + args = build_parser().parse_args() + if not torch.cuda.is_available(): + raise SystemExit("this demo requires CUDA devices") + local_world_size = int( + os.getenv("LOCAL_WORLD_SIZE", str(WORLD_SIZE)) + ) + if torch.cuda.device_count() < local_world_size: + raise SystemExit( + f"this worker requires {local_world_size} CUDA devices " + f"on its node, found {torch.cuda.device_count()}" + ) + work_dir = Path(args.work_dir) + work_dir.mkdir(parents=True, exist_ok=True) + if not args.recovery_command: + args.recovery_command = str( + work_dir / "recovery" / "command.json" + ) + train(args) + + +if __name__ == "__main__": + main() diff --git a/examples/in_place_restart/tp2_pp2_dp2/run.sh b/examples/in_place_restart/tp2_pp2_dp2/run.sh new file mode 100755 index 000000000..809755238 --- /dev/null +++ b/examples/in_place_restart/tp2_pp2_dp2/run.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +# Copyright 2026 The DLRover Authors. All rights reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +python "${SCRIPT_DIR}/run_in_place_demo.py" "$@" diff --git a/examples/in_place_restart/tp2_pp2_dp2/run_in_place_demo.py b/examples/in_place_restart/tp2_pp2_dp2/run_in_place_demo.py new file mode 100755 index 000000000..141053b68 --- /dev/null +++ b/examples/in_place_restart/tp2_pp2_dp2/run_in_place_demo.py @@ -0,0 +1,661 @@ +#!/usr/bin/env python3 +# Copyright 2026 The DLRover Authors. All rights reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Run the single-node TP2 x PP2 x DP2 recovery demonstration.""" + +import argparse +import os +import socket +import subprocess +import sys +import tempfile +import time +from pathlib import Path +from typing import Any, Dict + +EXAMPLE_ROOT = Path(__file__).resolve().parent.parent +if str(EXAMPLE_ROOT) not in sys.path: + sys.path.insert(0, str(EXAMPLE_ROOT)) + +from recovery_protocol import ( + RecoveryCommand, + atomic_write_json, + fresh_reconfigure_uuid, + read_json, +) + + +WORLD_SIZE = 8 +TP_SIZE = 2 +PP_SIZE = 2 +DP_SIZE = 2 + + +def _coordinates(rank: int) -> str: + tp_rank = rank % TP_SIZE + pp_rank = (rank // TP_SIZE) % PP_SIZE + dp_rank = rank // (TP_SIZE * PP_SIZE) + return f"TP{tp_rank}-PP{pp_rank}-DP{dp_rank}" + + +def _find_free_port() -> int: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: + sock.bind(("127.0.0.1", 0)) + return int(sock.getsockname()[1]) + + +def _wait_for_file(path: Path, timeout_seconds: float) -> Dict[str, Any]: + deadline = time.monotonic() + timeout_seconds + while time.monotonic() < deadline: + value = read_json(path) + if value is not None: + return value + time.sleep(0.2) + raise TimeoutError(f"timed out waiting for {path}") + + +def _launch_worker( + rank: int, + args: argparse.Namespace, + work_dir: Path, + recovery_command_path: Path, + replacement: bool, + process_generation: int = 0, + resume_checkpoint: str = "", + store_path: str = "", +) -> subprocess.Popen: + env = os.environ.copy() + env.update( + { + "RANK": str(rank), + "LOCAL_RANK": str(rank), + "WORLD_SIZE": str(WORLD_SIZE), + "MASTER_ADDR": "127.0.0.1", + "MASTER_PORT": str(_find_free_port()), + "PYTHONUNBUFFERED": "1", + "OMP_NUM_THREADS": env.get("OMP_NUM_THREADS", "1"), + "TORCH_NCCL_ASYNC_ERROR_HANDLING": "0", + "TORCH_NCCL_BLOCKING_WAIT": "0", + "NCCL_DEBUG": env.get("NCCL_DEBUG", "WARN"), + } + ) + script_dir = Path(__file__).resolve().parent + command = [ + sys.executable, + str(script_dir / "hybrid_moe_train.py"), + "--work-dir", + str(work_dir), + "--steps", + str(args.steps), + "--log-every", + str(args.log_every), + "--checkpoint-interval", + str(args.checkpoint_interval), + "--recovery-command", + str(recovery_command_path), + "--dist-timeout-seconds", + str(args.dist_timeout_seconds), + "--recovery-timeout-seconds", + str(args.recovery_timeout_seconds), + "--process-generation", + str(process_generation), + ] + if resume_checkpoint: + command.extend( + ["--resume-checkpoint", resume_checkpoint] + ) + if store_path: + command.extend(["--store-path", store_path]) + if replacement: + command.append("--replacement") + elif not resume_checkpoint: + command.extend( + [ + "--fail-rank", + str(args.fail_rank), + "--fail-at-step", + str(args.fail_at_step), + ] + ) + return subprocess.Popen( + command, + cwd=str(script_dir), + env=env, + ) + + +def _cleanup_workers( + processes: Dict[int, subprocess.Popen], +) -> None: + for process in processes.values(): + if process.poll() is None: + process.terminate() + for process in processes.values(): + try: + process.wait(timeout=10) + except subprocess.TimeoutExpired: + process.kill() + process.wait(timeout=10) + + +def _status_is_recovered( + status: Dict[str, Any], + recovery_mode: str, +) -> bool: + if status.get("phase") != "running": + return False + if recovery_mode == "in-place": + return int(status.get("recovery_count", 0)) > 0 + return int(status.get("restart_count", 0)) > 0 + + +def _write_run_summary( + work_dir: Path, + summary: Dict[str, Any], +) -> None: + atomic_write_json(work_dir / "metrics" / "summary.json", summary) + + +def run(args: argparse.Namespace) -> int: + if args.fail_rank < 0 or args.fail_rank >= WORLD_SIZE: + raise ValueError("invalid failure rank") + if args.steps <= 0: + raise ValueError("steps must be positive") + if args.checkpoint_interval <= 0: + raise ValueError( + "checkpoint interval must be positive for recovery" + ) + if args.fail_at_step < 0 or args.fail_at_step >= args.steps: + raise ValueError("fail step must be less than total steps") + + if args.work_dir: + work_dir = Path(args.work_dir).resolve() + work_dir.mkdir(parents=True, exist_ok=True) + if any(work_dir.iterdir()): + raise ValueError( + f"work directory is not empty: {work_dir}" + ) + else: + work_dir = Path( + tempfile.mkdtemp(prefix="dlrover-hybrid-moe-") + ).resolve() + + recovery_dir = work_dir / "recovery" + recovery_dir.mkdir(parents=True, exist_ok=True) + command_path = recovery_dir / "command.json" + started_at = time.time() + + processes: Dict[int, subprocess.Popen] = {} + initial_pids: Dict[int, int] = {} + replacement_pid = None + recovery_started = False + reported_recovery = set() + failure_at = None + recovery_started_at = None + all_resumed_at = None + recovery_checkpoint_step = None + + try: + for rank in range(WORLD_SIZE): + process = _launch_worker( + rank, + args, + work_dir, + command_path, + replacement=False, + ) + processes[rank] = process + initial_pids[rank] = process.pid + + print( + "[supervisor] topology: TP2 x PP2 x DP2, EP=1", + flush=True, + ) + print( + "[supervisor] initial processes: " + + ", ".join( + f"rank {rank}/{_coordinates(rank)}" + f"=pid {initial_pids[rank]}" + for rank in range(WORLD_SIZE) + ), + flush=True, + ) + + deadline = time.monotonic() + args.job_timeout_seconds + while True: + alive = [ + rank + for rank, process in processes.items() + if process.poll() is None + ] + if not alive: + break + if time.monotonic() >= deadline: + raise TimeoutError("hybrid recovery demo timed out") + + failed = { + rank: process + for rank, process in processes.items() + if process.poll() is not None + } + for rank, process in failed.items(): + return_code = process.returncode + if ( + rank == args.fail_rank + and not recovery_started + and return_code != 0 + ): + failure_at = time.time() + healthy = { + peer: peer_process.pid + for peer, peer_process in processes.items() + if peer != rank + and peer_process.poll() is None + } + latest = _wait_for_file( + work_dir / "checkpoints" / "latest.json", + args.recovery_timeout_seconds, + ) + recovery_checkpoint_step = int( + latest["checkpoint_step"] + ) + print( + f"[supervisor] FAILED rank {rank}/" + f"{_coordinates(rank)}: old_pid={process.pid}, " + f"exit_code={return_code}", + flush=True, + ) + if args.recovery_mode == "in-place": + print( + "[supervisor] healthy processes still running: " + + ", ".join( + f"rank {peer}=pid {pid}" + for peer, pid in sorted( + healthy.items() + ) + ), + flush=True, + ) + command = RecoveryCommand( + recovery_epoch=1, + attempt_id=( + f"hybrid-{int(time.time() * 1000)}-{rank}" + ), + failed_rank=rank, + checkpoint_path=latest[ + "checkpoint_path" + ], + checkpoint_step=int( + latest["checkpoint_step"] + ), + reconfigure_uuid=fresh_reconfigure_uuid(), + world_size=WORLD_SIZE, + deadline_unix_ms=int( + ( + time.time() + + args.recovery_timeout_seconds + ) + * 1000 + ), + ) + atomic_write_json( + command_path, + command.to_dict(), + ) + replacement = _launch_worker( + rank, + args, + work_dir, + command_path, + replacement=True, + process_generation=1, + ) + processes[rank] = replacement + replacement_pid = replacement.pid + recovery_started = True + recovery_started_at = time.time() + print( + f"[supervisor] REPLACED rank {rank}/" + f"{_coordinates(rank)}: " + f"old_pid={process.pid}, " + f"new_pid={replacement.pid}", + flush=True, + ) + else: + print( + "[supervisor] restart-all mode: " + "terminating all healthy processes", + flush=True, + ) + old_pids = { + peer: peer_process.pid + for peer, peer_process in processes.items() + } + _cleanup_workers(processes) + restart_store = ( + work_dir + / "rendezvous" + / "restart-generation-1" + / "store" + ) + processes.clear() + for restart_rank in range(WORLD_SIZE): + processes[restart_rank] = ( + _launch_worker( + restart_rank, + args, + work_dir, + command_path, + replacement=False, + process_generation=1, + resume_checkpoint=str( + latest["checkpoint_path"] + ), + store_path=str(restart_store), + ) + ) + recovery_started = True + recovery_started_at = time.time() + print( + "[supervisor] RESTARTED ALL ranks: " + + ", ".join( + f"rank {peer}: pid {old_pid} -> " + f"pid {processes[peer].pid}" + for peer, old_pid in sorted( + old_pids.items() + ) + ), + flush=True, + ) + elif not recovery_started and return_code != 0: + raise RuntimeError( + f"rank {rank} failed before recovery with " + f"exit code {return_code}" + ) + elif ( + recovery_started + and return_code != 0 + and rank != args.fail_rank + ): + label = ( + f"healthy rank {rank}" + if args.recovery_mode == "in-place" + else f"restarted rank {rank}" + ) + raise RuntimeError( + f"{label} failed during recovery" + ) + elif ( + recovery_started + and return_code != 0 + and rank == args.fail_rank + ): + label = ( + f"replacement rank {rank}" + if args.recovery_mode == "in-place" + else f"restarted rank {rank}" + ) + raise RuntimeError( + f"{label} failed" + ) + + for rank in range(WORLD_SIZE): + if rank in reported_recovery: + continue + status = read_json( + work_dir / "status" / f"rank_{rank}.json" + ) + if status is not None and _status_is_recovered( + status, + args.recovery_mode, + ): + print( + f"[supervisor] " + f"{args.recovery_mode}: " + f"rank {rank}/" + f"{_coordinates(rank)}: pid={status['pid']}, " + f"epoch={status['recovery_epoch']}, " + f"step={status['global_step']}, " + f"generation={status.get('process_generation')}", + flush=True, + ) + reported_recovery.add(rank) + if ( + len(reported_recovery) == WORLD_SIZE + and all_resumed_at is None + ): + all_resumed_at = time.time() + + time.sleep(0.2) + + for rank, process in processes.items(): + if process.returncode != 0: + raise RuntimeError( + f"rank {rank} exited with {process.returncode}" + ) + status = read_json( + work_dir / "status" / f"rank_{rank}.json" + ) + if status is None or status.get("phase") != "finished": + raise RuntimeError(f"rank {rank} did not finish") + if args.recovery_mode == "in-place": + if int(status.get("recovery_count", 0)) < 1: + raise RuntimeError( + f"rank {rank} did not recover" + ) + if rank != args.fail_rank: + if int(status["pid"]) != initial_pids[rank]: + raise RuntimeError( + f"healthy rank {rank} changed PID" + ) + elif replacement_pid is None or int( + status["pid"] + ) != replacement_pid: + raise RuntimeError( + f"rank {rank} replacement PID is incorrect" + ) + else: + if int(status.get("restart_count", 0)) < 1: + raise RuntimeError( + f"rank {rank} was not restarted" + ) + if int(status["pid"]) == initial_pids[rank]: + raise RuntimeError( + f"rank {rank} retained its old PID" + ) + + if all_resumed_at is None: + ready_times = [] + for rank in range(WORLD_SIZE): + status = read_json( + work_dir / "status" / f"rank_{rank}.json" + ) + if status is not None: + ready_times.append( + int(status.get("recovery_ready_unix_ms", 0)) + ) + if len(ready_times) == WORLD_SIZE and all(ready_times): + all_resumed_at = max(ready_times) / 1000.0 + + finished_at = time.time() + final_pids: Dict[int, int] = {} + for rank in range(WORLD_SIZE): + status = read_json( + work_dir / "status" / f"rank_{rank}.json" + ) + if status is None: + raise RuntimeError(f"missing status for rank {rank}") + final_pids[rank] = int(status["pid"]) + + def _milliseconds(value: float) -> int: + return int(value * 1000) + + summary = { + "mode": args.recovery_mode, + "topology": "TP2 x PP2 x DP2, EP=1", + "steps": args.steps, + "checkpoint_interval": args.checkpoint_interval, + "fail_rank": args.fail_rank, + "fail_at_step": args.fail_at_step, + "recovery_checkpoint_step": recovery_checkpoint_step, + "started_unix_ms": _milliseconds(started_at), + "failure_unix_ms": ( + _milliseconds(failure_at) + if failure_at is not None + else None + ), + "recovery_started_unix_ms": ( + _milliseconds(recovery_started_at) + if recovery_started_at is not None + else None + ), + "all_ranks_resumed_unix_ms": ( + _milliseconds(all_resumed_at) + if all_resumed_at is not None + else None + ), + "finished_unix_ms": _milliseconds(finished_at), + "total_wall_seconds": finished_at - started_at, + "failure_to_all_resumed_seconds": ( + all_resumed_at - failure_at + if ( + all_resumed_at is not None + and failure_at is not None + ) + else None + ), + "recovery_phase_seconds": ( + all_resumed_at - recovery_started_at + if ( + all_resumed_at is not None + and recovery_started_at is not None + ) + else None + ), + "process_restarts": ( + 1 if args.recovery_mode == "in-place" else WORLD_SIZE + ), + "healthy_processes_preserved": ( + WORLD_SIZE - 1 + if args.recovery_mode == "in-place" + else 0 + ), + "initial_pids": { + str(rank): pid + for rank, pid in sorted(initial_pids.items()) + }, + "final_pids": { + str(rank): pid + for rank, pid in sorted(final_pids.items()) + }, + "logger_rank": 2, + "loss_metric_path": str( + (work_dir / "metrics" / "loss_rank_2.jsonl") + .resolve() + ), + } + _write_run_summary(work_dir, summary) + + print("") + print( + "Hybrid in-place restart summary" + if args.recovery_mode == "in-place" + else "Hybrid restart-all summary" + ) + print(f" topology: TP2 x PP2 x DP2, EP=1") + print(f" work_dir: {work_dir}") + for rank in range(WORLD_SIZE): + status = read_json( + work_dir / "status" / f"rank_{rank}.json" + ) + status = status or {} + final_pid = final_pids[rank] + if ( + args.recovery_mode == "in-place" + and final_pid == initial_pids[rank] + ): + result = f"CONTINUED with pid {final_pid}" + else: + action = ( + "REPLACED" + if args.recovery_mode == "in-place" + else "RESTARTED" + ) + result = ( + f"{action}" + f" {initial_pids[rank]} -> {final_pid}" + ) + print( + f" rank {rank}/{_coordinates(rank)}: {result}, " + f"step={status.get('global_step')}, " + f"recoveries={status.get('recovery_count')}, " + f"restarts={status.get('restart_count')}" + ) + resume_seconds = summary["failure_to_all_resumed_seconds"] + resume_text = ( + f"{resume_seconds:.3f}" + if resume_seconds is not None + else "n/a" + ) + print( + " failure_to_all_resumed_seconds=" + f"{resume_text}" + ) + return 0 + finally: + _cleanup_workers(processes) + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--work-dir", type=str, default="") + parser.add_argument("--steps", type=int, default=40) + parser.add_argument("--log-every", type=int, default=5) + parser.add_argument("--checkpoint-interval", type=int, default=10) + parser.add_argument("--fail-rank", type=int, default=3) + parser.add_argument("--fail-at-step", type=int, default=23) + parser.add_argument( + "--recovery-mode", + choices=("in-place", "restart-all"), + default="in-place", + ) + parser.add_argument("--dist-timeout-seconds", type=int, default=30) + parser.add_argument( + "--recovery-timeout-seconds", + type=float, + default=240.0, + ) + parser.add_argument( + "--job-timeout-seconds", + type=float, + default=1200.0, + ) + return parser + + +def main() -> None: + args = build_parser().parse_args() + try: + raise SystemExit(run(args)) + except ( + OSError, + RuntimeError, + TimeoutError, + ValueError, + ) as error: + print(f"hybrid demo failed: {error}", file=sys.stderr) + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/examples/in_place_restart/tp2_pp2_dp2/run_recovery_comparison.py b/examples/in_place_restart/tp2_pp2_dp2/run_recovery_comparison.py new file mode 100755 index 000000000..e495d240d --- /dev/null +++ b/examples/in_place_restart/tp2_pp2_dp2/run_recovery_comparison.py @@ -0,0 +1,352 @@ +#!/usr/bin/env python3 +# Copyright 2026 The DLRover Authors. All rights reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Compare in-place recovery with restarting all eight training processes.""" + +import argparse +import json +import subprocess +import sys +import tempfile +from pathlib import Path +from typing import Any, Dict, List + +EXAMPLE_ROOT = Path(__file__).resolve().parent.parent +if str(EXAMPLE_ROOT) not in sys.path: + sys.path.insert(0, str(EXAMPLE_ROOT)) + +from recovery_protocol import atomic_write_json, read_json + + +MODES = ("in-place", "restart-all") + + +def _read_loss_trace( + path: Path, +) -> Dict[int, Dict[str, Any]]: + """Return the last recorded value for each training step.""" + + values: Dict[int, Dict[str, Any]] = {} + try: + with path.open("r", encoding="utf-8") as stream: + for line in stream: + line = line.strip() + if not line: + continue + value = json.loads(line) + values[int(value["global_step"])] = value + except FileNotFoundError as error: + raise RuntimeError(f"loss trace is missing: {path}") from error + return values + + +def _compare_losses( + in_place_path: Path, + restart_all_path: Path, + checkpoint_step: int, + tolerance: float, +) -> Dict[str, Any]: + in_place = _read_loss_trace(in_place_path) + restart_all = _read_loss_trace(restart_all_path) + common_steps = sorted(set(in_place) & set(restart_all)) + post_checkpoint_steps = [ + step for step in common_steps if step > checkpoint_step + ] + + def _statistics(steps: List[int]) -> Dict[str, Any]: + if not steps: + return { + "steps": [], + "count": 0, + "max_abs_diff": None, + "max_rel_diff": None, + "within_tolerance": False, + } + differences = [] + relative = [] + detail = [] + for step in steps: + left = float(in_place[step]["loss"]) + right = float(restart_all[step]["loss"]) + absolute = abs(left - right) + scale = max(abs(left), abs(right), 1e-12) + differences.append(absolute) + relative.append(absolute / scale) + detail.append( + { + "step": step, + "in_place_loss": left, + "restart_all_loss": right, + "abs_diff": absolute, + "rel_diff": absolute / scale, + } + ) + return { + "steps": steps, + "count": len(steps), + "max_abs_diff": max(differences), + "max_rel_diff": max(relative), + "within_tolerance": max(differences) <= tolerance, + "details": detail, + } + + return { + "common_steps": common_steps, + "post_checkpoint": _statistics(post_checkpoint_steps), + "all_common_steps": _statistics(common_steps), + "tolerance": tolerance, + } + + +def _run_mode( + root: Path, + mode: str, + args: argparse.Namespace, +) -> Dict[str, Any]: + mode_dir = root / mode.replace("-", "_") + command = [ + sys.executable, + str(Path(__file__).with_name("run_in_place_demo.py")), + "--work-dir", + str(mode_dir), + "--recovery-mode", + mode, + "--steps", + str(args.steps), + "--log-every", + str(args.log_every), + "--checkpoint-interval", + str(args.checkpoint_interval), + "--fail-rank", + str(args.fail_rank), + "--fail-at-step", + str(args.fail_at_step), + "--dist-timeout-seconds", + str(args.dist_timeout_seconds), + "--recovery-timeout-seconds", + str(args.recovery_timeout_seconds), + "--job-timeout-seconds", + str(args.job_timeout_seconds), + ] + print("") + print(f"=== Running recovery mode: {mode} ===") + completed = subprocess.run(command, check=False) + if completed.returncode != 0: + raise RuntimeError( + f"recovery mode {mode} failed with " + f"exit code {completed.returncode}" + ) + summary = read_json(mode_dir / "metrics" / "summary.json") + if summary is None: + raise RuntimeError( + f"recovery mode {mode} did not write a summary" + ) + return summary + + +def _recovery_delta( + in_place: Dict[str, Any], + restart_all: Dict[str, Any], +) -> Dict[str, Any]: + in_place_seconds = in_place.get( + "failure_to_all_resumed_seconds" + ) + restart_all_seconds = restart_all.get( + "failure_to_all_resumed_seconds" + ) + saved = None + if ( + in_place_seconds is not None + and restart_all_seconds is not None + ): + saved = restart_all_seconds - in_place_seconds + return { + "in_place_failure_to_all_resumed_seconds": in_place_seconds, + "restart_all_failure_to_all_resumed_seconds": restart_all_seconds, + "failure_to_all_resumed_seconds_saved": saved, + "in_place_process_restarts": in_place.get( + "process_restarts" + ), + "restart_all_process_restarts": restart_all.get( + "process_restarts" + ), + "process_restarts_avoided": ( + int(restart_all.get("process_restarts", 0)) + - int(in_place.get("process_restarts", 0)) + ), + "in_place_total_wall_seconds": in_place.get( + "total_wall_seconds" + ), + "restart_all_total_wall_seconds": restart_all.get( + "total_wall_seconds" + ), + } + + +def _print_result(result: Dict[str, Any]) -> None: + loss = result["loss"]["post_checkpoint"] + recovery = result["recovery"] + + def _seconds(value: Any) -> str: + return f"{value:.3f}s" if value is not None else "n/a" + + print("") + print("Recovery comparison") + print(f" checkpoint step: {result['checkpoint_step']}") + print( + f" loss steps compared: {loss['count']} " + f"({loss['steps'][0]}..{loss['steps'][-1]})" + if loss["steps"] + else " loss steps compared: 0" + ) + print( + " max loss abs diff: " + f"{loss['max_abs_diff']}" + ) + print( + " max loss rel diff: " + f"{loss['max_rel_diff']}" + ) + print( + " loss within tolerance: " + f"{loss['within_tolerance']}" + ) + print( + " in-place failure-to-resume: " + f"{_seconds(recovery['in_place_failure_to_all_resumed_seconds'])}" + ) + print( + " restart-all failure-to-resume: " + f"{_seconds(recovery['restart_all_failure_to_all_resumed_seconds'])}" + ) + print( + " failure-to-resume saved: " + f"{_seconds(recovery['failure_to_all_resumed_seconds_saved'])}" + ) + print( + " process restarts avoided: " + f"{recovery['process_restarts_avoided']}" + ) + print( + " in-place total wall time: " + f"{_seconds(recovery['in_place_total_wall_seconds'])}" + ) + print( + " restart-all total wall time: " + f"{_seconds(recovery['restart_all_total_wall_seconds'])}" + ) + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--work-dir", type=str, default="") + parser.add_argument("--steps", type=int, default=40) + parser.add_argument("--log-every", type=int, default=5) + parser.add_argument("--checkpoint-interval", type=int, default=10) + parser.add_argument("--fail-rank", type=int, default=3) + parser.add_argument("--fail-at-step", type=int, default=23) + parser.add_argument("--dist-timeout-seconds", type=int, default=30) + parser.add_argument( + "--recovery-timeout-seconds", + type=float, + default=240.0, + ) + parser.add_argument( + "--job-timeout-seconds", + type=float, + default=1200.0, + ) + parser.add_argument( + "--loss-tolerance", + type=float, + default=1e-6, + ) + parser.add_argument( + "--run-order", + choices=("in-place-first", "restart-all-first"), + default="in-place-first", + ) + return parser + + +def main() -> None: + args = build_parser().parse_args() + if args.fail_rank < 0 or args.fail_rank >= 8: + raise SystemExit("--fail-rank must be in [0, 7]") + if args.steps <= 0: + raise SystemExit("--steps must be positive") + if args.checkpoint_interval <= 0: + raise SystemExit("--checkpoint-interval must be positive") + if args.fail_at_step < 0 or args.fail_at_step >= args.steps: + raise SystemExit( + "--fail-at-step must be nonnegative and less than --steps" + ) + + if args.work_dir: + root = Path(args.work_dir).resolve() + root.mkdir(parents=True, exist_ok=True) + if any(root.iterdir()): + raise SystemExit( + f"comparison work directory is not empty: {root}" + ) + else: + root = Path( + tempfile.mkdtemp(prefix="dlrover-recovery-compare-") + ).resolve() + + try: + if args.run_order == "in-place-first": + in_place = _run_mode(root, "in-place", args) + restart_all = _run_mode(root, "restart-all", args) + else: + restart_all = _run_mode(root, "restart-all", args) + in_place = _run_mode(root, "in-place", args) + checkpoint_step = in_place.get("recovery_checkpoint_step") + if checkpoint_step is None: + raise RuntimeError( + "in-place summary does not record the recovery checkpoint" + ) + checkpoint_step = int(checkpoint_step) + if int( + restart_all.get("recovery_checkpoint_step", -1) + ) != checkpoint_step: + raise RuntimeError( + "the two modes resumed from different checkpoints" + ) + loss = _compare_losses( + root / "in_place" / "metrics" / "loss_rank_2.jsonl", + root / "restart_all" / "metrics" / "loss_rank_2.jsonl", + checkpoint_step, + args.loss_tolerance, + ) + result = { + "work_dir": str(root), + "checkpoint_step": checkpoint_step, + "in_place_summary": in_place, + "restart_all_summary": restart_all, + "loss": loss, + "recovery": _recovery_delta( + in_place, + restart_all, + ), + } + atomic_write_json(root / "comparison.json", result) + _print_result(result) + print(f" comparison file: {root / 'comparison.json'}") + except (OSError, RuntimeError, ValueError) as error: + print(f"recovery comparison failed: {error}", file=sys.stderr) + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/examples/in_place_restart/tp2_pp2_dp2/train.py b/examples/in_place_restart/tp2_pp2_dp2/train.py new file mode 100755 index 000000000..925f87378 --- /dev/null +++ b/examples/in_place_restart/tp2_pp2_dp2/train.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python3 +# Copyright 2026 The DLRover Authors. All rights reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Compatibility entry point for the eight-rank hybrid MoE example.""" + +from hybrid_moe_train import main + + +if __name__ == "__main__": + main() diff --git a/examples/in_place_restart/tp2_pp2_dp2_2nodes/README.md b/examples/in_place_restart/tp2_pp2_dp2_2nodes/README.md new file mode 100644 index 000000000..fedde05c5 --- /dev/null +++ b/examples/in_place_restart/tp2_pp2_dp2_2nodes/README.md @@ -0,0 +1,203 @@ +# Two-Node TP2 x PP2 x DP2 MoE Recovery + +This is the two-node variant of the hybrid in-place restart demo: + +```text +World size = 8 +TP = 2 +PP = 2 +DP = 2 +EP = 1 +GPUs per node = 4 +``` + +The global rank layout is: + +```text +rank = DP * 4 + PP_stage * 2 + TP_rank +``` + +The nodes own these logical ranks: + +```text +node 0: ranks 0, 1, 2, 3 +node 1: ranks 4, 5, 6, 7 +``` + +The process groups are the same as the single-node hybrid demo: + +```text +TP: [0,1] [2,3] [4,5] [6,7] +PP: [0,2] [1,3] [4,6] [5,7] +DP: [0,4] [1,5] [2,6] [3,7] +``` + +The default injected failure is rank 5, which is +`TP1-PP0-DP1` on node 1. Its TP and PP peers are local to node 1, while its +DP peer rank 1 is on node 0. The node 1 agent starts rank 5's replacement; +all other ranks keep their original processes and PIDs. + +## Shared directory + +Both node agents and all workers must use the same absolute path for +`--work-dir`. The path must be on a filesystem shared by both nodes and must +support the POSIX file operations used by `torch.distributed.FileStore`. +Both containers must run with write permission for the same directory. The +node clocks should be synchronized because the recovery command contains a +wall-clock deadline. + +For example: + +```bash +export RUN_DIR=/nas/shared/dlrover/two-node-run-001 +mkdir -p "$RUN_DIR" +``` + +Do not use each node's local `/tmp` path. The shared directory contains the +rendezvous store, checkpoint shards, recovery command, and node status files: + +```text +agents/ + node_0.json + node_1.json + failures/ +checkpoints/ + checkpoint-00000020/ + latest.json +recovery/ + command.json +rendezvous/ + store +status/ + rank_0.json ... rank_7.json +``` + +The demonstration requires a fresh directory. Node 0 initializes it; node 1 +waits for node 0's `run.json` marker. + +## Launch + +Run these commands from the same source checkout on the two nodes. Start +node 0 first, then start node 1 while node 0 is still waiting for the initial +all-rank rendezvous. + +Node 0: + +```bash +bash examples/in_place_restart/tp2_pp2_dp2_2nodes/run_node0.sh \ + --work-dir "$RUN_DIR" \ + --fail-rank 5 \ + --fail-at-step 23 +``` + +Node 1: + +```bash +bash examples/in_place_restart/tp2_pp2_dp2_2nodes/run_node1.sh \ + --work-dir "$RUN_DIR" \ + --fail-rank 5 \ + --fail-at-step 23 +``` + +The shell scripts select the two logical node ranks. The Python agent +launches four local workers with: + +```text +WORLD_SIZE=8 +LOCAL_WORLD_SIZE=4 +LOCAL_RANK=0,1,2,3 +NODE_RANK=0 or 1 +``` + +The training implementation is shared with +`../tp2_pp2_dp2/hybrid_moe_train.py`; this directory only supplies the +two-node launch and control layer. The `train.py` file is a compatibility +entry point for launching the same worker under a DLRover-compatible runner. + +For a real cross-node NCCL connection, the cluster must allow the required +NCCL/NVLink/TCP ports and must select the correct network interface. Pass +the interface explicitly when auto-detection is not appropriate: + +```bash +bash run_node0.sh \ + --work-dir "$RUN_DIR" \ + --nccl-socket-ifname eth0 +``` + +For InfiniBand, also pass `--nccl-ib-hca`, for example `mlx5_0:1`. +Use the network settings required by the cluster; the two-node demo does not +guess them. + +## What happens during recovery + +1. Both node agents launch their four workers and publish initial PIDs in + `agents/node_0.json` and `agents/node_1.json`. +2. The training script creates the default, TP, PP, and DP `nccl2` groups and + writes a checkpoint manifest. +3. Rank 5 exits at step 23. Node 1 records the failure in + `agents/failures/`. +4. Node 0 detects the failure, waits for a committed checkpoint, and writes + `recovery/command.json`. The command contains one ordered handle slot per + logical rank; the replacement's handle is placed at the original rank-5 + slot. +5. Node 1 starts only rank 5's replacement process. Node 0 starts no rank-5 + process because rank 5 belongs to node 1. +6. Every healthy worker keeps its process, CUDA context, model parameters, + optimizer, and dataset state. Its recovery watcher observes the command + and aborts local `nccl2` operations that are blocked on the failed rank. +7. The replacement enters the same recovery path: it creates group shells, + exchanges reconfigure handles, and reconfigures the default, TP, PP, and + DP groups. +8. All ranks validate their local groups, restore their own checkpoint shard + at step 20, rebuild only the DDP wrapper, and resume training to step 40. +9. Node 0 verifies all eight final statuses: ranks other than 5 retain their + initial PIDs, rank 5 has a replacement PID, and every rank reports the + same recovery count and final step. + +The healthy ranks do not recreate their model, optimizer, CUDA context, or +Python process. Reconfiguration changes the communicator state and rebuilds +the DDP reducer; it is not a full training-process restart. + +Expected supervisor output includes: + +```text +[node-agent 1] FAILED rank 5/TP1-PP0-DP1: pid=..., exit_code=17 +[node-agent 0] requested recovery epoch 1 for rank 5/TP1-PP0-DP1 +[node-agent 0] healthy ranks retain their initial processes: ... +[node-agent 1] REPLACED rank 5/TP1-PP0-DP1: old_pid=..., new_pid=... +[node-agent 0] RECOVERED rank 0/TP0-PP0-DP0: pid=..., ... +... +Two-node in-place restart summary +``` + +## Docker + +After rebuilding the image from this checkout, the existing Dockerfile is +sufficient because it copies all of `examples/in_place_restart`. On each node, +run a container with four GPUs and mount the shared directory at the same +path in both containers. For example: + +First run the four-rank local image check on each node: + +```bash +python examples/in_place_restart/verify_image.py --world-size 4 +``` + +This checks the PyTorch/NCCL versions, `nccl2` reconfigure APIs, four visible +GPUs, and a local four-rank reconfigure. It does not prove that NCCL traffic +or the shared filesystem works between the two hosts; the two-node run below +does that. + +```bash +docker run --rm -it --network=host \ + --gpus '"device=0,1,2,3"' \ + --shm-size=16g \ + -v "$RUN_DIR:$RUN_DIR" \ + dlrover-inplace-moe:26.09 \ + bash examples/in_place_restart/tp2_pp2_dp2_2nodes/run_node0.sh \ + --work-dir "$RUN_DIR" \ + --nccl-socket-ifname eth0 +``` + +Use `run_node1.sh` in the container on node 1. Both containers must see the +same `RUN_DIR`; mount the shared filesystem, not a container-local path. diff --git a/examples/in_place_restart/tp2_pp2_dp2_2nodes/run_node0.sh b/examples/in_place_restart/tp2_pp2_dp2_2nodes/run_node0.sh new file mode 100755 index 000000000..d477777e0 --- /dev/null +++ b/examples/in_place_restart/tp2_pp2_dp2_2nodes/run_node0.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +# Copyright 2026 The DLRover Authors. All rights reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PYTHON_BIN="${PYTHON:-}" +if [[ -z "${PYTHON_BIN}" ]]; then + if command -v python3 >/dev/null 2>&1; then + PYTHON_BIN="python3" + else + PYTHON_BIN="python" + fi +fi +exec "${PYTHON_BIN}" "${SCRIPT_DIR}/run_node_agent.py" \ + --node-rank 0 "$@" diff --git a/examples/in_place_restart/tp2_pp2_dp2_2nodes/run_node1.sh b/examples/in_place_restart/tp2_pp2_dp2_2nodes/run_node1.sh new file mode 100755 index 000000000..ee9ccbc2a --- /dev/null +++ b/examples/in_place_restart/tp2_pp2_dp2_2nodes/run_node1.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +# Copyright 2026 The DLRover Authors. All rights reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PYTHON_BIN="${PYTHON:-}" +if [[ -z "${PYTHON_BIN}" ]]; then + if command -v python3 >/dev/null 2>&1; then + PYTHON_BIN="python3" + else + PYTHON_BIN="python" + fi +fi +exec "${PYTHON_BIN}" "${SCRIPT_DIR}/run_node_agent.py" \ + --node-rank 1 "$@" diff --git a/examples/in_place_restart/tp2_pp2_dp2_2nodes/run_node_agent.py b/examples/in_place_restart/tp2_pp2_dp2_2nodes/run_node_agent.py new file mode 100755 index 000000000..7ea5435c6 --- /dev/null +++ b/examples/in_place_restart/tp2_pp2_dp2_2nodes/run_node_agent.py @@ -0,0 +1,835 @@ +#!/usr/bin/env python3 +# Copyright 2026 The DLRover Authors. All rights reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Run one node of the two-node TP2 x PP2 x DP2 demo. + +Start this script once on each node with the same shared ``--work-dir``. +Node 0 coordinates recovery. The node that owns the failed global rank +starts the replacement worker; healthy workers stay in their existing +processes. +""" + +import argparse +import os +import socket +import subprocess +import sys +import time +from pathlib import Path +from typing import ( + Any, + Callable, + Dict, + List, + Optional, + Set, + Tuple, +) + +EXAMPLE_ROOT = Path(__file__).resolve().parent.parent +if str(EXAMPLE_ROOT) not in sys.path: + sys.path.insert(0, str(EXAMPLE_ROOT)) + +from recovery_protocol import ( + RecoveryCommand, + atomic_write_json, + fresh_reconfigure_uuid, + load_recovery_command, + read_json, +) + + +WORLD_SIZE = 8 +NODE_COUNT = 2 +GPUS_PER_NODE = 4 +TP_SIZE = 2 +PP_SIZE = 2 +DP_SIZE = 2 +TRAIN_SCRIPT = ( + Path(__file__).resolve().parent.parent + / "tp2_pp2_dp2" + / "hybrid_moe_train.py" +) + + +def _coordinates(rank: int) -> str: + tp_rank = rank % TP_SIZE + pp_rank = (rank // TP_SIZE) % PP_SIZE + dp_rank = rank // (TP_SIZE * PP_SIZE) + return f"TP{tp_rank}-PP{pp_rank}-DP{dp_rank}" + + +def _node_ranks(node_rank: int) -> Tuple[int, ...]: + start = node_rank * GPUS_PER_NODE + return tuple(range(start, start + GPUS_PER_NODE)) + + +def _wait_for_json( + path: Path, + timeout_seconds: float, + description: str, +) -> Dict[str, Any]: + deadline = time.monotonic() + timeout_seconds + while time.monotonic() < deadline: + value = read_json(path) + if value is not None: + return value + time.sleep(0.2) + raise TimeoutError( + f"timed out waiting for {description}: {path}" + ) + + +def _wait_for_json_matching( + path: Path, + timeout_seconds: float, + description: str, + predicate: Callable[[Dict[str, Any]], bool], +) -> Dict[str, Any]: + deadline = time.monotonic() + timeout_seconds + while time.monotonic() < deadline: + value = read_json(path) + if value is not None and predicate(value): + return value + time.sleep(0.2) + raise TimeoutError( + f"timed out waiting for {description}: {path}" + ) + + +def _read_all_failures( + failure_dir: Path, +) -> List[Dict[str, Any]]: + failures: List[Dict[str, Any]] = [] + for path in sorted(failure_dir.glob("rank_*.json")): + value = read_json(path) + if value is not None: + failures.append(value) + return failures + + +class NodeAgent: + """Supervise the four local ranks and participate in recovery.""" + + def __init__( + self, + args: argparse.Namespace, + work_dir: Path, + ) -> None: + self.args = args + self.node_rank = args.node_rank + self.work_dir = work_dir + self.ranks = _node_ranks(self.node_rank) + self.state_path = ( + self.work_dir + / "agents" + / f"node_{self.node_rank}.json" + ) + self.failure_dir = self.work_dir / "agents" / "failures" + self.command_path = ( + self.work_dir / "recovery" / "command.json" + ) + self.processes: Dict[int, subprocess.Popen] = {} + self.initial_pids: Dict[int, int] = {} + self.replacement_pids: Dict[int, int] = {} + self.reported_failures: Set[Tuple[int, int]] = set() + self.reported_recoveries: Set[int] = set() + self.peer_initial_pids: Dict[int, int] = {} + self.recovery_started = False + self.last_command_epoch = 0 + + def _log(self, message: str) -> None: + print( + f"[node-agent {self.node_rank}] {message}", + flush=True, + ) + + def _write_state( + self, + phase: str, + **extra: Any, + ) -> None: + state: Dict[str, Any] = { + "node_rank": self.node_rank, + "node_count": NODE_COUNT, + "hostname": socket.gethostname(), + "agent_pid": os.getpid(), + "work_dir": str(self.work_dir), + "phase": phase, + "local_ranks": list(self.ranks), + "initial_pids": { + str(rank): pid + for rank, pid in sorted(self.initial_pids.items()) + }, + "replacement_pids": { + str(rank): pid + for rank, pid in sorted( + self.replacement_pids.items() + ) + }, + "timestamp_unix_ms": int(time.time() * 1000), + } + state.update(extra) + atomic_write_json(self.state_path, state) + + def _initialize_run(self) -> None: + if self.node_rank != 0: + raise RuntimeError( + "only node 0 may initialize a run directory" + ) + self.work_dir.mkdir(parents=True, exist_ok=True) + if any(self.work_dir.iterdir()): + raise ValueError( + "work directory is not empty: " + f"{self.work_dir}; use a new shared directory" + ) + metadata = { + "format": "dlrover-in-place-two-node", + "work_dir": str(self.work_dir), + "world_size": WORLD_SIZE, + "node_count": NODE_COUNT, + "gpus_per_node": GPUS_PER_NODE, + "tp_size": TP_SIZE, + "pp_size": PP_SIZE, + "dp_size": DP_SIZE, + "ep_size": 1, + "created_unix_ms": int(time.time() * 1000), + } + atomic_write_json(self.work_dir / "run.json", metadata) + self.failure_dir.mkdir(parents=True, exist_ok=True) + (self.work_dir / "recovery").mkdir( + parents=True, + exist_ok=True, + ) + + def _wait_for_run(self) -> None: + metadata = _wait_for_json( + self.work_dir / "run.json", + self.args.job_timeout_seconds, + "node 0 to initialize the shared work directory", + ) + if metadata.get("format") != "dlrover-in-place-two-node": + raise ValueError("unexpected run format") + if metadata.get("work_dir") != str(self.work_dir): + raise ValueError( + "both nodes must use the same absolute --work-dir" + ) + if int(metadata.get("world_size", 0)) != WORLD_SIZE: + raise ValueError("run world size does not match") + if int(metadata.get("node_count", 0)) != NODE_COUNT: + raise ValueError("run node count does not match") + if int(metadata.get("gpus_per_node", 0)) != GPUS_PER_NODE: + raise ValueError("run GPUs per node does not match") + + def _launch_worker( + self, + rank: int, + replacement: bool, + ) -> subprocess.Popen: + local_rank = rank % GPUS_PER_NODE + env = os.environ.copy() + env.update( + { + "RANK": str(rank), + "LOCAL_RANK": str(local_rank), + "WORLD_SIZE": str(WORLD_SIZE), + "LOCAL_WORLD_SIZE": str(GPUS_PER_NODE), + "NODE_RANK": str(self.node_rank), + "NODE_WORLD_SIZE": str(NODE_COUNT), + "PYTHONUNBUFFERED": "1", + "OMP_NUM_THREADS": env.get("OMP_NUM_THREADS", "1"), + "TORCH_NCCL_ASYNC_ERROR_HANDLING": "0", + "TORCH_NCCL_BLOCKING_WAIT": "0", + "NCCL_DEBUG": env.get("NCCL_DEBUG", "WARN"), + } + ) + # The demo uses FileStore rather than TCPStore. Do not inherit a + # single-node MASTER_ADDR/MASTER_PORT from the caller. + env.pop("MASTER_ADDR", None) + env.pop("MASTER_PORT", None) + env.pop("NCCL_ASYNC_ERROR_HANDLING", None) + if self.args.nccl_socket_ifname: + env["NCCL_SOCKET_IFNAME"] = ( + self.args.nccl_socket_ifname + ) + if self.args.nccl_ib_hca: + env["NCCL_IB_HCA"] = self.args.nccl_ib_hca + + command = [ + sys.executable, + str(TRAIN_SCRIPT), + "--work-dir", + str(self.work_dir), + "--steps", + str(self.args.steps), + "--log-every", + str(self.args.log_every), + "--checkpoint-interval", + str(self.args.checkpoint_interval), + "--recovery-command", + str(self.command_path), + "--dist-timeout-seconds", + str(self.args.dist_timeout_seconds), + "--recovery-timeout-seconds", + str(self.args.recovery_timeout_seconds), + ] + if replacement: + command.append("--replacement") + else: + command.extend( + [ + "--fail-rank", + str(self.args.fail_rank), + "--fail-at-step", + str(self.args.fail_at_step), + ] + ) + return subprocess.Popen( + command, + cwd=str(TRAIN_SCRIPT.parent), + env=env, + ) + + def _launch_initial_workers(self) -> None: + self._write_state("starting") + for rank in self.ranks: + process = self._launch_worker(rank, replacement=False) + self.processes[rank] = process + self.initial_pids[rank] = process.pid + self._write_state("running") + self._log( + "initial processes: " + + ", ".join( + f"rank {rank}/{_coordinates(rank)}=pid " + f"{self.initial_pids[rank]}" + for rank in self.ranks + ) + ) + + def _report_failure( + self, + rank: int, + pid: int, + returncode: int, + ) -> None: + failure_key = (rank, pid) + if failure_key in self.reported_failures: + return + self.reported_failures.add(failure_key) + status = read_json( + self.work_dir / "status" / f"rank_{rank}.json" + ) + is_replacement = ( + pid != self.initial_pids.get(rank) + ) + record = { + "rank": rank, + "pid": pid, + "old_pid": self.initial_pids.get(rank), + "returncode": returncode, + "node_rank": self.node_rank, + "is_replacement": is_replacement, + "status_phase": ( + status.get("phase") if status is not None else None + ), + "timestamp_unix_ms": int(time.time() * 1000), + } + atomic_write_json( + self.failure_dir / f"rank_{rank}_{pid}.json", + record, + ) + self._log( + f"FAILED rank {rank}/{_coordinates(rank)}: " + f"pid={pid}, exit_code={returncode}" + ) + + def _poll_local_failures(self) -> None: + for rank, process in self.processes.items(): + returncode = process.poll() + if returncode is None: + continue + status = read_json( + self.work_dir / "status" / f"rank_{rank}.json" + ) + if ( + returncode == 0 + and status is not None + and status.get("phase") == "finished" + ): + continue + self._report_failure( + rank, + process.pid, + returncode, + ) + + def _wait_for_peer_initial_pids(self) -> None: + for peer in range(NODE_COUNT): + state = _wait_for_json_matching( + self.work_dir + / "agents" + / f"node_{peer}.json", + self.args.job_timeout_seconds, + f"node {peer} agent to start", + lambda value, peer=peer: all( + str(rank) in value.get("initial_pids", {}) + for rank in _node_ranks(peer) + ), + ) + if int(state.get("node_rank", -1)) != peer: + raise RuntimeError( + f"node {peer} agent metadata is invalid" + ) + initial_pids = state.get("initial_pids", {}) + for rank in _node_ranks(peer): + if str(rank) not in initial_pids: + raise RuntimeError( + f"node {peer} did not report rank {rank}" + ) + self.peer_initial_pids[rank] = int( + initial_pids[str(rank)] + ) + + def _all_initial_pids(self) -> Dict[int, int]: + result = dict(self.initial_pids) + result.update(self.peer_initial_pids) + if set(result) != set(range(WORLD_SIZE)): + raise RuntimeError( + "initial PID map does not cover the global world" + ) + return result + + def _coordinate_recovery( + self, + failure: Dict[str, Any], + ) -> None: + if self.node_rank != 0: + raise RuntimeError("only node 0 coordinates recovery") + failed_rank = int(failure["rank"]) + if failed_rank != self.args.fail_rank: + raise RuntimeError( + f"unexpected failed rank {failed_rank}; " + f"expected {self.args.fail_rank}" + ) + if failure.get("is_replacement"): + raise RuntimeError("the replacement rank failed") + + latest = _wait_for_json( + self.work_dir / "checkpoints" / "latest.json", + self.args.recovery_timeout_seconds, + "a committed checkpoint", + ) + now = time.time() + command = RecoveryCommand( + recovery_epoch=1, + attempt_id=( + f"two-node-{int(now * 1000)}-{failed_rank}" + ), + failed_rank=failed_rank, + checkpoint_path=str(latest["checkpoint_path"]), + checkpoint_step=int(latest["checkpoint_step"]), + reconfigure_uuid=fresh_reconfigure_uuid(), + world_size=WORLD_SIZE, + deadline_unix_ms=int( + ( + now + + self.args.recovery_timeout_seconds + ) + * 1000 + ), + ) + self.recovery_started = True + atomic_write_json( + self.command_path, + command.to_dict(), + ) + self._write_state( + "recovering", + recovery_epoch=command.recovery_epoch, + failed_rank=failed_rank, + ) + initial_pids = self._all_initial_pids() + healthy = { + rank: pid + for rank, pid in initial_pids.items() + if rank != failed_rank + } + self._log( + f"requested recovery epoch {command.recovery_epoch} " + f"for rank {failed_rank}/{_coordinates(failed_rank)}" + ) + self._log( + "healthy ranks retain their initial processes: " + + ", ".join( + f"rank {rank}=pid {pid}" + for rank, pid in sorted(healthy.items()) + ) + ) + + def _unexpected_failure( + self, + failures: List[Dict[str, Any]], + ) -> Optional[Dict[str, Any]]: + initial_pids = self._all_initial_pids() + for failure in failures: + rank = int(failure["rank"]) + pid = int(failure["pid"]) + is_expected_old_target = ( + rank == self.args.fail_rank + and pid == initial_pids[rank] + and not failure.get("is_replacement", False) + ) + if not is_expected_old_target: + return failure + return None + + def _maybe_apply_recovery_command(self) -> None: + try: + command = load_recovery_command(self.command_path) + except (TypeError, ValueError): + return + if ( + command is None + or command.recovery_epoch <= self.last_command_epoch + ): + return + if command.world_size != WORLD_SIZE: + raise ValueError("recovery world size does not match") + if command.failed_rank not in range(WORLD_SIZE): + raise ValueError("recovery failure rank is invalid") + + self.last_command_epoch = command.recovery_epoch + self.recovery_started = True + self._write_state( + "recovering", + recovery_epoch=command.recovery_epoch, + failed_rank=command.failed_rank, + ) + self._log( + f"observed recovery epoch {command.recovery_epoch}" + ) + + failed_rank = command.failed_rank + if failed_rank not in self.ranks: + self._log( + f"rank {failed_rank} is owned by node " + f"{failed_rank // GPUS_PER_NODE}; this node " + "keeps all local processes running" + ) + return + + old_process = self.processes.get(failed_rank) + if old_process is None: + raise RuntimeError( + f"failed rank {failed_rank} is missing locally" + ) + if old_process.poll() is None: + raise RuntimeError( + f"refusing to replace live rank {failed_rank}" + ) + old_pid = old_process.pid + replacement = self._launch_worker( + failed_rank, + replacement=True, + ) + self.processes[failed_rank] = replacement + self.replacement_pids[failed_rank] = replacement.pid + self._write_state( + "recovery_started", + recovery_epoch=command.recovery_epoch, + failed_rank=failed_rank, + ) + self._log( + f"REPLACED rank {failed_rank}/" + f"{_coordinates(failed_rank)}: " + f"old_pid={old_pid}, new_pid={replacement.pid}" + ) + + def _report_recoveries(self) -> None: + for rank in self.ranks: + if rank in self.reported_recoveries: + continue + status = read_json( + self.work_dir / "status" / f"rank_{rank}.json" + ) + if ( + status is not None + and status.get("phase") == "running" + and int(status.get("recovery_count", 0)) > 0 + ): + self._log( + f"RECOVERED rank {rank}/" + f"{_coordinates(rank)}: pid={status['pid']}, " + f"epoch={status['recovery_epoch']}, " + f"step={status['global_step']}" + ) + self.reported_recoveries.add(rank) + + def _local_processes_finished(self) -> bool: + for rank, process in self.processes.items(): + if process.poll() is None: + return False + status = read_json( + self.work_dir / "status" / f"rank_{rank}.json" + ) + if status is None or status.get("phase") != "finished": + return False + return True + + def _wait_for_peer_finished(self) -> None: + if self.node_rank != 0: + return + for peer in range(NODE_COUNT): + _wait_for_json_matching( + self.work_dir + / "agents" + / f"node_{peer}.json", + self.args.job_timeout_seconds, + f"node {peer} to finish", + lambda value: value.get("phase") == "finished", + ) + + def _validate_global(self) -> None: + initial_pids = self._all_initial_pids() + states: Dict[int, Dict[str, Any]] = {} + for node_rank in range(NODE_COUNT): + state = read_json( + self.work_dir + / "agents" + / f"node_{node_rank}.json" + ) + if state is None: + raise RuntimeError( + f"missing state for node {node_rank}" + ) + states[node_rank] = state + + for rank in range(WORLD_SIZE): + status = read_json( + self.work_dir / "status" / f"rank_{rank}.json" + ) + if status is None or status.get("phase") != "finished": + raise RuntimeError(f"rank {rank} did not finish") + if int(status.get("global_step", -1)) != self.args.steps: + raise RuntimeError( + f"rank {rank} did not reach the final step" + ) + if int(status.get("recovery_count", 0)) < 1: + raise RuntimeError( + f"rank {rank} did not recover" + ) + final_pid = int(status["pid"]) + if rank == self.args.fail_rank: + if final_pid == initial_pids[rank]: + raise RuntimeError( + f"failed rank {rank} was not replaced" + ) + owner = rank // GPUS_PER_NODE + owner_state = states[owner] + replacement_pids = owner_state.get( + "replacement_pids", + {}, + ) + if ( + str(rank) in replacement_pids + and int(replacement_pids[str(rank)]) + != final_pid + ): + raise RuntimeError( + f"rank {rank} replacement PID mismatch" + ) + elif final_pid != initial_pids[rank]: + raise RuntimeError( + f"healthy rank {rank} changed PID" + ) + + print("") + print("Two-node in-place restart summary") + print(" topology: TP2 x PP2 x DP2, EP=1") + print(" nodes: 2, GPUs per node: 4") + print(f" work_dir: {self.work_dir}") + for rank in range(WORLD_SIZE): + status = read_json( + self.work_dir / "status" / f"rank_{rank}.json" + ) + assert status is not None + final_pid = int(status["pid"]) + initial_pid = initial_pids[rank] + if final_pid == initial_pid: + result = f"CONTINUED with pid {final_pid}" + else: + result = ( + f"REPLACED {initial_pid} -> {final_pid}" + ) + print( + f" rank {rank}/{_coordinates(rank)} " + f"on node {rank // GPUS_PER_NODE}: {result}, " + f"step={status.get('global_step')}, " + f"recoveries={status.get('recovery_count')}" + ) + + def run(self) -> None: + if self.node_rank == 0: + self._initialize_run() + else: + self._wait_for_run() + + try: + self._launch_initial_workers() + self._wait_for_peer_initial_pids() + deadline = ( + time.monotonic() + self.args.job_timeout_seconds + ) + while True: + self._poll_local_failures() + failures = _read_all_failures(self.failure_dir) + + if ( + self.node_rank == 0 + and not self.recovery_started + ): + target_failures = [ + failure + for failure in failures + if int(failure["rank"]) + == self.args.fail_rank + and not failure.get( + "is_replacement", + False, + ) + ] + if target_failures: + self._coordinate_recovery( + target_failures[0] + ) + + unexpected = self._unexpected_failure(failures) + if unexpected is not None: + raise RuntimeError( + "unexpected rank failure: " + f"{unexpected}" + ) + + self._maybe_apply_recovery_command() + self._report_recoveries() + + if self._local_processes_finished(): + break + if time.monotonic() >= deadline: + raise TimeoutError( + "two-node in-place recovery demo timed out" + ) + time.sleep(0.2) + + exit_codes = { + str(rank): process.returncode + for rank, process in sorted( + self.processes.items() + ) + } + if any(code != 0 for code in exit_codes.values()): + raise RuntimeError( + f"local workers exited unsuccessfully: " + f"{exit_codes}" + ) + self._write_state( + "finished", + exit_codes=exit_codes, + ) + self._log("local workers finished") + self._wait_for_peer_finished() + if self.node_rank == 0: + self._validate_global() + self._write_state("complete") + finally: + self._cleanup_workers() + + def _cleanup_workers(self) -> None: + for process in self.processes.values(): + if process.poll() is None: + process.terminate() + for process in self.processes.values(): + try: + process.wait(timeout=10) + except subprocess.TimeoutExpired: + process.kill() + process.wait(timeout=10) + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--node-rank", + type=int, + required=True, + choices=range(NODE_COUNT), + ) + parser.add_argument("--work-dir", type=str, required=True) + parser.add_argument("--steps", type=int, default=40) + parser.add_argument("--log-every", type=int, default=5) + parser.add_argument("--checkpoint-interval", type=int, default=10) + parser.add_argument("--fail-rank", type=int, default=5) + parser.add_argument("--fail-at-step", type=int, default=23) + parser.add_argument("--dist-timeout-seconds", type=int, default=30) + parser.add_argument( + "--recovery-timeout-seconds", + type=float, + default=240.0, + ) + parser.add_argument( + "--job-timeout-seconds", + type=float, + default=1200.0, + ) + parser.add_argument( + "--nccl-socket-ifname", + type=str, + default="", + help="optional NCCL_SOCKET_IFNAME for cross-node NCCL", + ) + parser.add_argument( + "--nccl-ib-hca", + type=str, + default="", + help="optional NCCL_IB_HCA for InfiniBand", + ) + return parser + + +def main() -> None: + args = build_parser().parse_args() + if args.fail_rank not in range(WORLD_SIZE): + raise SystemExit("--fail-rank must be in [0, 7]") + if args.fail_at_step < 0 or args.fail_at_step >= args.steps: + raise SystemExit( + "--fail-at-step must be nonnegative and less than --steps" + ) + work_dir = Path(args.work_dir).resolve() + agent = NodeAgent(args, work_dir) + try: + agent.run() + except ( + OSError, + RuntimeError, + TimeoutError, + ValueError, + ) as error: + print( + f"two-node demo failed on node {args.node_rank}: {error}", + file=sys.stderr, + ) + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/examples/in_place_restart/tp2_pp2_dp2_2nodes/train.py b/examples/in_place_restart/tp2_pp2_dp2_2nodes/train.py new file mode 100755 index 000000000..4a2f7c05e --- /dev/null +++ b/examples/in_place_restart/tp2_pp2_dp2_2nodes/train.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python3 +# Copyright 2026 The DLRover Authors. All rights reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Compatibility entry point for the two-node hybrid MoE example.""" + +import sys +from pathlib import Path + +SHARED_EXAMPLE = ( + Path(__file__).resolve().parent.parent / "tp2_pp2_dp2" +) +if str(SHARED_EXAMPLE) not in sys.path: + sys.path.insert(0, str(SHARED_EXAMPLE)) + +from hybrid_moe_train import main # noqa: E402 + + +if __name__ == "__main__": + main() diff --git a/examples/in_place_restart/verify_image.py b/examples/in_place_restart/verify_image.py new file mode 100755 index 000000000..0a4b5bf07 --- /dev/null +++ b/examples/in_place_restart/verify_image.py @@ -0,0 +1,266 @@ +#!/usr/bin/env python3 +# Copyright 2026 The DLRover Authors. All rights reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Verify that a container can run the in-place restart example.""" + +import argparse +import inspect +import os +import secrets +import shutil +import subprocess +import tempfile +from datetime import timedelta +from pathlib import Path +from typing import Tuple + +import torch +import torch.distributed as dist + + +MIN_TORCH = (2, 14) +MIN_NCCL = (2, 28, 0) +MIN_SHM_GB = 4.0 + + +def _version_tuple(value: str) -> Tuple[int, int]: + parts = value.split("+")[0].split(".") + return int(parts[0]), int(parts[1]) + + +def _check(label: str, condition: bool, detail: str = "") -> None: + suffix = f" ({detail})" if detail else "" + print(f"[{'PASS' if condition else 'FAIL'}] {label}{suffix}") + if not condition: + raise RuntimeError(f"{label} check failed{suffix}") + + +def _check_local_environment(world_size: int) -> None: + torch_version = _version_tuple(torch.__version__) + _check( + "PyTorch >= 2.14", + torch_version >= MIN_TORCH, + torch.__version__, + ) + _check( + "CUDA is available", + torch.cuda.is_available(), + f"build CUDA {torch.version.cuda}", + ) + _check( + f"at least {world_size} CUDA devices", + torch.cuda.device_count() >= world_size, + f"found {torch.cuda.device_count()}", + ) + + nccl_version = tuple(torch.cuda.nccl.version()) + _check( + "NCCL >= 2.28 for nccl2 reconfigure", + nccl_version >= MIN_NCCL, + ".".join(str(value) for value in nccl_version), + ) + _check( + "nccl2 backend is available", + dist.is_backend_available("nccl2"), + ) + + required_apis = ( + "_supports_reconfigure", + "_get_reconfigure_handle", + "_reconfigure", + "get_backend_impl", + ) + missing = [name for name in required_apis if not hasattr(dist, name)] + _check( + "process-group reconfigure APIs exist", + not missing, + f"missing={missing}" if missing else "", + ) + init_parameters = inspect.signature( + dist.init_process_group + ).parameters + _check( + "init_process_group accepts enable_reconfigure", + "enable_reconfigure" in init_parameters, + ) + + shared_memory = os.statvfs("/dev/shm") + shared_memory_gb = ( + shared_memory.f_bavail * shared_memory.f_frsize + ) / (1024**3) + minimum_shm_gb = max(MIN_SHM_GB, float(world_size)) + _check( + f"/dev/shm >= {minimum_shm_gb:g} GiB", + shared_memory_gb >= minimum_shm_gb, + f"{shared_memory_gb:.2f} GiB", + ) + + dlrover_run = shutil.which("dlrover-run") + _check("dlrover-run is installed", dlrover_run is not None) + dlrover_help = subprocess.run( + [dlrover_run, "--help"], + check=False, + capture_output=True, + text=True, + timeout=30, + ) + _check( + "dlrover-run starts successfully", + dlrover_help.returncode == 0, + dlrover_help.stderr.strip()[:300], + ) + try: + import dlrover # noqa: F401 + except ImportError as error: + raise RuntimeError("the dlrover package is not importable") from error + print("[PASS] dlrover package imports") + + example_dir = Path(__file__).resolve().parent + required_files = ( + "dp4/moe_train.py", + "dp4/run_in_place_demo.py", + "recovery_protocol.py", + "tp2_pp2_dp2/hybrid_moe_train.py", + "tp2_pp2_dp2/run_in_place_demo.py", + "tp2_pp2_dp2/run_recovery_comparison.py", + "tp2_pp2_dp2_2nodes/run_node_agent.py", + "tp2_pp2_dp2_2nodes/run_node0.sh", + "tp2_pp2_dp2_2nodes/run_node1.sh", + ) + missing_files = [ + name + for name in required_files + if not (example_dir / name).is_file() + ] + _check( + "example worker files exist", + not missing_files, + f"missing={missing_files}" if missing_files else "", + ) + + print("") + for rank in range(world_size): + print( + f"GPU {rank}: {torch.cuda.get_device_name(rank)}" + ) + + +def _distributed_worker( + rank: int, + store_path: str, + world_size: int, +) -> None: + torch.cuda.set_device(rank) + store = dist.FileStore(store_path, world_size) + dist.init_process_group( + backend="nccl2", + store=store, + rank=rank, + world_size=world_size, + timeout=timedelta(seconds=60), + enable_reconfigure=True, + ) + + try: + default_group = dist.distributed_c10d._get_default_group() + _check( + f"rank {rank}: nccl2 supports reconfigure", + bool(dist._supports_reconfigure(default_group)), + ) + + prefix = dist.PrefixStore("verify-image", store) + prefix.set( + f"handle/{rank}", + dist._get_reconfigure_handle(default_group), + ) + handle_keys = [ + f"handle/{handle_rank}" + for handle_rank in range(world_size) + ] + prefix.wait(handle_keys, timedelta(seconds=60)) + handles = [ + prefix.get(key).decode("utf-8") for key in handle_keys + ] + + uuid_key = "verify-image/uuid" + if rank == 0: + store.set(uuid_key, str(secrets.randbits(62) + 1)) + store.wait([uuid_key], timedelta(seconds=60)) + reconfigure_uuid = int(store.get(uuid_key).decode("utf-8")) + + work = dist._reconfigure( + uuid=reconfigure_uuid, + handles=handles, + group=default_group, + timeout=timedelta(seconds=60), + ) + work.wait() + + tensor = torch.ones(1, device=torch.device("cuda", rank)) + dist.all_reduce(tensor) + _check( + f"rank {rank}: reconfigured all-reduce", + tensor.item() == world_size, + f"sum={tensor.item()}", + ) + finally: + if dist.is_initialized(): + dist.destroy_process_group() + + +def _check_distributed(world_size: int) -> None: + with tempfile.TemporaryDirectory( + prefix="dlrover-image-check-" + ) as directory: + store_path = str(Path(directory) / "store") + try: + torch.multiprocessing.spawn( + _distributed_worker, + args=(store_path, world_size), + nprocs=world_size, + join=True, + ) + except Exception as error: + raise RuntimeError( + "four-rank nccl2 reconfigure test failed" + ) from error + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--world-size", type=int, default=4) + parser.add_argument( + "--skip-distributed", + action="store_true", + help="only check packages, APIs, GPUs, and container limits", + ) + return parser + + +def main() -> None: + args = build_parser().parse_args() + if args.world_size not in (4, 8): + raise SystemExit("--world-size must be 4 or 8") + + print("Container preflight checks") + _check_local_environment(args.world_size) + if not args.skip_distributed: + print("") + print("Four-rank nccl2 reconfigure check") + _check_distributed(args.world_size) + print("") + print("IMAGE PREFLIGHT PASSED") + + +if __name__ == "__main__": + main()