Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
51 changes: 29 additions & 22 deletions dlrover/python/master/args.py
Original file line number Diff line number Diff line change
Expand Up @@ -151,24 +151,28 @@ def _build_master_args_parser():
help='Node group sizes, e.g. --group-affinity="{0: 10, 1: 15}" '
"means group 0 has 10 pods and group 1 has 15 pods. The worker "
"replicas in the ElasticJob CRD must equal the sum of all group "
"sizes, otherwise the master fails to start.",
"sizes, otherwise the master fails to start. Placement-only: the "
"sizes are NOT coupled to the Megatron parallel sizes, and no "
"EP/PP/DP layout validation is applied to them when "
"--enable-topology-rerank is on (the EP-PP column packing runs "
"only on the legacy ep_pp_dp-without-rerank path).",
)
parser.add_argument(
"--node-group-strategy",
"--node_group_strategy",
default=None,
choices=["ep_dp_pp", "ep_pp_dp"],
help="Strategy used to map worker nodes (by creation order) into "
"node groups (== physical segments). 'ep_dp_pp' keeps the existing "
"behavior: groups occupy contiguous rank ranges. 'ep_pp_dp' packs "
"EP-group slots of a Megatron MoE pipeline stage into ep-pp columns "
"across groups so EP groups and PP stay intra-segment while only "
"the dense DP collective crosses segments. Required when "
"--group-affinity or --soft-group-affinity is configured; setting "
"a strategy with neither affinity is rejected, unless "
"--enable-topology-rerank is set (the rerank carries the "
"strategy itself). The parallel sizes below are consulted only "
"when a node-group affinity is configured.",
help="Strategy steering the EP/PP/DP layout. 'ep_dp_pp' keeps the "
"contiguous layout. 'ep_pp_dp' packs EP-group slots of a Megatron "
"MoE pipeline stage into ep-pp columns across segments so EP "
"groups and PP stay intra-segment while only the dense DP "
"collective crosses segments. With --enable-topology-rerank the "
"observed psw topology (not the strategy packing) decides the "
"world order and the strategy only selects the packing policy; "
"a strategy no longer requires --group-affinity (the rerank "
"carries the strategy itself), and --group-affinity may also be "
"set alone (placement-only, no strategy) since the parallel "
"sizes are decoupled from the groups.",
)
parser.add_argument(
"--tensor-model-parallel-size",
Expand All @@ -185,17 +189,19 @@ def _build_master_args_parser():
"--pp",
default=1,
type=pos_int,
help="Pipeline-parallel size. Consulted when a node-group affinity "
"is configured.",
help="Pipeline-parallel size. Consulted by "
"--enable-topology-rerank and the legacy ep_pp_dp node-group "
"schedule (no rerank).",
)
parser.add_argument(
"--expert-model-parallel-size",
"--expert_model_parallel_size",
"--ep",
default=1,
type=pos_int,
help="Expert-model-parallel size. Consulted when a node-group "
"affinity is configured; the EP group size is --etp * --ep.",
help="Expert-model-parallel size. Consulted by "
"--enable-topology-rerank and the legacy ep_pp_dp node-group "
"schedule (no rerank); the EP group size is --etp * --ep.",
)
parser.add_argument(
"--expert-tensor-parallel-size",
Expand All @@ -207,8 +213,8 @@ def _build_master_args_parser():
"--tensor-model-parallel-size. When set explicitly it must be "
"used together with --expert-model-parallel-size; the real EP "
"group size is expert-tensor-parallel-size * "
"expert-model-parallel-size. Consulted when a node-group affinity "
"is configured.",
"expert-model-parallel-size. Consulted by --enable-topology-rerank "
"and the legacy ep_pp_dp node-group schedule (no rerank).",
)
parser.add_argument(
"--context-parallel-size",
Expand Down Expand Up @@ -253,10 +259,11 @@ def _build_master_args_parser():
"sizes) so EP/PP communication groups stay intra-psw as much as "
"possible. Requires --node-group-strategy and the parallel "
"sizes; may be combined with --group-affinity or "
"--soft-group-affinity: the declared groups then steer the "
"initial pod placement (labels) while the observed psw groups "
"decide the final world order, and the failover drift is "
"absorbed by the next rerank (single-pod relaunch keeps its "
"--soft-group-affinity: the declared groups then only steer the "
"initial pod placement (labels), the EP/PP/DP layout is NOT "
"validated against the declared sizes, and the observed psw "
"groups decide the final world order, while the failover drift "
"is absorbed by the next rerank (single-pod relaunch keeps its "
"group labels). Only the world ORDER changes (node_rank "
"values stay the stable pod identity).",
)
Expand Down
87 changes: 59 additions & 28 deletions dlrover/python/master/node/dist_job_manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -311,24 +311,30 @@ def _check_expert_tensor_pairing(job_args: JobArgs):
def _init_group_affinity(self, job_args: JobArgs):
"""Validate and apply the ``--group-affinity`` configuration.

When ``group_affinity`` is configured, the worker replicas declared
in the ElasticJob CRD must equal the sum of all group sizes and
``--node-group-strategy`` must be set explicitly; the master fails
to start otherwise. On success the mapping is forwarded to the
``JobResource`` so that worker nodes are partitioned into the
configured node groups.

When ``node_group_strategy == ep_pp_dp`` the parallel topology is
additionally validated via :func:`validate_topology` and a
:class:`NodeGroupSchedule` is attached to the ``JobResource`` so
that the scaler packs EP-group slots into ep-pp columns across
segments (EP/PP intra-segment, DP cross-segment). The parallel
sizes are read from ``job_args`` only when a node-group affinity
is configured; an unset ``--expert-tensor-parallel-size`` (None)
inherits ``--tensor-model-parallel-size``.
``group_affinity`` is a placement-only declaration: the worker
replicas declared in the ElasticJob CRD must equal the sum of all
group sizes (the master fails to start otherwise), and on success
the mapping is forwarded to the ``JobResource`` so that worker
nodes are partitioned into the configured scheduling segments
(the contiguous ``resolve_group_id`` mapping; pods carry the
``scheduling/rack-group`` labels). It is decoupled from the
Megatron parallel sizes (TP/PP/EP/CP): no parallel-size
constraint is applied to the declared group sizes, and
``--node-group-strategy`` is no longer required.

``enable_topology_rerank`` decides the parallel layout: with
rerank, the observed psw groups at the training rendezvous
decide the world order, and the decelerated affinity only
steers the initial pod placement. Without rerank, ``ep_pp_dp``
keeps its legacy coupled behavior: the advised topology is
validated via :func:`validate_topology` and a
:class:`NodeGroupSchedule` is attached to the ``JobResource``
so that the scaler packs EP-group slots into ep-pp columns
across segments (EP/PP intra-segment, DP cross-segment).
"""
strategy = getattr(job_args, "node_group_strategy", None)
ep_pp_dp = strategy == NodeGroupStrategy.EP_PP_DP
rerank_enabled = getattr(self, "_topology_rerank_enabled", False)
if not job_args.group_affinity:
# --soft-group-affinity configures the groups its own way
# (unequal sizes) and --enable-topology-rerank carries the
Expand All @@ -338,7 +344,7 @@ def _init_group_affinity(self, job_args: JobArgs):
if (
strategy is not None
and not getattr(job_args, "soft_group_affinity", None)
and not getattr(self, "_topology_rerank_enabled", False)
and not rerank_enabled
):
raise ValueError(
"--node-group-strategy requires --group-affinity or "
Expand All @@ -347,13 +353,6 @@ def _init_group_affinity(self, job_args: JobArgs):
"strategy itself)."
)
return
if strategy is None:
raise ValueError(
"--group-affinity requires --node-group-strategy to be set "
"(ep_dp_pp keeps the contiguous layout; ep_pp_dp packs "
"EP-PP columns across groups)."
)
self._check_expert_tensor_pairing(job_args)
worker_resource = self._job_resource.node_group_resources.get(
NodeType.WORKER
)
Expand All @@ -364,12 +363,26 @@ def _init_group_affinity(self, job_args: JobArgs):
f"--group-affinity requires worker replicas={expected}, "
f"but got CRD worker replicas={worker_count}"
)

# For ep_pp_dp, validate the full parallel topology BEFORE mutating any
# JobResource state so a violation leaves the resource untouched and
# the master fails to start cleanly.
# Args-level self-consistency (doc: ``etp`` requires ``ep``): the
# EP/PP sizes only have a consumer when a strategy or the rerank
# is configured; a purely placement-only affinity (no strategy,
# no rerank) does not consult them and skips the pairing check.
if strategy is not None or rerank_enabled:
self._check_expert_tensor_pairing(job_args)

# Without --enable-topology-rerank there is no join-time world
# reordering, so the ep_pp_dp packing IS the layout: keep the
# legacy coupled behavior (validate the EP/PP/DP parallel topology
# against the declared segment sizes and attach the NodeGroupSchedule
# so the scaler packs EP-group slots into ep-pp columns). With
# rerank enabled the declared affinity steers the initial pod
# placement only (placement-only mode): no NodeGroupSchedule is
# attached, resolve_group_id keeps the contiguous mapping, and the
# observed psw groups decide the final world order. In both cases
# validate BEFORE mutating any JobResource state so a violation
# leaves the resource untouched and the master fails cleanly.
schedule = None
if ep_pp_dp:
if ep_pp_dp and not rerank_enabled:
ranks_per_node = (
worker_resource.node_resource.gpu_num
if worker_resource is not None
Expand Down Expand Up @@ -416,6 +429,24 @@ def _init_group_affinity(self, job_args: JobArgs):
schedule.etp * schedule.ep,
len(job_args.group_affinity),
)
elif ep_pp_dp:
logger.info(
"group-affinity is placement-only with --enable-topology-"
"rerank: the declared %d segments (ids %s) steer the "
"initial pod labels via the contiguous resolver; no "
"EP/PP/DP validation runs and no NodeGroupSchedule is "
"attached — the rendezvous world order is recomputed from "
"the psw topology observed at join time.",
len(job_args.group_affinity),
sorted(job_args.group_affinity.keys()),
)
else:
logger.info(
"group-affinity without a strategy: the declared %d groups "
"only steer the initial pod labels via the contiguous "
"resolver; no parallel-size coupling applies.",
len(job_args.group_affinity),
)

def _init_soft_group_affinity(self, job_args: JobArgs):
"""Validate and apply the ``--soft-group-affinity`` configuration.
Expand Down
17 changes: 11 additions & 6 deletions dlrover/python/master/resource/job.py
Original file line number Diff line number Diff line change
Expand Up @@ -166,13 +166,18 @@ def resolve_group_id(
- With the default ``schedule`` (None) or its strategy being
:data:`NodeGroupStrategy.EP_DP_PP`, groups are laid out in ascending
group-id order, each occupying a contiguous ``[start, end)`` rank range
sized by ``group_affinity[group_id]``. This is the existing behavior.
sized by ``group_affinity[group_id]``. This is the placement-only
behavior running whenever the affinity is decoupled from the
parallel topology (``--enable-topology-rerank`` on, or no ep_pp_dp
schedule attached).
- With strategy :data:`NodeGroupStrategy.EP_PP_DP`, the stripe mapping in
:func:`_resolve_stripe_group_id` is used instead. This requires a
validated topology (see :func:`validate_topology`); as the topology
is fixed-size, the rank is asserted to be within
``schedule.num_nodes``. Group ids are opaque: they only label the
segments, with the ascending id order fixing the segment order.
:func:`_resolve_stripe_group_id` is used instead. This applies only
on the legacy coupled path (ep_pp_dp with NO rerank, where the
packing IS the layout) and requires a validated topology (see
:func:`validate_topology`); as the topology is fixed-size, the rank
is asserted to be within ``schedule.num_nodes``. Group ids are
opaque: they only label the segments, with the ascending id order
fixing the segment order.

Returns ``None`` when group affinity is not configured or the rank
falls outside the declared worker range. For ``ep_pp_dp``, an
Expand Down
90 changes: 79 additions & 11 deletions dlrover/python/tests/test_job_manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -388,14 +388,80 @@ def test_init_group_affinity_ep_dp_pp_leaves_schedule_unset(self):
self.assertEqual(manager._job_resource.group_affinity, {0: 10, 1: 15})
self.assertIsNone(manager._job_resource.node_group_schedule)

def test_init_group_affinity_requires_strategy(self):
# group-affinity configured without the strategy is a hard error.
def test_init_group_affinity_without_strategy_placement_only(self):
# Since the placement-only decoupling, --group-affinity works
# standalone (without --node-group-strategy): it only partitions
# the worker pods into scheduling segments and no schedule is
# attached.
manager = self._new_manager_with_worker_count(25)
with self.assertRaisesRegex(
ValueError, "requires --node-group-strategy"
):
manager._init_group_affinity(
SimpleNamespace(group_affinity={0: 10, 1: 15})
)
self.assertEqual(manager._job_resource.group_affinity, {0: 10, 1: 15})
self.assertIsNone(manager._job_resource.node_group_schedule)

def test_init_group_affinity_placement_only_with_rerank(self):
# ep_pp_dp WITH --enable-topology-rerank: the declared affinity
# steers the initial pod placement only. Parallel sizes that
# violate the previously mandatory ep_pp_dp topology (here
# dense_dp=G=... never even validates) must NOT fail the master;
# no NodeGroupSchedule is attached so resolve_group_id keeps the
# contiguous mapping.
manager = self._new_manager_with_worker_count(24, gpu_num=8)
manager._topology_rerank_enabled = True
manager._init_group_affinity(
SimpleNamespace(
group_affinity={0: 8, 1: 8, 2: 8},
node_group_strategy=NodeGroupStrategy.EP_PP_DP,
tensor_model_parallel_size=1,
pipeline_model_parallel_size=3,
expert_model_parallel_size=8,
context_parallel_size=1,
)
)
self.assertEqual(
manager._job_resource.group_affinity, {0: 8, 1: 8, 2: 8}
)
self.assertIsNone(manager._job_resource.node_group_schedule)

def test_init_group_affinity_placement_only_sizes_independent(self):
# The placement-only path does not even require the group sizes
# to divide anything: arbitrary sizes are accepted with rerank.
manager = self._new_manager_with_worker_count(1152, gpu_num=8)
manager._topology_rerank_enabled = True
sizes = {3: 128, 5: 128, 7: 128, 11: 256, 17: 512}
manager._init_group_affinity(
SimpleNamespace(
group_affinity=dict(sizes),
node_group_strategy=NodeGroupStrategy.EP_PP_DP,
pipeline_model_parallel_size=4,
expert_model_parallel_size=17,
context_parallel_size=1,
)
)
self.assertEqual(manager._job_resource.group_affinity, sizes)
self.assertIsNone(manager._job_resource.node_group_schedule)

def test_init_group_affinity_ep_pp_dp_without_rerank_still_validates(
self,
):
# Backward compatibility: ep_pp_dp with an affinity but WITHOUT
# --enable-topology-rerank keeps the coupled behavior — the
# topology is validated and the schedule is attached (the EP-PP
# packing is then the layout, not the rerank).
manager = self._new_manager_with_worker_count(24, gpu_num=8)
sizes = {0: 8, 1: 8, 2: 8}
with self.assertRaisesRegex(ValueError, "EP group size"):
manager._init_group_affinity(
SimpleNamespace(group_affinity={0: 10, 1: 15})
SimpleNamespace(
group_affinity=dict(sizes),
node_group_strategy=NodeGroupStrategy.EP_PP_DP,
pipeline_model_parallel_size=1,
# EP group 5*3=15 does not divide S=dense_dp/G=64 and
# is too large for the segment block.
expert_model_parallel_size=5,
context_parallel_size=1,
)
)
self.assertIsNone(manager._job_resource.group_affinity)
self.assertIsNone(manager._job_resource.node_group_schedule)
Expand Down Expand Up @@ -756,9 +822,11 @@ def test_init_topology_rerank_via_job_manager_init(self):
def test_init_group_affinity_rerank_coexist_via_job_manager_init(
self,
):
# End-to-end coexistence: the declared hard groups steer the
# initial pod placement (labels) while the rerank reorders the
# world from the psw topology observed at the rendezvous.
# End-to-end coexistence (placement-only affinity): the declared
# hard groups steer the initial pod placement (labels) while the
# rerank reorders the world from the psw topology observed at the
# rendezvous; no ep_pp_dp schedule is attached so the contiguous
# resolver labels the pods and the ranks-per-node check is off.
params = MockK8sAllreduceJobArgs()
params.initilize(16)
params.enable_topology_rerank = True
Expand All @@ -771,9 +839,9 @@ def test_init_group_affinity_rerank_coexist_via_job_manager_init(
manager = create_job_manager(params, PerfMonitor())
self.assertTrue(manager._topology_rerank_enabled)
self.assertEqual(manager._job_resource.group_affinity, {0: 8, 1: 8})
self.assertIsNotNone(manager._job_resource.node_group_schedule)
self.assertIsNone(manager._job_resource.node_group_schedule)
self.assertIsNone(manager._job_resource.soft_group_schedule)
self.assertEqual(manager.get_expected_ranks_per_node(), 8)
self.assertIsNone(manager.get_expected_ranks_per_node())

def test_init_soft_group_rerank_coexist_via_job_manager_init(self):
# End-to-end coexistence: the declared soft groups steer the
Expand Down
Loading