From c69a539752783c83ad1faae8b902fb72c93779cf Mon Sep 17 00:00:00 2001 From: Ma Jie Yue Date: Mon, 16 Nov 2026 12:49:23 +0800 Subject: [PATCH] Feat: decouple --group-affinity from the EP/PP/DP layout With --enable-topology-rerank the declared --group-affinity becomes a placement-only declaration: the master no longer validates the EP/PP/DP parallel topology against the declared segment sizes and no longer attaches a NodeGroupSchedule, so any group sizes now work with any PP/EP (e.g. ep_pp_dp + 9 x 128 segments with PP=12). resolve_group_id keeps the contiguous mapping for pod labels (scheduling/rack-group) and the observed psw topology decides the rendezvous world order via the rerank. --group-affinity also works standalone (no --node-group-strategy). The legacy coupled behavior is preserved for ep_pp_dp WITHOUT rerank (the EP-PP packing is then the layout): validate_topology still runs and the schedule is still attached there; the EP-PP stripe resolver and validate_topology are unchanged for that path. Co-Authored-By: Claude --- dlrover/python/master/args.py | 51 ++++++----- .../python/master/node/dist_job_manager.py | 87 ++++++++++++------ dlrover/python/master/resource/job.py | 17 ++-- dlrover/python/tests/test_job_manager.py | 90 ++++++++++++++++--- 4 files changed, 178 insertions(+), 67 deletions(-) diff --git a/dlrover/python/master/args.py b/dlrover/python/master/args.py index f6915deb5..f150629b6 100644 --- a/dlrover/python/master/args.py +++ b/dlrover/python/master/args.py @@ -151,24 +151,28 @@ def _build_master_args_parser(): help='Node group sizes, e.g. --group-affinity="{0: 10, 1: 15}" ' "means group 0 has 10 pods and group 1 has 15 pods. The worker " "replicas in the ElasticJob CRD must equal the sum of all group " - "sizes, otherwise the master fails to start.", + "sizes, otherwise the master fails to start. Placement-only: the " + "sizes are NOT coupled to the Megatron parallel sizes, and no " + "EP/PP/DP layout validation is applied to them when " + "--enable-topology-rerank is on (the EP-PP column packing runs " + "only on the legacy ep_pp_dp-without-rerank path).", ) parser.add_argument( "--node-group-strategy", "--node_group_strategy", default=None, choices=["ep_dp_pp", "ep_pp_dp"], - help="Strategy used to map worker nodes (by creation order) into " - "node groups (== physical segments). 'ep_dp_pp' keeps the existing " - "behavior: groups occupy contiguous rank ranges. 'ep_pp_dp' packs " - "EP-group slots of a Megatron MoE pipeline stage into ep-pp columns " - "across groups so EP groups and PP stay intra-segment while only " - "the dense DP collective crosses segments. Required when " - "--group-affinity or --soft-group-affinity is configured; setting " - "a strategy with neither affinity is rejected, unless " - "--enable-topology-rerank is set (the rerank carries the " - "strategy itself). The parallel sizes below are consulted only " - "when a node-group affinity is configured.", + help="Strategy steering the EP/PP/DP layout. 'ep_dp_pp' keeps the " + "contiguous layout. 'ep_pp_dp' packs EP-group slots of a Megatron " + "MoE pipeline stage into ep-pp columns across segments so EP " + "groups and PP stay intra-segment while only the dense DP " + "collective crosses segments. With --enable-topology-rerank the " + "observed psw topology (not the strategy packing) decides the " + "world order and the strategy only selects the packing policy; " + "a strategy no longer requires --group-affinity (the rerank " + "carries the strategy itself), and --group-affinity may also be " + "set alone (placement-only, no strategy) since the parallel " + "sizes are decoupled from the groups.", ) parser.add_argument( "--tensor-model-parallel-size", @@ -185,8 +189,9 @@ def _build_master_args_parser(): "--pp", default=1, type=pos_int, - help="Pipeline-parallel size. Consulted when a node-group affinity " - "is configured.", + help="Pipeline-parallel size. Consulted by " + "--enable-topology-rerank and the legacy ep_pp_dp node-group " + "schedule (no rerank).", ) parser.add_argument( "--expert-model-parallel-size", @@ -194,8 +199,9 @@ def _build_master_args_parser(): "--ep", default=1, type=pos_int, - help="Expert-model-parallel size. Consulted when a node-group " - "affinity is configured; the EP group size is --etp * --ep.", + help="Expert-model-parallel size. Consulted by " + "--enable-topology-rerank and the legacy ep_pp_dp node-group " + "schedule (no rerank); the EP group size is --etp * --ep.", ) parser.add_argument( "--expert-tensor-parallel-size", @@ -207,8 +213,8 @@ def _build_master_args_parser(): "--tensor-model-parallel-size. When set explicitly it must be " "used together with --expert-model-parallel-size; the real EP " "group size is expert-tensor-parallel-size * " - "expert-model-parallel-size. Consulted when a node-group affinity " - "is configured.", + "expert-model-parallel-size. Consulted by --enable-topology-rerank " + "and the legacy ep_pp_dp node-group schedule (no rerank).", ) parser.add_argument( "--context-parallel-size", @@ -253,10 +259,11 @@ def _build_master_args_parser(): "sizes) so EP/PP communication groups stay intra-psw as much as " "possible. Requires --node-group-strategy and the parallel " "sizes; may be combined with --group-affinity or " - "--soft-group-affinity: the declared groups then steer the " - "initial pod placement (labels) while the observed psw groups " - "decide the final world order, and the failover drift is " - "absorbed by the next rerank (single-pod relaunch keeps its " + "--soft-group-affinity: the declared groups then only steer the " + "initial pod placement (labels), the EP/PP/DP layout is NOT " + "validated against the declared sizes, and the observed psw " + "groups decide the final world order, while the failover drift " + "is absorbed by the next rerank (single-pod relaunch keeps its " "group labels). Only the world ORDER changes (node_rank " "values stay the stable pod identity).", ) diff --git a/dlrover/python/master/node/dist_job_manager.py b/dlrover/python/master/node/dist_job_manager.py index c2228ca39..09ebd61c6 100644 --- a/dlrover/python/master/node/dist_job_manager.py +++ b/dlrover/python/master/node/dist_job_manager.py @@ -311,24 +311,30 @@ def _check_expert_tensor_pairing(job_args: JobArgs): def _init_group_affinity(self, job_args: JobArgs): """Validate and apply the ``--group-affinity`` configuration. - When ``group_affinity`` is configured, the worker replicas declared - in the ElasticJob CRD must equal the sum of all group sizes and - ``--node-group-strategy`` must be set explicitly; the master fails - to start otherwise. On success the mapping is forwarded to the - ``JobResource`` so that worker nodes are partitioned into the - configured node groups. - - When ``node_group_strategy == ep_pp_dp`` the parallel topology is - additionally validated via :func:`validate_topology` and a - :class:`NodeGroupSchedule` is attached to the ``JobResource`` so - that the scaler packs EP-group slots into ep-pp columns across - segments (EP/PP intra-segment, DP cross-segment). The parallel - sizes are read from ``job_args`` only when a node-group affinity - is configured; an unset ``--expert-tensor-parallel-size`` (None) - inherits ``--tensor-model-parallel-size``. + ``group_affinity`` is a placement-only declaration: the worker + replicas declared in the ElasticJob CRD must equal the sum of all + group sizes (the master fails to start otherwise), and on success + the mapping is forwarded to the ``JobResource`` so that worker + nodes are partitioned into the configured scheduling segments + (the contiguous ``resolve_group_id`` mapping; pods carry the + ``scheduling/rack-group`` labels). It is decoupled from the + Megatron parallel sizes (TP/PP/EP/CP): no parallel-size + constraint is applied to the declared group sizes, and + ``--node-group-strategy`` is no longer required. + + ``enable_topology_rerank`` decides the parallel layout: with + rerank, the observed psw groups at the training rendezvous + decide the world order, and the decelerated affinity only + steers the initial pod placement. Without rerank, ``ep_pp_dp`` + keeps its legacy coupled behavior: the advised topology is + validated via :func:`validate_topology` and a + :class:`NodeGroupSchedule` is attached to the ``JobResource`` + so that the scaler packs EP-group slots into ep-pp columns + across segments (EP/PP intra-segment, DP cross-segment). """ strategy = getattr(job_args, "node_group_strategy", None) ep_pp_dp = strategy == NodeGroupStrategy.EP_PP_DP + rerank_enabled = getattr(self, "_topology_rerank_enabled", False) if not job_args.group_affinity: # --soft-group-affinity configures the groups its own way # (unequal sizes) and --enable-topology-rerank carries the @@ -338,7 +344,7 @@ def _init_group_affinity(self, job_args: JobArgs): if ( strategy is not None and not getattr(job_args, "soft_group_affinity", None) - and not getattr(self, "_topology_rerank_enabled", False) + and not rerank_enabled ): raise ValueError( "--node-group-strategy requires --group-affinity or " @@ -347,13 +353,6 @@ def _init_group_affinity(self, job_args: JobArgs): "strategy itself)." ) return - if strategy is None: - raise ValueError( - "--group-affinity requires --node-group-strategy to be set " - "(ep_dp_pp keeps the contiguous layout; ep_pp_dp packs " - "EP-PP columns across groups)." - ) - self._check_expert_tensor_pairing(job_args) worker_resource = self._job_resource.node_group_resources.get( NodeType.WORKER ) @@ -364,12 +363,26 @@ def _init_group_affinity(self, job_args: JobArgs): f"--group-affinity requires worker replicas={expected}, " f"but got CRD worker replicas={worker_count}" ) - - # For ep_pp_dp, validate the full parallel topology BEFORE mutating any - # JobResource state so a violation leaves the resource untouched and - # the master fails to start cleanly. + # Args-level self-consistency (doc: ``etp`` requires ``ep``): the + # EP/PP sizes only have a consumer when a strategy or the rerank + # is configured; a purely placement-only affinity (no strategy, + # no rerank) does not consult them and skips the pairing check. + if strategy is not None or rerank_enabled: + self._check_expert_tensor_pairing(job_args) + + # Without --enable-topology-rerank there is no join-time world + # reordering, so the ep_pp_dp packing IS the layout: keep the + # legacy coupled behavior (validate the EP/PP/DP parallel topology + # against the declared segment sizes and attach the NodeGroupSchedule + # so the scaler packs EP-group slots into ep-pp columns). With + # rerank enabled the declared affinity steers the initial pod + # placement only (placement-only mode): no NodeGroupSchedule is + # attached, resolve_group_id keeps the contiguous mapping, and the + # observed psw groups decide the final world order. In both cases + # validate BEFORE mutating any JobResource state so a violation + # leaves the resource untouched and the master fails cleanly. schedule = None - if ep_pp_dp: + if ep_pp_dp and not rerank_enabled: ranks_per_node = ( worker_resource.node_resource.gpu_num if worker_resource is not None @@ -416,6 +429,24 @@ def _init_group_affinity(self, job_args: JobArgs): schedule.etp * schedule.ep, len(job_args.group_affinity), ) + elif ep_pp_dp: + logger.info( + "group-affinity is placement-only with --enable-topology-" + "rerank: the declared %d segments (ids %s) steer the " + "initial pod labels via the contiguous resolver; no " + "EP/PP/DP validation runs and no NodeGroupSchedule is " + "attached — the rendezvous world order is recomputed from " + "the psw topology observed at join time.", + len(job_args.group_affinity), + sorted(job_args.group_affinity.keys()), + ) + else: + logger.info( + "group-affinity without a strategy: the declared %d groups " + "only steer the initial pod labels via the contiguous " + "resolver; no parallel-size coupling applies.", + len(job_args.group_affinity), + ) def _init_soft_group_affinity(self, job_args: JobArgs): """Validate and apply the ``--soft-group-affinity`` configuration. diff --git a/dlrover/python/master/resource/job.py b/dlrover/python/master/resource/job.py index fd48ddbea..88699ef84 100644 --- a/dlrover/python/master/resource/job.py +++ b/dlrover/python/master/resource/job.py @@ -166,13 +166,18 @@ def resolve_group_id( - With the default ``schedule`` (None) or its strategy being :data:`NodeGroupStrategy.EP_DP_PP`, groups are laid out in ascending group-id order, each occupying a contiguous ``[start, end)`` rank range - sized by ``group_affinity[group_id]``. This is the existing behavior. + sized by ``group_affinity[group_id]``. This is the placement-only + behavior running whenever the affinity is decoupled from the + parallel topology (``--enable-topology-rerank`` on, or no ep_pp_dp + schedule attached). - With strategy :data:`NodeGroupStrategy.EP_PP_DP`, the stripe mapping in - :func:`_resolve_stripe_group_id` is used instead. This requires a - validated topology (see :func:`validate_topology`); as the topology - is fixed-size, the rank is asserted to be within - ``schedule.num_nodes``. Group ids are opaque: they only label the - segments, with the ascending id order fixing the segment order. + :func:`_resolve_stripe_group_id` is used instead. This applies only + on the legacy coupled path (ep_pp_dp with NO rerank, where the + packing IS the layout) and requires a validated topology (see + :func:`validate_topology`); as the topology is fixed-size, the rank + is asserted to be within ``schedule.num_nodes``. Group ids are + opaque: they only label the segments, with the ascending id order + fixing the segment order. Returns ``None`` when group affinity is not configured or the rank falls outside the declared worker range. For ``ep_pp_dp``, an diff --git a/dlrover/python/tests/test_job_manager.py b/dlrover/python/tests/test_job_manager.py index bbaa95deb..61dc23447 100644 --- a/dlrover/python/tests/test_job_manager.py +++ b/dlrover/python/tests/test_job_manager.py @@ -388,14 +388,80 @@ def test_init_group_affinity_ep_dp_pp_leaves_schedule_unset(self): self.assertEqual(manager._job_resource.group_affinity, {0: 10, 1: 15}) self.assertIsNone(manager._job_resource.node_group_schedule) - def test_init_group_affinity_requires_strategy(self): - # group-affinity configured without the strategy is a hard error. + def test_init_group_affinity_without_strategy_placement_only(self): + # Since the placement-only decoupling, --group-affinity works + # standalone (without --node-group-strategy): it only partitions + # the worker pods into scheduling segments and no schedule is + # attached. manager = self._new_manager_with_worker_count(25) - with self.assertRaisesRegex( - ValueError, "requires --node-group-strategy" - ): + manager._init_group_affinity( + SimpleNamespace(group_affinity={0: 10, 1: 15}) + ) + self.assertEqual(manager._job_resource.group_affinity, {0: 10, 1: 15}) + self.assertIsNone(manager._job_resource.node_group_schedule) + + def test_init_group_affinity_placement_only_with_rerank(self): + # ep_pp_dp WITH --enable-topology-rerank: the declared affinity + # steers the initial pod placement only. Parallel sizes that + # violate the previously mandatory ep_pp_dp topology (here + # dense_dp=G=... never even validates) must NOT fail the master; + # no NodeGroupSchedule is attached so resolve_group_id keeps the + # contiguous mapping. + manager = self._new_manager_with_worker_count(24, gpu_num=8) + manager._topology_rerank_enabled = True + manager._init_group_affinity( + SimpleNamespace( + group_affinity={0: 8, 1: 8, 2: 8}, + node_group_strategy=NodeGroupStrategy.EP_PP_DP, + tensor_model_parallel_size=1, + pipeline_model_parallel_size=3, + expert_model_parallel_size=8, + context_parallel_size=1, + ) + ) + self.assertEqual( + manager._job_resource.group_affinity, {0: 8, 1: 8, 2: 8} + ) + self.assertIsNone(manager._job_resource.node_group_schedule) + + def test_init_group_affinity_placement_only_sizes_independent(self): + # The placement-only path does not even require the group sizes + # to divide anything: arbitrary sizes are accepted with rerank. + manager = self._new_manager_with_worker_count(1152, gpu_num=8) + manager._topology_rerank_enabled = True + sizes = {3: 128, 5: 128, 7: 128, 11: 256, 17: 512} + manager._init_group_affinity( + SimpleNamespace( + group_affinity=dict(sizes), + node_group_strategy=NodeGroupStrategy.EP_PP_DP, + pipeline_model_parallel_size=4, + expert_model_parallel_size=17, + context_parallel_size=1, + ) + ) + self.assertEqual(manager._job_resource.group_affinity, sizes) + self.assertIsNone(manager._job_resource.node_group_schedule) + + def test_init_group_affinity_ep_pp_dp_without_rerank_still_validates( + self, + ): + # Backward compatibility: ep_pp_dp with an affinity but WITHOUT + # --enable-topology-rerank keeps the coupled behavior — the + # topology is validated and the schedule is attached (the EP-PP + # packing is then the layout, not the rerank). + manager = self._new_manager_with_worker_count(24, gpu_num=8) + sizes = {0: 8, 1: 8, 2: 8} + with self.assertRaisesRegex(ValueError, "EP group size"): manager._init_group_affinity( - SimpleNamespace(group_affinity={0: 10, 1: 15}) + SimpleNamespace( + group_affinity=dict(sizes), + node_group_strategy=NodeGroupStrategy.EP_PP_DP, + pipeline_model_parallel_size=1, + # EP group 5*3=15 does not divide S=dense_dp/G=64 and + # is too large for the segment block. + expert_model_parallel_size=5, + context_parallel_size=1, + ) ) self.assertIsNone(manager._job_resource.group_affinity) self.assertIsNone(manager._job_resource.node_group_schedule) @@ -756,9 +822,11 @@ def test_init_topology_rerank_via_job_manager_init(self): def test_init_group_affinity_rerank_coexist_via_job_manager_init( self, ): - # End-to-end coexistence: the declared hard groups steer the - # initial pod placement (labels) while the rerank reorders the - # world from the psw topology observed at the rendezvous. + # End-to-end coexistence (placement-only affinity): the declared + # hard groups steer the initial pod placement (labels) while the + # rerank reorders the world from the psw topology observed at the + # rendezvous; no ep_pp_dp schedule is attached so the contiguous + # resolver labels the pods and the ranks-per-node check is off. params = MockK8sAllreduceJobArgs() params.initilize(16) params.enable_topology_rerank = True @@ -771,9 +839,9 @@ def test_init_group_affinity_rerank_coexist_via_job_manager_init( manager = create_job_manager(params, PerfMonitor()) self.assertTrue(manager._topology_rerank_enabled) self.assertEqual(manager._job_resource.group_affinity, {0: 8, 1: 8}) - self.assertIsNotNone(manager._job_resource.node_group_schedule) + self.assertIsNone(manager._job_resource.node_group_schedule) self.assertIsNone(manager._job_resource.soft_group_schedule) - self.assertEqual(manager.get_expected_ranks_per_node(), 8) + self.assertIsNone(manager.get_expected_ranks_per_node()) def test_init_soft_group_rerank_coexist_via_job_manager_init(self): # End-to-end coexistence: the declared soft groups steer the