diff --git a/dlrover/python/master/elastic_training/topology_rerank.py b/dlrover/python/master/elastic_training/topology_rerank.py index f9d8a971b..c8a7963cd 100644 --- a/dlrover/python/master/elastic_training/topology_rerank.py +++ b/dlrover/python/master/elastic_training/topology_rerank.py @@ -160,13 +160,22 @@ def _ep_pp_dp_group_sequence( The world is cut into COLUMNS: a column holds ``pp`` EP slots and spans all the pipeline stages of one Megatron PP column, and an EP slot is ``ep_workers`` consecutive nodes of one EP communication - group. Each column is filled by a greedy that repeatedly takes slots - from the group with the MOST remaining whole slots (ties broken by - the smallest group id): a group with at least ``pp`` remaining slots - fills whole columns alone so its PP stages never leave the psw, and - only residual fragments share a column. The columns are flattened - stage-major: pipeline stage ``s`` consumes the s-th slot of every - column. + group. Columns come in two filling phases: + + - Whole columns: while some group holds at least all the slots the + column still needs, the largest-remaining group (ties broken by + the smallest group id) fills it alone, so its PP stages never + leave the psw (pure columns). + - Residue columns: once no group can own the rest of a column, the + column accumulates from the SMALLEST remaining group (ties by the + smallest group id). Smallest-first pairing keeps every fragment + group fully consumed inside exactly two columns: e.g. 9 psws x 8 + leftover slots pack into 6 strictly two-segment columns ([4|8] + each) instead of drifting into a three-segment column, which + minimizes the PP p2p boundaries that cross segments. + + The columns are flattened stage-major: pipeline stage ``s`` consumes + the s-th slot of every column. Unlike the --soft-group-affinity path there is NO alignment constraint here: the group sizes are the observed per-psw node @@ -196,11 +205,31 @@ def _ep_pp_dp_group_sequence( for _ in range(columns): slots: List[int] = [] while len(slots) < pp and any(v > 0 for v in remaining.values()): - group_id = max( + need = pp - len(slots) + max_key = max( sorted(remaining.keys()), key=lambda k: (remaining[k], -k), ) - take = min(remaining[group_id], pp - len(slots)) + if remaining[max_key] >= need: + # Whole-column take: this group owns the rest of the + # column alone, so its PP stages stay inside the psw. + group_id, take = max_key, need + else: + # Residue column: no group can own the rest. Accumulate + # from the SMALLEST remaining group (ties by the smallest + # id) so each fragment group is spent inside (at most) two + # columns: a group taken as the 4-slot head half comes + # back as the next smallest and is finished there, and the + # residue columns then stay two-segment whenever the sizes + # allow it. + # The smallest non-empty remaining only; zero-remaining + # groups must stay excluded so every take advances the + # packing. + group_id = min( + (k for k, v in remaining.items() if v > 0), + key=lambda k: (remaining[k], k), + ) + take = min(remaining[group_id], need) slots.extend([group_id] * take) remaining[group_id] -= take column_slots.append(slots) diff --git a/dlrover/python/tests/test_topology_rerank.py b/dlrover/python/tests/test_topology_rerank.py index b5ef5f0ea..c025e2396 100644 --- a/dlrover/python/tests/test_topology_rerank.py +++ b/dlrover/python/tests/test_topology_rerank.py @@ -217,6 +217,78 @@ def test_ep_pp_dp_fills_empty_columns_with_crumb_slots(self): [0, 0, 1, 1, 0, 1, 0, 0, 2, 2, 2, 3], ) + def test_ep_pp_dp_residues_pack_into_two_segment_columns(self): + # 9 psws x 128 nodes, pp=12 and ep_workers=4 (the ling-max scale): + # each psw holds 32 slots, so 18 pure columns consume 24 slots of + # every psw and the leftover 9 x 8 slots make 6 residue columns. + # The residue packing accumulates smallest-first and spends each + # fragment group inside exactly two columns, so ALL 6 residue + # columns are strictly two-segment ([4|8] pairings, zero + # three-segment columns) and the PP p2p boundaries that cross + # segments stay minimal (6, one per residue column). + sizes = {i: 128 for i in range(9)} + pp, ep_workers = 12, 4 + sequence = _ep_pp_dp_group_sequence(sizes, pp, ep_workers) + self.assertEqual(len(sequence), sum(sizes.values())) + columns = (sum(sizes.values()) // pp) // ep_workers + # slot (stage, col) = the ep_workers nodes of the s-th slot of + # every column (stage-major flatten) + slot_groups = {} + for stage in range(pp): + for col in range(columns): + start = stage * columns * ep_workers + col * ep_workers + slot_groups[(stage, col)] = set( + sequence[start : start + ep_workers] + ) + # every EP slot stays inside one psw group + for key, groups_in_slot in slot_groups.items(): + self.assertEqual(len(groups_in_slot), 1, key) + # every PP column spans at most 2 segments; all 9 x 8 residue + # slots pack into exactly 6 two-segment columns + col_groups = {} + for col in range(columns): + groups = set() + for stage in range(pp): + groups |= slot_groups[(stage, col)] + col_groups[col] = groups + mixed = [c for c in range(columns) if len(col_groups[c]) > 1] + self.assertEqual(len(mixed), 6) + for col, groups in col_groups.items(): + self.assertLessEqual( + len(groups), + 2, + f"column {col} spans {len(groups)} groups", + ) + cross_boundaries = 0 + for col in range(columns): + stage_seq = [next(iter(slot_groups[(s, col)])) for s in range(pp)] + cross_boundaries += sum( + 1 for s in range(pp - 1) if stage_seq[s] != stage_seq[s + 1] + ) + self.assertEqual(cross_boundaries, 6) + + def test_ep_pp_dp_residue_column_accumulates_smallest_first(self): + # Every group holds fewer slots than a whole column: the residue + # packing accumulates from the SMALLEST remaining group (ties by + # the smallest id), so the fragment of a half-used group comes + # back as the next smallest and finishes inside the very next + # column. 4 groups x 2 slots on pp=3: col0 = [0,0,1], col1 = + # [1,2,2] and the two leftover whole slots drain to the tail as + # [3,3] (the pre-existing tail behavior). + self.assertEqual( + _ep_pp_dp_group_sequence( + {0: 2, 1: 2, 2: 2, 3: 2}, pp=3, ep_workers=1 + ), + [0, 1, 0, 2, 1, 2, 3, 3], + ) + + def test_ep_pp_dp_sort_empty_nodes_keeps_empty(self): + # A no-node rendezvous keeps the empty mapping (early return). + sorter = GroupTopologySorter( + strategy=NodeGroupStrategy.EP_PP_DP, pp=2, ep=8 + ) + self.assertEqual(sorter.sort({}), {}) + class GroupTopologySorterTest(unittest.TestCase): def test_ep_dp_pp_groups_are_contiguous(self):