diff --git a/python/sglang/srt/layers/communicator.py b/python/sglang/srt/layers/communicator.py index d547e04787..583d3b7c81 100644 --- a/python/sglang/srt/layers/communicator.py +++ b/python/sglang/srt/layers/communicator.py @@ -16,7 +16,7 @@ from contextlib import contextmanager from dataclasses import dataclass from enum import Enum, auto from functools import cached_property, partial -from typing import Callable, Dict, Optional, Tuple, Union +from typing import Callable, Dict, List, Optional, Tuple, Union import msgspec import torch @@ -767,6 +767,17 @@ class LayerCommunicator: if sides is not None else self._select_boundaries_from_scatter_modes() ) + # The steps a batch that shards its tokens over attention CP runs; None + # without CP or where the steps come from the scatter modes. + self._cp_steps = ( + _select_boundary_steps( + self._declared_sides(cp_active=True), + fusions=self._select_mlp_input_fusions(), + force_layernorm_before_gather=force_layernorm_before_dp_gather, + ) + if sides is not None and get_parallel().attn_cp_size > 1 + else None + ) # The steps a batch with input-scattered attention runs; None where it # cannot run or the steps come from the scatter modes. self._input_scattered_steps = ( @@ -815,11 +826,16 @@ class LayerCommunicator: and get_moe_a2a_backend().is_none() ) - def _declared_sides(self) -> Optional[DecoderLayerSides]: + def _declared_sides( + self, *, cp_active: bool = False + ) -> Optional[DecoderLayerSides]: """The declarations this layer's boundaries are chosen from: an - attention and an FFN, without CP, in a layer whose previous layer is - known. None when the steps come from the scatter modes. An active - LayerNorm SP region and input-scattered attention have their own.""" + attention and an FFN, in a layer whose previous layer is known, with + attention CP only for a prefill CP whose FFN gathers over the whole CP + group. None when the steps come from the scatter modes. ``cp_active`` + gives a batch that shards its tokens over CP; the others hold every + token on each CP rank. An active LayerNorm SP region and input-scattered + attention have their own.""" modes = self.layer_scatter_modes parallel = get_parallel() # A MoE dispatched per DP shard computes on this rank's local rows and @@ -827,13 +843,16 @@ class LayerCommunicator: moe_on_local_rows = is_moe_input_scattered_across_dp_ranks() if not ( self._takes_declared_boundaries - and parallel.attn_cp_size == 1 + and (parallel.attn_cp_size == 1 or _cp_on_declarations()) and not enable_moe_dense_fully_dp() and (modes.is_first_layer or modes.is_previous_layer_sparse is not None) ): return None + # A sum over rows gathered across CP stays with this layer: the next + # layer's input holds only this rank's chunk. + may_leave = not cp_active return decoder_layer_sides( - axis_sizes=_token_axis_sizes(), + axis_sizes=_token_axis_sizes(cp_active=cp_active), ffn_on_local_rows=modes.is_layer_sparse and moe_on_local_rows, previous_on_local_rows=( not modes.is_first_layer @@ -843,13 +862,14 @@ class LayerCommunicator: is_last_layer=modes.is_last_layer, attention_gathers_local_rows=_use_ag_after_qlora, ffn_group=SumGroup.MOE_OUTPUT if modes.is_layer_sparse else SumGroup.TP, - leaves_for_next_layer=self.allow_deferred_ffn_reduction, - leaves_for_reduce_scatter=self.allow_reduce_scatter, + leaves_for_next_layer=self.allow_deferred_ffn_reduction and may_leave, + leaves_for_reduce_scatter=self.allow_reduce_scatter and may_leave, # A MoE block leaves its sum to reduce_scatterv whenever that combine # applies (should_skip_post_experts_all_reduce). leaves_for_reduce_scatterv=( self.allow_reduce_scatter or modes.is_layer_sparse - ), + ) + and may_leave, ) def _select_boundaries_from_scatter_modes(self) -> "BoundarySteps": @@ -924,7 +944,9 @@ class LayerCommunicator: post_residual_addition=post_residual_addition, ) if captured_last_layer_outputs is not None: - gathered_last_layer_output = self._batch_steps().attention_input( + gathered_last_layer_output = self._batch_steps( + forward_batch + ).attention_input( hidden_states=residual, forward_batch=forward_batch, context=self._context, @@ -933,13 +955,15 @@ class LayerCommunicator: gathered_last_layer_output is residual # An accumulator that copies on append already holds a snapshot. and not getattr(captured_last_layer_outputs, "copies_on_append", False) - and not self._post_attn_residual_is_read_only(residual) + and not self._post_attn_residual_is_read_only(residual, forward_batch) ): gathered_last_layer_output = residual.clone() captured_last_layer_outputs.append(gathered_last_layer_output) return hidden_states, residual - def _post_attn_residual_is_read_only(self, residual: torch.Tensor) -> bool: + def _post_attn_residual_is_read_only( + self, residual: torch.Tensor, forward_batch: ForwardBatch + ) -> bool: """True if ``prepare_mlp``'s post-attention RMSNorm leaves ``residual`` untouched, so Eagle3 aux capture can keep its reference and skip the clone. @@ -950,7 +974,10 @@ class LayerCommunicator: the batch is not input-scattered. """ return ( - any(f.may_return_new_residual for f in self._batch_steps().fused) + any( + f.may_return_new_residual + for f in self._batch_steps(forward_batch).fused + ) and not get_attn_tp_context().input_scattered and apply_flashinfer_allreduce_fusion(residual.shape[0]) ) @@ -1012,7 +1039,7 @@ class LayerCommunicator: ) if get_forward().sp_active: hidden_states = layernorm_sp.sp_entry_scatter(hidden_states) - layer_input = self._batch_steps().layer_input + layer_input = self._batch_steps(forward_batch).layer_input if layer_input is not None: hidden_states, residual = layer_input( hidden_states, residual, self._context @@ -1038,9 +1065,10 @@ class LayerCommunicator: def _in_sp_region(self) -> bool: return self._sp_steps is not None and get_forward().sp_active - def _batch_steps(self) -> "BoundarySteps": + def _batch_steps(self, forward_batch: ForwardBatch) -> "BoundarySteps": """The steps this batch runs: an active LayerNorm SP region's, - input-scattered attention's, or the layer's ordinary ones.""" + input-scattered attention's, a CP extend's, or the layer's ordinary + ones.""" if self._in_sp_region(): return self._sp_steps if ( @@ -1048,11 +1076,13 @@ class LayerCommunicator: and get_attn_tp_context().input_scattered ): return self._input_scattered_steps + if self._cp_steps is not None and moe_cp_gathered_rows(forward_batch): + return self._cp_steps return self._steps def _finish_prepare_attn(self, hidden_states, residual, forward_batch): """Tail every prepare_attn path must run, or ``attn_inputs`` is unset.""" - hidden_states = self._batch_steps().attention_input( + hidden_states = self._batch_steps(forward_batch).attention_input( hidden_states=hidden_states, forward_batch=forward_batch, context=self._context, @@ -1195,7 +1225,7 @@ class LayerCommunicator: if cache is not None: self._context.cache = cache - return self._batch_steps().ffn_input( + return self._batch_steps(forward_batch).ffn_input( hidden_states, residual, forward_batch, @@ -1216,7 +1246,7 @@ class LayerCommunicator: residual: torch.Tensor, forward_batch: ForwardBatch, ): - steps = self._batch_steps() + steps = self._batch_steps(forward_batch) if steps.returns_over_dp: step = self._postprocess_dp_step(forward_batch) or _redistribute_output return _to_local_tokens(step, forward_batch, hidden_states), residual @@ -1229,10 +1259,12 @@ class LayerCommunicator: is_layer_sparse=self.layer_scatter_modes.is_layer_sparse, ) - def _local_token_move_can_go_to_next_layer(self) -> bool: + def _local_token_move_can_go_to_next_layer( + self, forward_batch: ForwardBatch + ) -> bool: """Whether the next layer's input can run this layer's move of its FFN output back to this rank's tokens: the base postprocess scatter.""" - return self._batch_steps().returns_over_dp + return self._batch_steps(forward_batch).returns_over_dp def _postprocess_dp_step( self, forward_batch: ForwardBatch @@ -1240,7 +1272,7 @@ class LayerCommunicator: """The reduce-scatter that brings this layer's FFN output back to this rank's tokens under attention DP; None when the base postprocess would only scatter, or does not move tokens.""" - steps = self._batch_steps() + steps = self._batch_steps(forward_batch) if not steps.returns_over_dp: return None return _reduce_and_redistribute_output_step( @@ -1255,7 +1287,7 @@ class LayerCommunicator: """Whether the FFN leaves its sum out because a reduce-scatter completes it: the attention-DP one ``dp_step`` names, or the CP / input-scattered one.""" - if not self._batch_steps().ffn_output.leaves_for_reduce_scatter: + if not self._batch_steps(forward_batch).ffn_output.leaves_for_reduce_scatter: return False if dp_step is not None: return True @@ -1266,10 +1298,10 @@ class LayerCommunicator: return True return get_attn_tp_context().input_scattered and not self.is_last_layer - def ffn_reduction_group(self) -> GroupCoordinator: + def ffn_reduction_group(self, forward_batch: ForwardBatch) -> GroupCoordinator: """The group this layer's FFN output owes its sum over: the MoE output's group on a sparse layer, the TP group a dense MLP reduces over.""" - return _sum_group(self._batch_steps().ffn_output.group) + return _sum_group(self._batch_steps(forward_batch).ffn_output.group) def _select_ffn_completion(self, forward_batch: ForwardBatch) -> "FfnCompletion": """Decide once, before the FFN runs, what it skips and what completes its @@ -1283,7 +1315,7 @@ class LayerCommunicator: forward_batch=forward_batch, dp_step=dp_step, ) - steps = self._batch_steps() + steps = self._batch_steps(forward_batch) if not steps.ffn_output.leaves_for_next_layer: return FfnCompletion( defer_moe_finalize=False, @@ -1297,7 +1329,7 @@ class LayerCommunicator: forward_batch, mlp_reduce_scatter=mlp_reduce_scatter, dp_step=dp_step ) if fuse_mlp_allreduce: - group = self.ffn_reduction_group() + group = self.ffn_reduction_group(forward_batch) if steps.returns_over_dp: # Under attention DP the next layer also brings the sum back to # this rank's tokens. @@ -1313,7 +1345,7 @@ class LayerCommunicator: elif ( dp_step is not None and not self.is_last_layer - and self._local_token_move_can_go_to_next_layer() + and self._local_token_move_can_go_to_next_layer(forward_batch) ): complete = partial( _leave_to_next_layer, @@ -1343,7 +1375,7 @@ class LayerCommunicator: ) -> Tuple[torch.Tensor, torch.Tensor]: """This layer's postprocess, run with the attention-DP step the FFN exit chose.""" - if self._local_token_move_can_go_to_next_layer(): + if self._local_token_move_can_go_to_next_layer(forward_batch): return ( _to_local_tokens( dp_step or _redistribute_output, forward_batch, hidden_states @@ -1381,10 +1413,13 @@ class LayerCommunicator: """Whether the MoE may hand an unfinalized output to the next layer.""" return False - def _ffn_sum_can_move_to_next_layer(self) -> bool: + def _ffn_sum_can_move_to_next_layer(self, forward_batch: ForwardBatch) -> bool: # Under the MoE-CP all-gather the fusion path would skip postprocess_layer # and its MoE-CP scatter, leaving hidden_states longer than the residual. - if is_enable_moe_cp_allgather() or not self._batch_steps().ffn_sum_is_movable: + if ( + is_enable_moe_cp_allgather() + or not self._batch_steps(forward_batch).ffn_sum_is_movable + ): return False # The fused residual+LN reduces over a single group. Hybrid EP+TP spans @@ -1412,7 +1447,7 @@ class LayerCommunicator: def should_fuse_mlp_allreduce_with_next_layer( self, forward_batch: ForwardBatch ) -> bool: - if not self._ffn_sum_can_move_to_next_layer(): + if not self._ffn_sum_can_move_to_next_layer(forward_batch): return False batch_size = _batch_size(forward_batch) return ( @@ -1446,14 +1481,17 @@ class LayerCommunicator: return ( self._context.tp_size > 1 and not self.is_last_layer - and self._ffn_sum_can_move_to_next_layer() + and self._ffn_sum_can_move_to_next_layer(forward_batch) and _unfused_completion_matches_the_ffn(forward_batch) and not mlp_reduce_scatter # Under attention DP the next layer must also run postprocess's # scatter back to this rank's tokens, and nothing more. and ( not is_dp_attention_enabled() - or (self._local_token_move_can_go_to_next_layer() and dp_step is None) + or ( + self._local_token_move_can_go_to_next_layer(forward_batch) + and dp_step is None + ) ) ) @@ -1689,14 +1727,28 @@ def _reduce_and_redistribute_output_to_attn_tp_shards( return local_hidden_states +def moe_cp_gathered_rows(forward_batch: ForwardBatch) -> Optional[List[int]]: + """The real rows each MoE-CP rank contributes when this batch's FFN input is + gathered across the MoE-CP group, or None when it is not: only a context + parallel extend with CP metadata gathers, and only when the group has more + than one rank. The batch is read first, so a batch that is not a CP extend + never reads the group.""" + if ( + forward_batch.forward_mode.is_context_parallel_extend() + and forward_batch.attn_cp_metadata is not None + and get_moe_cp_size() > 1 + ): + return forward_batch.attn_cp_metadata.per_rank_actual_token + return None + + def _redistribute_input_to_moe_cp( - hidden_states: torch.Tensor, forward_batch: ForwardBatch, moe_cp_size: int + hidden_states: torch.Tensor, rows: List[int], moe_cp_size: int ) -> torch.Tensor: # Zigzag split can produce unequal token counts across CP ranks # (when seq_len % (cp_size * 2) != 0). NCCL allgather requires # equal input sizes, so pad to the max per-rank token count. - per_rank_tokens = forward_batch.attn_cp_metadata.per_rank_actual_token - max_tokens = max(per_rank_tokens) + max_tokens = max(rows) pad_size = max_tokens - hidden_states.shape[0] if pad_size > 0: hidden_states = torch.nn.functional.pad(hidden_states, [0, 0, 0, pad_size]) @@ -1853,15 +1905,30 @@ def _sum_group(group: SumGroup) -> GroupCoordinator: return post_experts_reduction_group() -def _token_axis_sizes() -> Dict[TokenAxis, int]: +def _token_axis_sizes(*, cp_active: bool = False) -> Dict[TokenAxis, int]: + """The token axes' sizes for a batch: attention CP shards tokens only on + a CP extend (``cp_active``); otherwise every CP rank holds them all.""" parallel = get_parallel() return { TokenAxis.ATTN_DP: parallel.attn_dp_size, - TokenAxis.ATTN_CP: parallel.attn_cp_size, + TokenAxis.ATTN_CP: parallel.attn_cp_size if cp_active else 1, TokenAxis.ATTN_TP_SCATTER: parallel.attn_tp_size, } +def _cp_on_declarations() -> bool: + """Whether attention CP is one the declarations cover: a prefill CP that + shards tokens, without attention DP, whose FFN input gathers over a MoE-CP + group that is the whole CP group. DSA and MLA CP pick their own steps.""" + parallel = get_parallel() + return ( + _generic_prefill_cp_shards_tokens() + and not (is_dsa_enable_prefill_cp() or is_mla_cp_enabled()) + and parallel.attn_dp_size == 1 + and parallel.moe_dp_size == 1 + ) + + class BoundarySteps(msgspec.Struct, frozen=True): """The steps a batch runs at a layer's boundaries: into the attention, from the attention output to the FFN input, and the FFN output on to the @@ -1999,6 +2066,19 @@ def _select_ffn_input( partial(_mlp_input_scatter, scatters_residual=residual != residual_to), (), ) + if gathered == {TokenAxis.ATTN_CP}: + # Each CP rank completes its own chunk, then the chunks are gathered + # over the MoE-CP group, each padded to the longest. + on_chunk, fused = _select_ffn_input( + produced, + residual=residual, + residual_to=residual_to, + need=StageInput(produced.layout), + force_layernorm_before_gather=force_layernorm_before_gather, + fusions=fusions, + residual_joins_sum=residual_joins_sum, + ) + return partial(_mlp_input_gather_moe_cp, gather=on_chunk), fused if ( residual_to != produced.layout or gathered not in (frozenset(), {TokenAxis.ATTN_DP}) @@ -2055,11 +2135,12 @@ def _select_ffn_output_move( return False, CommunicateSummableTensorPairFn._gather raise NotImplementedError(f"{produced=} {residual=} {to=}") returned = residual.sharded - produced.layout.sharded - if ( - to != residual - or not produced.layout.sharded <= residual.sharded - or returned != {TokenAxis.ATTN_DP} - ): + if to != residual or not produced.layout.sharded <= residual.sharded: + raise NotImplementedError(f"{produced=} {residual=} {to=}") + if returned == {TokenAxis.ATTN_CP}: + # This rank's chunk of the rows gathered over CP; no collective. + return False, CommunicateSummableTensorPairFn._scatter_hidden_states_moe + if returned != {TokenAxis.ATTN_DP}: raise NotImplementedError(f"{produced=} {residual=} {to=}") return True, None @@ -2240,16 +2321,10 @@ def _mlp_input_gather_moe_cp( hidden_states, residual, forward_batch, layernorm, context ) - # Only active during prefill (context-parallel extend); decode keeps existing path. - moe_cp_size = get_moe_cp_size() - if ( - moe_cp_size > 1 - and hidden_states.shape[0] > 0 - and forward_batch.forward_mode.is_context_parallel_extend() - and forward_batch.attn_cp_metadata is not None - ): + rows = moe_cp_gathered_rows(forward_batch) + if rows is not None and hidden_states.shape[0] > 0: hidden_states = _redistribute_input_to_moe_cp( - hidden_states, forward_batch, moe_cp_size + hidden_states, rows, get_moe_cp_size() ) return hidden_states, residual @@ -2291,14 +2366,13 @@ def _redistribute_output( def _redistribute_output_from_moe_cp( - hidden_states: torch.Tensor, forward_batch: ForwardBatch + hidden_states: torch.Tensor, rows: List[int] ) -> torch.Tensor: moe_cp_rank = get_moe_cp_rank() # The allgather was padded to max_tokens_per_rank (equal chunks). # Extract this rank's actual (non-padded) tokens from its chunk. - per_rank_tokens = forward_batch.attn_cp_metadata.per_rank_actual_token - max_tokens_per_rank = max(per_rank_tokens) - actual_local_tokens = per_rank_tokens[moe_cp_rank] + max_tokens_per_rank = max(rows) + actual_local_tokens = rows[moe_cp_rank] return hidden_states.narrow( 0, moe_cp_rank * max_tokens_per_rank, actual_local_tokens ).contiguous() @@ -2459,15 +2533,9 @@ class CommunicateSummableTensorPairFn: # Only scatter back during prefill; decode was never allgathered so no-op. # Safe w.r.t. empty tensors: same reasoning as _mlp_input_gather_moe_cp # — CP extend always has non-zero tokens per rank, and decode skips this path. - moe_cp_size = get_moe_cp_size() - if ( - moe_cp_size > 1 - and forward_batch.forward_mode.is_context_parallel_extend() - and forward_batch.attn_cp_metadata is not None - ): - hidden_states = _redistribute_output_from_moe_cp( - hidden_states, forward_batch - ) + rows = moe_cp_gathered_rows(forward_batch) + if rows is not None: + hidden_states = _redistribute_output_from_moe_cp(hidden_states, rows) if context.attn_dp_size > 1: hidden_states = _to_local_tokens( diff --git a/python/sglang/srt/model_executor/forward_batch_info.py b/python/sglang/srt/model_executor/forward_batch_info.py index 5ecbb9232e..5f06a44696 100644 --- a/python/sglang/srt/model_executor/forward_batch_info.py +++ b/python/sglang/srt/model_executor/forward_batch_info.py @@ -1230,7 +1230,11 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin): def moe_num_token_non_padded(self) -> Optional[torch.Tensor]: """Bound for masking a sparse MoE's padded rows, or None when the MoE input is a gathered buffer whose real rows are not a prefix of it.""" - from sglang.srt.layers.communicator import ScatterMode, sparse_mlp_scatter_mode + from sglang.srt.layers.communicator import ( + ScatterMode, + moe_cp_gathered_rows, + sparse_mlp_scatter_mode, + ) if self.num_token_non_padded is None: return None @@ -1239,7 +1243,7 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin): if mode == ScatterMode.SCATTERED: # a2a dispatch, FP4 all-gather and dwdp all route the local shard. return self.num_token_non_padded - if mode == ScatterMode.MOE_FULL and self._moe_input_gathered_across_moe_cp(): + if mode == ScatterMode.MOE_FULL and moe_cp_gathered_rows(self) is not None: return None # DSA / MLA CP take the FULL mode but all-gather across CP on a prefill, # which leaves the real rows zigzag-permuted rather than in a prefix. @@ -1255,16 +1259,6 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin): # None under cuda-graph replay: its static batch carries no GLOBAL count. return self.global_num_token_non_padded - def _moe_input_gathered_across_moe_cp(self) -> bool: - from sglang.srt.layers.dp_attention import get_moe_cp_size - - # Mirrors the communicator's MOE_FULL gather guard: decode stays FULL. - return ( - self.forward_mode.is_context_parallel_extend() - and self.attn_cp_metadata is not None - and get_moe_cp_size() > 1 - ) - def _moe_input_gathered_across_cp(self) -> bool: from sglang.srt.layers.attention.dsa.utils import dsa_use_prefill_cp from sglang.srt.layers.cp.utils import is_mla_cp_active diff --git a/python/sglang/srt/models/nemotron_h.py b/python/sglang/srt/models/nemotron_h.py index 837ebce7ad..983f393a4c 100644 --- a/python/sglang/srt/models/nemotron_h.py +++ b/python/sglang/srt/models/nemotron_h.py @@ -428,7 +428,8 @@ class NemotronHMLPLikeDecoderLayer(nn.Module): hidden_states = self.mixer.forward(hidden_states) if fuse_mlp_allreduce: hidden_states = UnreducedOutput( - hidden_states, group=self.layer_communicator.ffn_reduction_group() + hidden_states, + group=self.layer_communicator.ffn_reduction_group(forward_batch), ) else: hidden_states, residual = self.layer_communicator.postprocess_layer( diff --git a/test/registered/unit/layers/test_communicator_ffn_exit.py b/test/registered/unit/layers/test_communicator_ffn_exit.py index dd1a0add9d..5e3a6553d5 100644 --- a/test/registered/unit/layers/test_communicator_ffn_exit.py +++ b/test/registered/unit/layers/test_communicator_ffn_exit.py @@ -91,6 +91,7 @@ def make_communicator( communicator.is_last_layer = False communicator._sp_steps = None communicator._input_scattered_steps = None + communicator._cp_steps = None communicator._postprocess_dp_step = MagicMock(return_value=reduce_scatter_step) communicator.ffn_reduction_group = MagicMock(return_value=group or make_group()) communicator.postprocess_layer = MagicMock( @@ -324,6 +325,7 @@ class TestSelectFfnCompletion(CustomTestCase): communicator.is_last_layer = is_last_layer communicator._sp_steps = sp_region_steps() if sp_region else None communicator._input_scattered_steps = None + communicator._cp_steps = None communicator._steps = ordinary_steps( StageOutput( Layout(frozenset()), @@ -340,7 +342,7 @@ class TestSelectFfnCompletion(CustomTestCase): ) communicator._complete_ffn_output_now = lambda h, r, **_: ("now", r) self.group = make_group() - communicator.ffn_reduction_group = lambda: self.group + communicator.ffn_reduction_group = lambda forward_batch: self.group return communicator def left(self, communicator, step, forward_batch=None): diff --git a/test/registered/unit/layers/test_declared_decoder_boundary.py b/test/registered/unit/layers/test_declared_decoder_boundary.py index 6b05969745..cfc2bb828d 100644 --- a/test/registered/unit/layers/test_declared_decoder_boundary.py +++ b/test/registered/unit/layers/test_declared_decoder_boundary.py @@ -75,11 +75,19 @@ def planning(parallel, *, sp=False, a2a=False): process-wide parallel state. ``parallel`` may be a callable, for a per-thread parallel state.""" get_parallel = parallel if callable(parallel) else (lambda: parallel) + + def moe_cp_gathers(): + return get_parallel().attn_cp_size > get_parallel().moe_dp_size + with ( patch.object(comm, "get_parallel", get_parallel), patch.object(comm, "is_dsa_enable_prefill_cp", lambda: False), patch.object(comm, "is_mla_cp_enabled", lambda: False), - patch.object(comm, "get_moe_cp_size", lambda: 1), + patch.object( + comm, + "get_moe_cp_size", + lambda: get_parallel().attn_cp_size if moe_cp_gathers() else 1, + ), patch.object(comm.layernorm_sp, "layernorm_sp_enabled", lambda: sp), patch.object( comm, "get_spec", lambda: SimpleNamespace(speculative_algorithm=None) @@ -93,7 +101,7 @@ def planning(parallel, *, sp=False, a2a=False): patch.object( comm, "should_use_flashinfer_cutlass_moe_fp4_allgather", lambda: False ), - patch.object(comm, "is_enable_moe_cp_allgather", lambda: False), + patch.object(comm, "is_enable_moe_cp_allgather", moe_cp_gathers), patch.object(comm, "get_lora", lambda: SimpleNamespace(enable_lora=False)), ): yield @@ -700,7 +708,7 @@ class TestTheSequenceParallelRegion(CustomTestCase): patch.object(comm, "is_dp_attention_enabled", lambda: False), ): self.assertIs( - communicator._ffn_sum_can_move_to_next_layer(), not active + communicator._ffn_sum_can_move_to_next_layer(None), not active ) def test_without_sp_there_is_no_region(self): @@ -818,7 +826,7 @@ class TestInputScatteredAttention(CustomTestCase): ), ): self.assertIs( - communicator._batch_steps(), + communicator._batch_steps(None), communicator._input_scattered_steps if scattered else communicator._steps, @@ -901,7 +909,7 @@ class TestOneRepresentation(CustomTestCase): self.assertIsInstance(communicator._steps, comm.BoundarySteps) for attribute in self.SEPARATE: self.assertFalse(hasattr(communicator, attribute), attribute) - self.assertIs(communicator._batch_steps(), communicator._steps) + self.assertIs(communicator._batch_steps(None), communicator._steps) # Under attention DP both bring the FFN output back to this # rank's tokens with the step the FFN exit chooses. self.assertTrue(communicator._steps.returns_over_dp) @@ -928,6 +936,101 @@ class TestOneRepresentation(CustomTestCase): ) +class TestPrefillCP(CustomTestCase): + """A prefill CP shards a batch's tokens over attention CP only on a CP + extend; the layer runs its CP steps on those batches and its ordinary ones + on every other.""" + + def cp_parallel(self, **overrides): + return parallel_of( + attn_dp=1, attn_tp=2, attn_cp=2, enable_prefill_cp=True, **overrides + ) + + def test_a_cp_extend_gathers_over_cp_and_takes_its_chunk_back(self): + communicator = build(layer_facts(1, 3), self.cp_parallel()) + cp = communicator._cp_steps + self.assertIs(cp.ffn_input.func, comm._mlp_input_gather_moe_cp) + # Each rank completes its own chunk before the gather. + self.assertIs(cp.ffn_input.keywords["gather"].func, comm._mlp_input_without_dp) + self.assertIs( + cp.ffn_output_move, + comm.CommunicateSummableTensorPairFn._scatter_hidden_states_moe, + ) + self.assertFalse(cp.returns_over_dp) + self.assertFalse(cp.ffn_output.leaves_for_next_layer) + self.assertFalse(cp.ffn_output.leaves_for_reduce_scatter) + self.assertIs(communicator._steps.ffn_input.func, comm._mlp_input_without_dp) + self.assertIs( + communicator._steps.ffn_output_move, + comm.CommunicateSummableTensorPairFn._trivial, + ) + + def test_the_fused_kernels_run_on_each_chunk(self): + with planning(self.cp_parallel()): + communicator = LayerCommunicator( + layer_scatter_modes=layer_facts(1, 3), + input_layernorm=FusableNorm(), + post_attention_layernorm=FusableNorm(), + ) + chunk = communicator._cp_steps.ffn_input.keywords["gather"] + self.assertEqual( + chunk.keywords["fusions"], + (communicator._mlp_input_reduce_output_and_update_and_read_residual,), + ) + + def test_which_cp_the_declarations_cover(self): + for name, parallel, declared in ( + ("prefill CP", self.cp_parallel(), True), + ( + "CP without prefill CP", + parallel_of(attn_dp=1, attn_tp=2, attn_cp=2), + False, + ), + ( + "CP under attention DP", + parallel_of(attn_dp=2, attn_tp=1, attn_cp=2, enable_prefill_cp=True), + False, + ), + ( + "a MoE-CP group narrower than CP", + parallel_of( + attn_dp=1, + attn_tp=1, + attn_cp=4, + enable_prefill_cp=True, + moe_dp_size=2, + ), + False, + ), + ): + with self.subTest(name): + modes = planned_modes( + 1, 3, sparse=False, previous_sparse=False, parallel=parallel + ) + communicator = build(modes, parallel) + self.assertIs(communicator._cp_steps is not None, declared) + + def test_a_batch_runs_them_only_on_a_cp_extend(self): + communicator = build(layer_facts(1, 3), self.cp_parallel()) + for rows, expected in ( + (None, communicator._steps), + ([2, 1], communicator._cp_steps), + ): + with ( + self.subTest(gathered_rows=rows), + patch.object(comm, "moe_cp_gathered_rows", lambda fb: rows), + patch.object( + comm, "get_forward", lambda: SimpleNamespace(sp_active=False) + ), + patch.object( + comm, + "get_attn_tp_context", + lambda: SimpleNamespace(input_scattered=False), + ), + ): + self.assertIs(communicator._batch_steps(None), expected) + + # --------------------------------------------------------------------------- # Two consecutive layers, rank by rank. diff --git a/test/registered/unit/layers/test_layer_communicator_fusion_gate.py b/test/registered/unit/layers/test_layer_communicator_fusion_gate.py index 6975e0cf76..323a895b22 100644 --- a/test/registered/unit/layers/test_layer_communicator_fusion_gate.py +++ b/test/registered/unit/layers/test_layer_communicator_fusion_gate.py @@ -66,6 +66,7 @@ def _fake_communicator(mlp_mode=ScatterMode.TP_ATTN_FULL): ) communicator._sp_steps = None communicator._input_scattered_steps = None + communicator._cp_steps = None communicator.is_last_layer = False communicator._context = types.SimpleNamespace(tp_size=4) return communicator @@ -366,6 +367,7 @@ class TestDeferFfnReduction(CustomTestCase): communicator.is_last_layer = is_last_layer communicator._sp_steps = sp_region_steps() if sp_active else None communicator._input_scattered_steps = None + communicator._cp_steps = None communicator._steps = _steps( ffn_output=StageOutput( Layout(frozenset()), diff --git a/test/registered/unit/layers/test_layernorm_sp.py b/test/registered/unit/layers/test_layernorm_sp.py index 50fb90394b..f8e73e6a75 100644 --- a/test/registered/unit/layers/test_layernorm_sp.py +++ b/test/registered/unit/layers/test_layernorm_sp.py @@ -170,6 +170,7 @@ class TestSpRegionSteps(CustomTestCase): c = comm.LayerCommunicator.__new__(comm.LayerCommunicator) c._sp_steps = sp_region_steps() c._input_scattered_steps = None + c._cp_steps = None c.layer_scatter_modes = SimpleNamespace( is_first_layer=first_layer, is_layer_sparse=False ) diff --git a/test/registered/unit/layers/test_postprocess_reduce_scatterv.py b/test/registered/unit/layers/test_postprocess_reduce_scatterv.py index 0856aa597b..b5b51588d8 100644 --- a/test/registered/unit/layers/test_postprocess_reduce_scatterv.py +++ b/test/registered/unit/layers/test_postprocess_reduce_scatterv.py @@ -154,6 +154,7 @@ class TestPostprocessReduceScatterv(CustomTestCase): communicator = LayerCommunicator.__new__(LayerCommunicator) communicator._sp_steps = None communicator._input_scattered_steps = None + communicator._cp_steps = None communicator._context = None communicator.allow_reduce_scatter = False communicator.layer_scatter_modes = types.SimpleNamespace(is_layer_sparse=True) @@ -171,6 +172,7 @@ class TestPostprocessReduceScatterv(CustomTestCase): communicator = LayerCommunicator.__new__(LayerCommunicator) communicator._sp_steps = None communicator._input_scattered_steps = None + communicator._cp_steps = None communicator._steps = steps(ffn_output_move=None) communicator._postprocess_dp_step = lambda forward_batch: step with patch.object( diff --git a/test/registered/unit/layers/test_prepare_attn_steps.py b/test/registered/unit/layers/test_prepare_attn_steps.py index dbf9c4c95e..04705d3c32 100644 --- a/test/registered/unit/layers/test_prepare_attn_steps.py +++ b/test/registered/unit/layers/test_prepare_attn_steps.py @@ -44,6 +44,7 @@ def communicator(norm): c.input_layernorm = norm c._sp_steps = None c._input_scattered_steps = None + c._cp_steps = None c.qkv_latent_func = None c._context = None c.enable_fused_ar_quant = False diff --git a/test/registered/unit/lora/test_triton_dp_attention_unit.py b/test/registered/unit/lora/test_triton_dp_attention_unit.py index f352e4edd7..5dc8c6f0c3 100644 --- a/test/registered/unit/lora/test_triton_dp_attention_unit.py +++ b/test/registered/unit/lora/test_triton_dp_attention_unit.py @@ -224,6 +224,7 @@ def test_communicator_publishes_layout_at_each_transition( communicator._context = SimpleNamespace() communicator._sp_steps = None communicator._input_scattered_steps = None + communicator._cp_steps = None communicator.post_attention_layernorm = None communicator.input_layernorm = lambda x: x communicator.qkv_latent_func = None diff --git a/test/registered/unit/model_executor/test_num_token_non_padded_localization.py b/test/registered/unit/model_executor/test_num_token_non_padded_localization.py index 0d3ee96d86..3262fe7100 100644 --- a/test/registered/unit/model_executor/test_num_token_non_padded_localization.py +++ b/test/registered/unit/model_executor/test_num_token_non_padded_localization.py @@ -22,6 +22,7 @@ from sglang.test.ci.ci_register import register_cpu_ci register_cpu_ci(est_time=10, suite="base-a-test-cpu") import unittest +from types import SimpleNamespace from unittest.mock import patch import torch @@ -163,14 +164,15 @@ class TestMoeNumTokenNonPaddedTable(CustomTestCase): self.assertIsNone(_value(_forward_batch(sharded=sharded))) def test_moe_full_gathers_only_on_a_context_parallel_extend(self): + cp_metadata = SimpleNamespace(per_rank_actual_token=[2, 2]) """A moe-cp config keeps its bound on every forward but the CP extend, which is the only one that all-gathers over the MoE-CP group.""" # get_moe_cp_size reads a live process group, so it is stubbed rather # than reached through a topology published without distributed init. for label, forward_mode, metadata, expected in [ - ("cp_extend", ForwardMode.EXTEND, object(), None), + ("cp_extend", ForwardMode.EXTEND, cp_metadata, None), ("extend_without_cp_metadata", ForwardMode.EXTEND, None, LOCAL), - ("decode", ForwardMode.DECODE, object(), LOCAL), + ("decode", ForwardMode.DECODE, cp_metadata, LOCAL), ]: with ( self.subTest(case=label), @@ -178,7 +180,7 @@ class TestMoeNumTokenNonPaddedTable(CustomTestCase): patch.object( comm, "sparse_mlp_scatter_mode", return_value=ScatterMode.MOE_FULL ), - patch("sglang.srt.layers.dp_attention.get_moe_cp_size", return_value=2), + patch.object(comm, "get_moe_cp_size", return_value=2), patch( "sglang.srt.layers.attention.dsa.utils.dsa_use_prefill_cp", return_value=False, diff --git a/test/registered/unit/models/test_aux_capture_deferred_allreduce.py b/test/registered/unit/models/test_aux_capture_deferred_allreduce.py index 3c6e2e737d..b9d5f40579 100644 --- a/test/registered/unit/models/test_aux_capture_deferred_allreduce.py +++ b/test/registered/unit/models/test_aux_capture_deferred_allreduce.py @@ -64,7 +64,8 @@ class DeferringLayer(nn.Module): self.layer_communicator.is_last_layer = False self.layer_communicator._sp_steps = None self.layer_communicator._input_scattered_steps = None - self.layer_communicator.ffn_reduction_group = lambda: GROUP + self.layer_communicator._cp_steps = None + self.layer_communicator.ffn_reduction_group = lambda forward_batch: GROUP self.layer_communicator._ffn_leaves_sum_to_reduce_scatter = ( lambda batch, dp_step: False )