diff --git a/python/sglang/srt/hardware_backend/npu/modules/deepseek_v2_attention_mla_npu.py b/python/sglang/srt/hardware_backend/npu/modules/deepseek_v2_attention_mla_npu.py index 3755509475..37701d22ee 100644 --- a/python/sglang/srt/hardware_backend/npu/modules/deepseek_v2_attention_mla_npu.py +++ b/python/sglang/srt/hardware_backend/npu/modules/deepseek_v2_attention_mla_npu.py @@ -15,7 +15,7 @@ from sglang.srt.layers.attention.dsa.dsa_npu_indexer import scattered_to_tp_attn from sglang.srt.layers.attention.dsa.utils import ( dsa_use_prefill_cp, ) -from sglang.srt.layers.communicator import ScatterMode, get_attn_tp_context +from sglang.srt.layers.communicator import get_attn_tp_context from sglang.srt.model_executor.forward_context import get_token_to_kv_pool from sglang.srt.runtime_context import get_disagg @@ -34,7 +34,7 @@ def forward_mha_prepare_npu( hidden_states: torch.Tensor, forward_batch: "ForwardBatch", zero_allocator: "BumpAllocator", - layer_scatter_modes, + input_on_attention_tp_slices: bool, ): if m.q_lora_rank is not None: q, latent_cache = ( @@ -62,11 +62,7 @@ def forward_mha_prepare_npu( else: q = m.q_a_layernorm(q) - if ( - _use_ag_after_qlora - and layer_scatter_modes.layer_input_mode == ScatterMode.SCATTERED - and layer_scatter_modes.attn_mode == ScatterMode.TP_ATTN_FULL - ): + if _use_ag_after_qlora and input_on_attention_tp_slices: q = scattered_to_tp_attn_full(q, forward_batch) latent_cache = scattered_to_tp_attn_full(latent_cache, forward_batch) q = m.q_b_proj(q)[0].view(-1, m.num_local_heads, m.qk_head_dim) @@ -160,7 +156,7 @@ def forward_mla_prepare_npu( hidden_states: torch.Tensor, forward_batch: "ForwardBatch", zero_allocator: "BumpAllocator", - layer_scatter_modes, + input_on_attention_tp_slices: bool, ): if is_mla_preprocess_enabled(): if not hasattr(m, "mla_preprocess"): @@ -193,11 +189,7 @@ def forward_mla_prepare_npu( q_lora = None if m.q_lora_rank is not None: qkv_latent = get_attn_tp_context().fetch_qkv_latent() - if ( - _use_ag_after_qlora - and layer_scatter_modes.layer_input_mode == ScatterMode.SCATTERED - and layer_scatter_modes.attn_mode == ScatterMode.TP_ATTN_FULL - ): + if _use_ag_after_qlora and input_on_attention_tp_slices: q, latent_cache = qkv_latent.split( [m.q_lora_rank, m.kv_lora_rank + m.qk_rope_head_dim], dim=-1, @@ -369,7 +361,7 @@ def forward_dsa_prepare_npu( hidden_states: torch.Tensor, forward_batch: "ForwardBatch", zero_allocator: "BumpAllocator", - layer_scatter_modes, + input_on_attention_tp_slices: bool, prev_topk_indices: torch.Tensor = None, ): dynamic_scale = None @@ -403,11 +395,7 @@ def forward_dsa_prepare_npu( ) # overlap qk norm q = m.q_a_layernorm(q) - if ( - _use_ag_after_qlora - and layer_scatter_modes.layer_input_mode == ScatterMode.SCATTERED - and layer_scatter_modes.attn_mode == ScatterMode.TP_ATTN_FULL - ): + if _use_ag_after_qlora and input_on_attention_tp_slices: q = scattered_to_tp_attn_full(q, forward_batch) latent_cache = scattered_to_tp_attn_full(latent_cache, forward_batch) q_lora = q.clone() # required for topk_indices @@ -496,7 +484,7 @@ def forward_dsa_prepare_npu( positions, forward_batch, m.layer_id, - layer_scatter_modes, + input_on_attention_tp_slices, dynamic_scale, ) else: diff --git a/python/sglang/srt/layers/attention/dsa/dsa_npu_indexer.py b/python/sglang/srt/layers/attention/dsa/dsa_npu_indexer.py index a06c25e869..b607e72218 100644 --- a/python/sglang/srt/layers/attention/dsa/dsa_npu_indexer.py +++ b/python/sglang/srt/layers/attention/dsa/dsa_npu_indexer.py @@ -5,7 +5,6 @@ from functools import lru_cache import torch from sglang.srt.environ import envs -from sglang.srt.layers.communicator import ScatterMode from sglang.srt.layers.cp.utils import cp_gather_full_sequence_states from sglang.srt.layers.dp_attention import attn_tp_all_gather_into_tensor from sglang.srt.model_executor.forward_batch_info import ForwardBatch @@ -61,7 +60,7 @@ class DSANPUIndexerMixin: positions: torch.Tensor, forward_batch: ForwardBatch, layer_id: int, - layer_scatter_modes=None, + input_on_attention_tp_slices: bool = False, dynamic_scale: torch.Tensor = None, ) -> torch.Tensor: if get_attn_backend().forward_metadata.seq_lens_cpu_int is None: @@ -141,11 +140,7 @@ class DSANPUIndexerMixin: k_proj = self.wk(x)[0] # [b, s, 7168] @ [7168, 128] = [b, s, 128] k = self.k_norm(k_proj) - if ( - _use_ag_after_qlora - and layer_scatter_modes.layer_input_mode == ScatterMode.SCATTERED - and layer_scatter_modes.attn_mode == ScatterMode.TP_ATTN_FULL - ): + if _use_ag_after_qlora and input_on_attention_tp_slices: k = scattered_to_tp_attn_full(k, forward_batch) k_pe, k_nope = torch.split( k, @@ -289,11 +284,7 @@ class DSANPUIndexerMixin: torch.npu.current_stream().wait_event(q_rope_event) if envs.SGLANG_NPU_USE_MULTI_STREAM.get(): torch.npu.current_stream().wait_event(weights_event) - if ( - _use_ag_after_qlora - and layer_scatter_modes.layer_input_mode == ScatterMode.SCATTERED - and layer_scatter_modes.attn_mode == ScatterMode.TP_ATTN_FULL - ): + if _use_ag_after_qlora and input_on_attention_tp_slices: weights = scattered_to_tp_attn_full(weights, forward_batch) block_table = get_attn_backend().forward_metadata.block_tables if ( diff --git a/python/sglang/srt/layers/communicator.py b/python/sglang/srt/layers/communicator.py index 78f43ba607..fcf3cca238 100644 --- a/python/sglang/srt/layers/communicator.py +++ b/python/sglang/srt/layers/communicator.py @@ -15,7 +15,7 @@ import logging from contextlib import contextmanager from dataclasses import dataclass from enum import Enum, auto -from functools import cached_property, partial +from functools import partial from typing import Callable, Dict, List, Optional, Protocol, Tuple, Union import msgspec @@ -877,17 +877,13 @@ class LayerCommunicator: # The steps the layer's ordinary batches run. sides = self._declared_sides() self._declared = sides - self._steps = ( - self._steps_from_declarations( - sides, - fusions=self._select_mlp_input_fusions(), - force_layernorm_before_gather=force_layernorm_before_dp_gather, - ) - if sides is not None - else self._select_boundaries_from_scatter_modes() + self._steps = self._steps_from_declarations( + sides, + fusions=self._select_mlp_input_fusions(), + force_layernorm_before_gather=force_layernorm_before_dp_gather, ) # The steps a batch that shards its tokens over attention CP runs; None - # without CP or where the steps come from the scatter modes. + # when no batch does. self._cp_steps = ( self._steps_from_declarations( self._declared_sides(cp_active=True), @@ -895,14 +891,14 @@ class LayerCommunicator: force_layernorm_before_gather=force_layernorm_before_dp_gather, cp_moves=_cp_moves(), ) - if sides is not None and get_parallel().attn_cp_size > 1 + if _generic_prefill_cp_shards_tokens() else None ) # The steps a batch with input-scattered attention runs; None where it - # cannot run or the steps come from the scatter modes. + # cannot run. self._input_scattered_steps = ( self._steps_for_input_scattered(sides) - if sides is not None and self._input_can_be_scattered() + if self._input_can_be_scattered() else None ) @@ -963,47 +959,52 @@ class LayerCommunicator: runs its ordinary steps.""" if self._declared is not None: return self._declared.input_rows - if self.stage_edges is not None: - return self.stage_edges[0].residual - # Such a batch holds every token on each CP rank. - return scatter_mode_layouts( - attn_dp_size=self._context.attn_dp_size, - attn_cp_size=1, - attn_tp_size=self._context.attn_tp_size, - )[self.layer_scatter_modes.layer_input_mode] + return self.stage_edges[0].residual + + @property + def input_on_attention_tp_slices(self) -> bool: + """Whether the layer's input arrives on each attention-TP rank's slice + of its rows (after a layer on local rows), in a batch that runs its + ordinary steps.""" + return TokenAxis.ATTN_TP_SCATTER in self.input_rows.sharded def _input_can_be_scattered(self) -> bool: """Whether a batch may run this layer with input-scattered attention: - configured, on pure TP without an a2a backend or a dense MLP on every - rank. The rest of what ``AttnTpContext.init_context`` requires is only + configured, on TP without attention DP, a prefill CP, an a2a backend or + a dense MLP on every rank. The rest of what ``AttnTpContext.init_context`` requires is only known once the model is built.""" parallel = get_parallel() return ( parallel.enable_attn_tp_input_scattered and parallel.tp_size > 1 and parallel.attn_dp_size == 1 - and parallel.attn_cp_size == 1 + and not _generic_prefill_cp_shards_tokens() and get_moe_a2a_backend().is_none() and not enable_moe_dense_fully_dp() ) - def _declared_sides( - self, *, cp_active: bool = False - ) -> Optional[DecoderLayerSides]: + def _declared_sides(self, *, cp_active: bool = False) -> DecoderLayerSides: """The declarations this layer's boundaries are chosen from: an - attention and an FFN, in a layer whose previous layer is known, with - attention CP only for a prefill CP whose FFN gathers over the whole CP - group. None when the steps come from the scatter modes. ``cp_active`` - gives a batch that shards its tokens over CP; the others hold every - token on each CP rank. An active LayerNorm SP region and input-scattered - attention have their own.""" + attention and an FFN, in a layer whose previous layer is known. + ``cp_active`` gives a batch that shards its tokens over CP; the others + hold every token on each CP rank. An active LayerNorm SP region and + input-scattered attention have their own. Raises NotImplementedError + for the combinations the steps do not cover.""" modes = self.layer_scatter_modes parallel = get_parallel() + if not (modes.is_first_layer or modes.is_previous_layer_sparse is not None): + raise NotImplementedError( + "a layer built without the facts of the layer before it" + ) # A MoE dispatched per DP shard computes on this rank's local rows and # hands its layer's output on there; so does a dense MLP on every rank. moe_on_local_rows = is_moe_input_scattered_across_dp_ranks() dense_on_local_rows = enable_moe_dense_fully_dp() + # Some batch shards its tokens over attention CP. + cp_shards = _generic_prefill_cp_shards_tokens() + if parallel.attn_cp_size > 1 and modes.is_layer_sparse: + self._refuse_uncovered_cp_moe(moe_on_local_rows, cp_shards) # A MoE whose data-parallel groups are the CP ranks computes each CP # shard of a GQA prefill CP on its own ranks. ffn_on_cp_shards = ( @@ -1027,32 +1028,21 @@ class LayerCommunicator: and bool(next_sparse) ) - if not ( - (parallel.attn_cp_size == 1 or _cp_on_declarations()) - # A MoE dispatched per DP shard under attention DP and GQA prefill - # CP keeps the scatter-mode steps: it would dispatch each rank's CP - # shard of the tokens, which is not covered. - and not ( - parallel.attn_cp_size > 1 - and parallel.attn_dp_size > 1 - and modes.is_layer_sparse - and moe_on_local_rows - and not _gathers_over_attention_cp() - ) - and (modes.is_first_layer or modes.is_previous_layer_sparse is not None) - ): - return None - if parallel.attn_cp_size > 1 and _cp_moves().reduce_scatter is not None: + if cp_shards and _cp_moves().reduce_scatter is not None: # A CP extend's FFN may leave its sum to the reduce-scatter that # takes each rank's shard back (DSA and MLA CP). may_leave = not cp_active may_leave_to_reduce_scatter = True - else: + elif cp_shards or parallel.attn_dp_size > 1: # Otherwise under CP an FFN over every CP shard completes its own sum; # one on its own CP shard hands on its rows as without CP. may_leave = may_leave_to_reduce_scatter = ( parallel.attn_cp_size == 1 or ffn_on_cp_shards ) + else: + # No batch shards its tokens and there is no attention DP: as + # without CP. + may_leave = may_leave_to_reduce_scatter = True return decoder_layer_sides( axis_sizes=token_axis_sizes(cp_active=cp_active), ffn_on_local_rows=on_local_rows(modes.is_layer_sparse), @@ -1081,52 +1071,32 @@ class LayerCommunicator: and may_leave, ) - def _select_boundaries_from_scatter_modes(self) -> "BoundarySteps": - """The layer's steps chosen from its scatter modes, for the layers the - declarations do not cover yet.""" - attention_input, postprocess = self._post_init_communicate() - ffn_input, fused = self._select_mlp_input() - modes = self.layer_scatter_modes - ops = self._residual_ops - ffn_rows = self._context.layouts[modes.mlp_mode] - ffn_output = StageOutput( - ffn_rows, - group=SumGroup.MOE_OUTPUT if modes.is_layer_sparse else SumGroup.TP, - leaves_for_next_layer=self.allow_deferred_ffn_reduction, - leaves_for_reduce_scatter=self.allow_reduce_scatter, - # A MoE block leaves its sum to reduce_scatterv whenever it applies - # (should_skip_post_experts_all_reduce); a dense MLP does only under - # the published mlp_reduce_scatter. - leaves_for_reduce_scatterv=( - self.allow_reduce_scatter or modes.is_layer_sparse - ), - ) - return BoundarySteps( - attention_input=attention_input, - ffn_input=ffn_input, - ffn_input_rows=ffn_rows, - ffn_output=ffn_output, - # Under attention DP the base postprocess scatters the FFN output - # back to this rank's tokens, which the next layer's input can run - # instead; the MHC and DSA-CP postprocess do more. - ffn_output_move=( - None - if postprocess is CommunicateSummableTensorPairFn._scatter_hidden_states - else partial(postprocess, residual_ops=ops) - ), - # Not when the way back is the MoE-CP scatter, nor when a SCATTERED - # FFN computes whole tokens with nothing left to sum. - ffn_sum_is_movable=modes.mlp_mode - not in (ScatterMode.MOE_FULL, ScatterMode.SCATTERED), - fused=fused, - attention_prepare=partial( - _attention_input_step, - layer_input=_complete_scattered_input, - fusions=self._attn_input_fusions, - enters_stack=modes.is_first_layer, - residual_ops=ops, - ), - ) + @staticmethod + def _refuse_uncovered_cp_moe(moe_on_local_rows: bool, cp_shards: bool) -> None: + """A MoE layer under attention CP whose tokens the steps cannot bring + to it: under a prefill CP, one dispatched per DP shard under attention DP + and GQA CP, and one on the TP group whose data-parallel groups are the CP + ranks under DSA or MLA CP; and under attention DP, one on the TP group + whose data-parallel groups are the CP ranks.""" + parallel = get_parallel() + gqa = not _gathers_over_attention_cp() + if moe_on_local_rows: + if cp_shards and gqa and parallel.attn_dp_size > 1: + raise NotImplementedError( + "a MoE dispatched per DP shard under attention DP and GQA " + "prefill CP" + ) + elif parallel.moe_dp_size == parallel.attn_cp_size: + if cp_shards and not gqa: + raise NotImplementedError( + "a MoE on the TP group with moe_dp_size == attn_cp_size under " + "DSA or MLA prefill CP" + ) + if parallel.attn_dp_size > 1: + raise NotImplementedError( + "a MoE on the TP group with moe_dp_size == attn_cp_size under " + "attention DP and attention CP" + ) def _steps_for_input_scattered(self, sides: DecoderLayerSides) -> "BoundarySteps": """The steps a batch with input-scattered attention runs at this layer, @@ -1170,28 +1140,6 @@ class LayerCommunicator: **kwargs, ) - def _post_init_communicate(self) -> Tuple[Callable, Callable]: - """The attention input move and the postprocess the scatter modes - choose.""" - if _generic_prefill_cp_shards_tokens() and _gathers_over_attention_cp(): - # These tables have no attention-CP gather. - raise NotImplementedError( - "a DSA or MLA prefill CP layer outside the declarations" - ) - return ( - CommunicateSimpleFn.get_fn( - input_mode=self.layer_scatter_modes.layer_input_mode, - output_mode=self.layer_scatter_modes.attn_mode, - context=self._context, - ), - CommunicateSummableTensorPairFn.get_fn( - hidden_states_input_mode=self.layer_scatter_modes.mlp_mode, - residual_input_mode=self.layer_scatter_modes.middle_residual_mode, - output_mode=self.layer_scatter_modes.layer_output_mode, - context=self._context, - ), - ) - def prepare_attn_and_capture_last_layer_outputs( self, hidden_states: torch.Tensor, @@ -1391,42 +1339,6 @@ class LayerCommunicator: hidden_states, residual, use_attn_tp_group=False ) - def _select_mlp_input(self) -> Tuple[Callable, Tuple["FusedMlpInput", ...]]: - """The attention-TP -> FFN boundary's steps, chosen from the layouts, and - the fused kernels they try first.""" - kind = mlp_input_kind(self.layer_scatter_modes, self._context) - residual_input_mode = self.layer_scatter_modes.layer_input_mode - ops = self._residual_ops - if kind is MlpInputKind.NORM: - return partial(_mlp_input_norm, residual_ops=ops), () - if kind is MlpInputKind.ATTN_TP_ALL_REDUCE: - return partial(_mlp_input_attn_tp_all_reduce, residual_ops=ops), () - if kind is MlpInputKind.SCATTER: - return ( - partial( - _mlp_input_scatter, - scatters_residual=residual_input_mode == ScatterMode.TP_ATTN_FULL, - residual_ops=ops, - ), - (), - ) - # Neither fused kernel runs under attention DP. - fusions = ( - self._select_mlp_input_fusions() if self._context.attn_dp_size == 1 else () - ) - steps = partial( - _mlp_input_gather, - order=_mlp_input_order( - self._context, - residual_input_mode, - tuple(f.run for f in fusions), - residual_ops=ops, - ), - ) - if kind is MlpInputKind.GATHER_MOE_CP: - steps = partial(_mlp_input_gather_moe_cp, gather=steps) - return steps, fusions - def _select_mlp_input_fusions(self) -> Tuple["FusedMlpInput", ...]: """The fused kernels that can take the attention -> FFN steps, in the order they are tried. They add the residual plainly.""" @@ -1537,15 +1449,6 @@ class LayerCommunicator: return False if dp_step is not None or steps.ffn_output_move_completes_sum: return True - # The scatter-mode steps of a DSA or MLA CP extend (the subclasses that - # pick their own steps). Prefill CP predicates must stay out of decode - # graph capture. - if ( - self._cp_steps is None - and forward_batch.forward_mode.is_context_parallel_extend() - and (dsa_use_prefill_cp(forward_batch) or is_mla_cp_active(forward_batch)) - ): - return True return get_attn_tp_context().input_scattered and not self.is_last_layer def ffn_reduction_group(self, forward_batch: ForwardBatch) -> GroupCoordinator: @@ -1672,15 +1575,10 @@ class LayerCommunicator: "a branch on a batch with attention CP, LayerNorm SP or " "input-scattered attention" ) - if self._declared is not None: - sides = self._declared - return sides.ffn.layout, sides.ffn_residual_rows, sides.output_rows - layouts, modes = self._context.layouts, self.layer_scatter_modes - return ( - layouts[modes.mlp_mode], - layouts[modes.middle_residual_mode], - layouts[modes.layer_output_mode], - ) + if self._declared is None: + raise NotImplementedError("a branch on a layer that is one stage") + sides = self._declared + return sides.ffn.layout, sides.ffn_residual_rows, sides.output_rows def branch_input( self, @@ -1846,33 +1744,6 @@ class LayerCommunicator: ) -# MOE_FULL gathers across the MoE-CP group, which spans every CP rank when CP is on. -_SCATTER_MODE_SHARDED_AXES = { - ScatterMode.SCATTERED: ( - TokenAxis.ATTN_DP, - TokenAxis.ATTN_CP, - TokenAxis.ATTN_TP_SCATTER, - ), - ScatterMode.TP_ATTN_FULL: (TokenAxis.ATTN_DP, TokenAxis.ATTN_CP), - ScatterMode.FULL: (TokenAxis.ATTN_CP,), - ScatterMode.MOE_FULL: (), -} - - -def scatter_mode_layouts( - *, attn_dp_size: int, attn_cp_size: int, attn_tp_size: int -) -> Dict[ScatterMode, Layout]: - axis_sizes = { - TokenAxis.ATTN_DP: attn_dp_size, - TokenAxis.ATTN_CP: attn_cp_size, - TokenAxis.ATTN_TP_SCATTER: attn_tp_size, - } - return { - mode: Layout.sharded_over(*axes, axis_sizes=axis_sizes) - for mode, axes in _SCATTER_MODE_SHARDED_AXES.items() - } - - class FfnExitFusion(Enum): """What an FFN exit does when a fused kernel of the next layer's input takes its sum.""" @@ -1998,20 +1869,6 @@ class CommunicateContext: tp_rank: int force_layernorm_before_dp_gather: bool = False - def is_same_group_size(self, a: ScatterMode, b: ScatterMode): - return self.process_group_sizes[a] == self.process_group_sizes[b] - - @cached_property - def layouts(self) -> Dict[ScatterMode, Layout]: - return scatter_mode_layouts( - attn_dp_size=self.attn_dp_size, - attn_cp_size=self.attn_cp_size, - attn_tp_size=self.attn_tp_size, - ) - - def is_same_layout(self, a: ScatterMode, b: ScatterMode): - return self.layouts[a] == self.layouts[b] - @classmethod def init_new(cls): attn_tp_rank = get_parallel().attn_tp_rank @@ -2044,24 +1901,6 @@ class CommunicateContext: class CommunicateSimpleFn: - @staticmethod - def get_fn( - input_mode: ScatterMode, - output_mode: ScatterMode, - context: CommunicateContext, - ): - if context.is_same_layout(input_mode, output_mode): - return CommunicateSimpleFn._trivial - - if (input_mode == ScatterMode.SCATTERED) and ( - output_mode == ScatterMode.TP_ATTN_FULL - ): - if _use_ag_after_qlora: - return CommunicateSimpleFn._trivial - return CommunicateSimpleFn._scattered_to_tp_attn_full - - raise NotImplementedError(f"{input_mode=} {output_mode=}") - @staticmethod def _trivial( hidden_states: torch.Tensor, @@ -2140,6 +1979,27 @@ def moe_cp_gathered_rows(forward_batch: ForwardBatch) -> Optional[List[int]]: return None +def sparse_moe_gathers_over_moe_cp() -> bool: + """Whether a sparse MoE's input is gathered over the MoE-CP group on a CP + extend: a MoE on the TP group under GQA CP whose MoE-CP group is wider than + the MoE's data-parallel groups. DSA and MLA CP gather over attention CP + instead.""" + return ( + not is_moe_input_scattered_across_dp_ranks() + and is_enable_moe_cp_allgather() + and not _gathers_over_attention_cp() + ) + + +def moe_cp_gathers_sparse_moe_input(forward_batch: ForwardBatch) -> bool: + """Whether a sparse MoE's input is gathered over the MoE-CP group on this + batch: a CP extend, for a MoE that gathers there.""" + return ( + sparse_moe_gathers_over_moe_cp() + and moe_cp_gathered_rows(forward_batch) is not None + ) + + def _redistribute_input_to_moe_cp( hidden_states: torch.Tensor, rows: List[int], moe_cp_size: int ) -> torch.Tensor: @@ -2338,38 +2198,6 @@ def _mlp_input_dp_partial( return hidden_states, residual -def _mlp_input_order( - context: CommunicateContext, - residual_input_mode: ScatterMode, - fusions: Tuple[Callable, ...], - residual_ops: ResidualOps = ADD_AND_NORM, -) -> Callable: - """The steps from the attention output to the FFN input, chosen from facts - fixed at construction.""" - gathers_residual = ( - residual_input_mode == ScatterMode.SCATTERED and context.attn_tp_size > 1 - ) - if context.attn_dp_size == 1: - return partial( - _mlp_input_without_dp, - gathers_residual=gathers_residual, - fusions=fusions, - residual_ops=residual_ops, - ) - if ( - context.force_layernorm_before_dp_gather - or context.attn_tp_size == 1 - or not residual_ops.adds_plainly - ): - return partial( - _mlp_input_dp_replicate, - gathers_residual=gathers_residual, - reduces_attention_tp=context.attn_tp_size > 1, - residual_ops=residual_ops, - ) - return partial(_mlp_input_dp_partial, gathers_residual=gathers_residual) - - def _sum_group(group: SumGroup) -> GroupCoordinator: parallel = get_parallel() if group is SumGroup.ATTN_TP: @@ -2407,22 +2235,6 @@ def tbo_split_moves(layer_input_rows: Layout) -> Tuple[Callable, Callable]: raise NotImplementedError(f"{layer_input_rows=}") -def _cp_on_declarations() -> bool: - """Whether attention CP is one the declarations cover: a prefill CP that - shards tokens, with DSA or MLA attention, with the FFN input gathered over a - MoE-CP group that is the whole CP group, or, without attention DP, with the - MoE's data-parallel groups the CP ranks (a MoE then computes each CP shard - on its own ranks).""" - parallel = get_parallel() - return _generic_prefill_cp_shards_tokens() and ( - _gathers_over_attention_cp() - or parallel.moe_dp_size == 1 - or ( - parallel.moe_dp_size == parallel.attn_cp_size and parallel.attn_dp_size == 1 - ) - ) - - def _gathers_over_attention_cp() -> bool: """Whether a CP extend gathers the FFN input over the attention-CP group in equal shards and takes the output back with a reduce-scatter there: DSA and @@ -2788,19 +2600,6 @@ def _select_boundary_steps( ) -def _complete_scattered_input( - hidden_states: torch.Tensor, - residual: Optional[torch.Tensor], - context: "CommunicateContext", -) -> Tuple[torch.Tensor, Optional[torch.Tensor]]: - """The scatter-mode path's input completion: with input-scattered - attention the layer's input is a TP partial that a reduce-scatter completes - onto this rank's slice.""" - if get_attn_tp_context().input_scattered: - return tp_reduce_scatter(hidden_states, residual, context) - return hidden_states, residual - - def _select_attention_input_move(rows: Layout, need: StageInput) -> Callable: """How the rows a layer takes become its attention's input: as they are, or gathered over attention TP from each rank's slice, unless the attention @@ -3030,61 +2829,6 @@ def _select_ffn_output_move( return True, None, False -class MlpInputKind(Enum): - """What the attention-TP -> FFN boundary does, given the two sides' layouts.""" - - # The layouts agree: add the residual and normalize. - NORM = auto() - # All-reduce over attention TP, normalize, and gather for the FFN group. - GATHER = auto() - # The same, then gather over the MoE-CP group (moe_dp_size < attn_cp_size). - GATHER_MOE_CP = auto() - # Reduce-scatter over attention TP to this rank's tokens, then normalize. - SCATTER = auto() - # All-reduce over attention TP for a dense MLP on that group. - ATTN_TP_ALL_REDUCE = auto() - - -def mlp_input_kind( - modes: LayerScatterModes, context: CommunicateContext -) -> MlpInputKind: - hidden_in, residual_in = modes.attn_mode, modes.layer_input_mode - hidden_out, residual_out = modes.mlp_mode, modes.middle_residual_mode - if ( - context.is_same_layout(hidden_in, hidden_out) - and context.is_same_layout(residual_in, residual_out) - and context.attn_tp_size == 1 - ): - return MlpInputKind.NORM - if hidden_in == ScatterMode.TP_ATTN_FULL and residual_in in ( - ScatterMode.SCATTERED, - ScatterMode.TP_ATTN_FULL, - ): - kind = { - (ScatterMode.FULL, ScatterMode.TP_ATTN_FULL): MlpInputKind.GATHER, - ( - ScatterMode.MOE_FULL, - ScatterMode.TP_ATTN_FULL, - ): MlpInputKind.GATHER_MOE_CP, - (ScatterMode.SCATTERED, ScatterMode.SCATTERED): MlpInputKind.SCATTER, - }.get((hidden_out, residual_out)) - if kind is not None: - return kind - if ( - hidden_out == ScatterMode.TP_ATTN_FULL - and residual_out == ScatterMode.TP_ATTN_FULL - and context.attn_tp_size > 1 - ): - # Used when the dense MLP is tensor-parallelized along the - # attention TP group (``moe_dense_tp_size > 1``): hidden states - # need an all-reduce inside the attention TP group before the - # next layernorm, while staying in TP_ATTN_FULL on both sides. - return MlpInputKind.ATTN_TP_ALL_REDUCE - raise NotImplementedError( - f"{hidden_in=} {residual_in=} {hidden_out=} {residual_out=}" - ) - - def _mlp_input_completing_owed( hidden_states: Union[torch.Tensor, "UnreducedOutput"], residual: torch.Tensor, @@ -3113,25 +2857,6 @@ def _mlp_input_norm( return residual_ops.update_and_read_ffn_input(hidden_states, residual, layernorm) -def _mlp_input_attn_tp_all_reduce( - hidden_states: torch.Tensor, - residual: torch.Tensor, - forward_batch: ForwardBatch, - layernorm: torch.nn.Module, - context: CommunicateContext, - *, - residual_ops: ResidualOps = ADD_AND_NORM, -): - """All-reduce hidden states inside the attention TP group, then layernorm. - - Used when the dense MLP shares the attention TP group - (``moe_dense_tp_size > 1``): both hidden states and residual stay in - ``TP_ATTN_FULL`` across the boundary. - """ - hidden_states = get_parallel().attn_tp_group.all_reduce(hidden_states) - return residual_ops.update_and_read_ffn_input(hidden_states, residual, layernorm) - - def _mlp_input_scatter( hidden_states: torch.Tensor, residual: torch.Tensor, @@ -3216,24 +2941,6 @@ def _tp_all_reduce_with_scattered_residual( return hidden_states, residual -def _mlp_input_gather( - hidden_states: torch.Tensor, - residual: torch.Tensor, - forward_batch: ForwardBatch, - layernorm: torch.nn.Module, - context: CommunicateContext, - *, - order: Callable, -): - """Run ``order``, the steps ``_mlp_input_order`` chose at construction, - unless this batch's attention input is scattered.""" - if get_attn_tp_context().input_scattered: - return _tp_all_reduce_with_scattered_residual( - hidden_states, residual, layernorm, context - ) - return order(hidden_states, residual, forward_batch, layernorm, context) - - def _mlp_input_gather_attention_cp( hidden_states: torch.Tensor, residual: torch.Tensor, @@ -3375,47 +3082,6 @@ def _all_reduce_then_to_local_tokens( class CommunicateSummableTensorPairFn: """It is allowed to make (hidden_states, residual) := (hidden_states + residual, None) if needed.""" - @staticmethod - def get_fn( - hidden_states_input_mode: ScatterMode, - residual_input_mode: ScatterMode, - output_mode: ScatterMode, - context: CommunicateContext, - ): - if context.is_same_layout( - hidden_states_input_mode, output_mode - ) and context.is_same_layout(residual_input_mode, output_mode): - return CommunicateSummableTensorPairFn._trivial - - fn = { - ( - ScatterMode.FULL, - ScatterMode.TP_ATTN_FULL, - ScatterMode.TP_ATTN_FULL, - ): CommunicateSummableTensorPairFn._scatter_hidden_states, - ( - ScatterMode.SCATTERED, - ScatterMode.SCATTERED, - ScatterMode.TP_ATTN_FULL, - ): CommunicateSummableTensorPairFn._gather, - ( - ScatterMode.TP_ATTN_FULL, - ScatterMode.TP_ATTN_FULL, - ScatterMode.SCATTERED, - ): CommunicateSummableTensorPairFn._scatter, - ( - ScatterMode.MOE_FULL, - ScatterMode.TP_ATTN_FULL, - ScatterMode.TP_ATTN_FULL, - ): CommunicateSummableTensorPairFn._scatter_hidden_states_moe, - }.get((hidden_states_input_mode, residual_input_mode, output_mode)) - if fn is not None: - return fn - - raise NotImplementedError( - f"{hidden_states_input_mode=} {residual_input_mode=} {output_mode=}" - ) - @staticmethod def _trivial( hidden_states: torch.Tensor, diff --git a/python/sglang/srt/layers/communicator_mhc.py b/python/sglang/srt/layers/communicator_mhc.py index c8dc5a702f..e276a6cfd9 100644 --- a/python/sglang/srt/layers/communicator_mhc.py +++ b/python/sglang/srt/layers/communicator_mhc.py @@ -22,7 +22,7 @@ from sglang.srt.layers.communicator import ( BoundarySteps, LayerCommunicator, LayerScatterModes, - ScatterMode, + sparse_moe_gathers_over_moe_cp, ) @@ -169,6 +169,12 @@ class MHCLayerCommunicator(LayerCommunicator): hc_ffn_post_pre=hc_ffn_post_pre, is_last_layer=layer_scatter_modes.is_last_layer, ) + if layer_scatter_modes.is_layer_sparse and sparse_moe_gathers_over_moe_cp(): + raise NotImplementedError( + "MHCLayerCommunicator does not support a MoE gathered over the " + "MoE-CP group (moe_dp_size < attention_context_parallel_size). " + "Increase moe_dp_size to match attention_context_parallel_size." + ) # The postprocess writes the FFN output into the streams, so the FFN's # sum never waits for the next layer. super().__init__( @@ -180,6 +186,11 @@ class MHCLayerCommunicator(LayerCommunicator): allow_deferred_ffn_reduction=False, residual_ops=self.mhc, ) + # MHC has not been run with input-scattered attention under attention CP. + if self._context.attn_cp_size > 1 and self._input_can_be_scattered(): + raise NotImplementedError( + "MHCLayerCommunicator with input-scattered attention under attention CP" + ) def _steps_from_declarations( self, sides: DecoderLayerSides, **kwargs @@ -193,12 +204,3 @@ class MHCLayerCommunicator(LayerCommunicator): f"MHCLayerCommunicator with a gather over attention CP: {sides=}" ) return super()._steps_from_declarations(sides, **kwargs) - - def _post_init_communicate(self): - if self.layer_scatter_modes.mlp_mode == ScatterMode.MOE_FULL: - raise NotImplementedError( - "MHCLayerCommunicator does not support MOE_FULL " - "(moe_dp_size < attention_context_parallel_size). Increase " - "moe_dp_size to match attention_context_parallel_size." - ) - return super()._post_init_communicate() diff --git a/python/sglang/srt/model_executor/forward_batch_info.py b/python/sglang/srt/model_executor/forward_batch_info.py index 5f06a44696..cf0382740b 100644 --- a/python/sglang/srt/model_executor/forward_batch_info.py +++ b/python/sglang/srt/model_executor/forward_batch_info.py @@ -1230,23 +1230,19 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin): def moe_num_token_non_padded(self) -> Optional[torch.Tensor]: """Bound for masking a sparse MoE's padded rows, or None when the MoE input is a gathered buffer whose real rows are not a prefix of it.""" - from sglang.srt.layers.communicator import ( - ScatterMode, - moe_cp_gathered_rows, - sparse_mlp_scatter_mode, - ) + from sglang.srt.layers.communicator import moe_cp_gathers_sparse_moe_input + from sglang.srt.layers.moe.utils import is_moe_input_scattered_across_dp_ranks if self.num_token_non_padded is None: return None - mode = sparse_mlp_scatter_mode() - if mode == ScatterMode.SCATTERED: + if is_moe_input_scattered_across_dp_ranks(): # a2a dispatch, FP4 all-gather and dwdp all route the local shard. return self.num_token_non_padded - if mode == ScatterMode.MOE_FULL and moe_cp_gathered_rows(self) is not None: + if moe_cp_gathers_sparse_moe_input(self): return None - # DSA / MLA CP take the FULL mode but all-gather across CP on a prefill, - # which leaves the real rows zigzag-permuted rather than in a prefix. + # DSA / MLA CP all-gather across attention CP on a prefill, which leaves + # the real rows zigzag-permuted rather than in a prefix. if get_parallel().attn_cp_size > 1 and self._moe_input_gathered_across_cp(): return None if get_parallel().attn_dp_size != 1: diff --git a/python/sglang/srt/models/deepseek_v2.py b/python/sglang/srt/models/deepseek_v2.py index d869bead24..b19df05080 100644 --- a/python/sglang/srt/models/deepseek_v2.py +++ b/python/sglang/srt/models/deepseek_v2.py @@ -2288,7 +2288,7 @@ class DeepseekV2AttentionMLA( hidden_states: torch.Tensor, forward_batch: ForwardBatch, zero_allocator: BumpAllocator, - layer_scatter_modes: LayerScatterModes = None, + input_on_attention_tp_slices: bool = False, llama_4_scaling: Optional[torch.Tensor] = None, prev_topk_indices: Optional[torch.Tensor] = None, ): @@ -2297,7 +2297,7 @@ class DeepseekV2AttentionMLA( hidden_states=hidden_states, forward_batch=forward_batch, zero_allocator=zero_allocator, - layer_scatter_modes=layer_scatter_modes, + input_on_attention_tp_slices=input_on_attention_tp_slices, llama_4_scaling=llama_4_scaling, prev_topk_indices=prev_topk_indices, ) @@ -2309,7 +2309,7 @@ class DeepseekV2AttentionMLA( hidden_states: torch.Tensor, forward_batch: ForwardBatch, zero_allocator: BumpAllocator, - layer_scatter_modes: LayerScatterModes = None, + input_on_attention_tp_slices: bool = False, llama_4_scaling: Optional[torch.Tensor] = None, prev_topk_indices: Optional[torch.Tensor] = None, ): @@ -2394,7 +2394,7 @@ class DeepseekV2AttentionMLA( hidden_states, forward_batch, zero_allocator, - layer_scatter_modes, + input_on_attention_tp_slices, ) elif attn_forward_method == AttnForwardMethod.MLA_NPU: inner_state = forward_mla_prepare_npu( @@ -2403,7 +2403,7 @@ class DeepseekV2AttentionMLA( hidden_states, forward_batch, zero_allocator, - layer_scatter_modes, + input_on_attention_tp_slices, ) elif attn_forward_method == AttnForwardMethod.DSA_NPU: inner_state = forward_dsa_prepare_npu( @@ -2412,7 +2412,7 @@ class DeepseekV2AttentionMLA( hidden_states, forward_batch, zero_allocator, - layer_scatter_modes, + input_on_attention_tp_slices, prev_topk_indices, ) else: @@ -2724,7 +2724,9 @@ class DeepseekV2DecoderLayer(nn.Module): forward_batch=forward_batch, zero_allocator=zero_allocator, llama_4_scaling=llama_4_scaling, - layer_scatter_modes=self.layer_scatter_modes, + input_on_attention_tp_slices=( + self.layer_communicator.input_on_attention_tp_slices + ), prev_topk_indices=prev_topk_indices, ) if isinstance(hidden_states, tuple): diff --git a/python/sglang/srt/models/gigachat35.py b/python/sglang/srt/models/gigachat35.py index 6bf8215c08..9bb4af479e 100644 --- a/python/sglang/srt/models/gigachat35.py +++ b/python/sglang/srt/models/gigachat35.py @@ -479,7 +479,9 @@ class GigaChat35DecoderLayer(deepseek_v2.DeepseekV2DecoderLayer): hidden_states=hidden_states, forward_batch=forward_batch, zero_allocator=zero_allocator, - layer_scatter_modes=self.layer_scatter_modes, + input_on_attention_tp_slices=( + self.layer_communicator.input_on_attention_tp_slices + ), ) get_attn_tp_context().clear_attn_inputs() diff --git a/python/sglang/srt/models/glm4_moe_lite.py b/python/sglang/srt/models/glm4_moe_lite.py index 3028e2f0d9..6e21969626 100644 --- a/python/sglang/srt/models/glm4_moe_lite.py +++ b/python/sglang/srt/models/glm4_moe_lite.py @@ -638,7 +638,9 @@ class Glm4MoeLiteDecoderLayer(nn.Module): hidden_states=hidden_states, forward_batch=forward_batch, zero_allocator=zero_allocator, - layer_scatter_modes=self.layer_scatter_modes, + input_on_attention_tp_slices=( + self.layer_communicator.input_on_attention_tp_slices + ), ) if isinstance(hidden_states, tuple): hidden_states = hidden_states[0] diff --git a/python/sglang/srt/models/glm5_next.py b/python/sglang/srt/models/glm5_next.py index c6ae5b304d..8631154f4a 100644 --- a/python/sglang/srt/models/glm5_next.py +++ b/python/sglang/srt/models/glm5_next.py @@ -932,7 +932,9 @@ class Glm5NextDecoderLayer(nn.Module): hidden_states=hidden_states, forward_batch=forward_batch, zero_allocator=zero_allocator, - layer_scatter_modes=self.layer_scatter_modes, + input_on_attention_tp_slices=( + self.layer_communicator.input_on_attention_tp_slices + ), prev_topk_indices=prev_topk_indices, ) if isinstance(hidden_states, tuple): diff --git a/test/registered/unit/layers/test_communicator_layout.py b/test/registered/unit/layers/test_communicator_layout.py deleted file mode 100644 index 542821fe2a..0000000000 --- a/test/registered/unit/layers/test_communicator_layout.py +++ /dev/null @@ -1,332 +0,0 @@ -import unittest -from functools import partial -from types import SimpleNamespace -from unittest.mock import patch - -from sglang.srt.layers import communicator as comm -from sglang.srt.layers.boundary_layout import Layout, TokenAxis -from sglang.srt.layers.communicator import ( - CommunicateContext, - CommunicateSimpleFn, - CommunicateSummableTensorPairFn, - ScatterMode, -) -from sglang.test.ci.ci_register import register_cpu_ci -from sglang.test.test_utils import CustomTestCase - -register_cpu_ci(est_time=5, suite="base-a-test-cpu") - -SCATTERED = ScatterMode.SCATTERED -TP_ATTN_FULL = ScatterMode.TP_ATTN_FULL -FULL = ScatterMode.FULL -MOE_FULL = ScatterMode.MOE_FULL -Pair = CommunicateSummableTensorPairFn - - -def make_context(*, dp=1, cp=1, tp=1): - """A context for attention DP x CP x TP, built without the process-wide - parallel state.""" - return CommunicateContext( - process_group_sizes={}, - attn_tp_rank=0, - attn_tp_size=tp, - attn_dp_size=dp, - attn_cp_rank=0, - attn_cp_size=cp, - tp_size=dp * cp * tp, - tp_rank=0, - ) - - -class TestScatterModeLayouts(CustomTestCase): - def test_modes_shard_nested_axes(self): - layouts = make_context(dp=2, cp=2, tp=2).layouts - self.assertEqual( - layouts[SCATTERED], - Layout( - frozenset( - {TokenAxis.ATTN_DP, TokenAxis.ATTN_CP, TokenAxis.ATTN_TP_SCATTER} - ) - ), - ) - self.assertEqual( - layouts[TP_ATTN_FULL], - Layout(frozenset({TokenAxis.ATTN_DP, TokenAxis.ATTN_CP})), - ) - self.assertEqual(layouts[FULL], Layout(frozenset({TokenAxis.ATTN_CP}))) - self.assertEqual(layouts[MOE_FULL], Layout(frozenset())) - - def test_single_rank_axes_shard_nothing(self): - cases = [ - # (topology, pairs with equal layouts, pairs with different layouts) - ( - dict(tp=2), - [(FULL, TP_ATTN_FULL), (MOE_FULL, FULL)], - [(SCATTERED, TP_ATTN_FULL)], - ), - (dict(dp=2, tp=2), [(MOE_FULL, FULL)], [(FULL, TP_ATTN_FULL)]), - ( - dict(cp=2), - [(SCATTERED, TP_ATTN_FULL), (FULL, TP_ATTN_FULL)], - [(MOE_FULL, FULL)], - ), - (dict(), [(SCATTERED, MOE_FULL)], []), - ] - for topology, same, different in cases: - context = make_context(**topology) - for a, b in same: - with self.subTest(topology=topology, same=(a, b)): - self.assertTrue(context.is_same_layout(a, b)) - for a, b in different: - with self.subTest(topology=topology, different=(a, b)): - self.assertFalse(context.is_same_layout(a, b)) - - -def modes(attn, layer_input, mlp, middle_residual): - return SimpleNamespace( - attn_mode=attn, - layer_input_mode=layer_input, - mlp_mode=mlp, - middle_residual_mode=middle_residual, - ) - - -class TestBoundarySelection(CustomTestCase): - def test_prepare_mlp(self): - dp_tp = make_context(dp=2, tp=2) - Kind = comm.MlpInputKind - for layout, context, kind in ( - ((TP_ATTN_FULL, SCATTERED, FULL, TP_ATTN_FULL), dp_tp, Kind.GATHER), - ( - (TP_ATTN_FULL, TP_ATTN_FULL, MOE_FULL, TP_ATTN_FULL), - make_context(cp=2), - Kind.GATHER_MOE_CP, - ), - ((TP_ATTN_FULL, TP_ATTN_FULL, SCATTERED, SCATTERED), dp_tp, Kind.SCATTER), - ): - with self.subTest(layout=layout): - self.assertIs(comm.mlp_input_kind(modes(*layout), context), kind) - - def test_prepare_mlp_without_attention_dp(self): - # FULL and TP_ATTN_FULL share a layout here but select different paths. - tp = make_context(tp=2) - Kind = comm.MlpInputKind - for layout, context, kind in ( - ((TP_ATTN_FULL, TP_ATTN_FULL, FULL, TP_ATTN_FULL), tp, Kind.GATHER), - ( - (TP_ATTN_FULL, TP_ATTN_FULL, TP_ATTN_FULL, TP_ATTN_FULL), - tp, - Kind.ATTN_TP_ALL_REDUCE, - ), - ( - (TP_ATTN_FULL, TP_ATTN_FULL, FULL, TP_ATTN_FULL), - make_context(), - Kind.NORM, - ), - ): - with self.subTest(layout=layout): - self.assertIs(comm.mlp_input_kind(modes(*layout), context), kind) - for layout in ( - (SCATTERED, SCATTERED, TP_ATTN_FULL, TP_ATTN_FULL), - # The LayerNorm SP region chooses its steps from its declarations. - (SCATTERED, SCATTERED, SCATTERED, SCATTERED), - ): - with self.subTest(layout=layout), self.assertRaises(NotImplementedError): - comm.mlp_input_kind(modes(*layout), tp) - - def test_postprocess(self): - self.assertIs( - Pair.get_fn(FULL, TP_ATTN_FULL, TP_ATTN_FULL, make_context(dp=2)), - Pair._scatter_hidden_states, - ) - self.assertIs( - Pair.get_fn(FULL, TP_ATTN_FULL, TP_ATTN_FULL, make_context(tp=2)), - Pair._trivial, - ) - self.assertIs( - Pair.get_fn(MOE_FULL, TP_ATTN_FULL, TP_ATTN_FULL, make_context(cp=2)), - Pair._scatter_hidden_states_moe, - ) - tp = make_context(tp=2) - self.assertIs(Pair.get_fn(SCATTERED, SCATTERED, TP_ATTN_FULL, tp), Pair._gather) - self.assertIs( - Pair.get_fn(TP_ATTN_FULL, TP_ATTN_FULL, SCATTERED, tp), Pair._scatter - ) - - def test_prepare_attn(self): - with patch.object(comm, "_use_ag_after_qlora", False): - self.assertIs( - CommunicateSimpleFn.get_fn(SCATTERED, TP_ATTN_FULL, make_context(tp=2)), - CommunicateSimpleFn._scattered_to_tp_attn_full, - ) - self.assertIs( - CommunicateSimpleFn.get_fn(SCATTERED, TP_ATTN_FULL, make_context(cp=2)), - CommunicateSimpleFn._trivial, - ) - - -class Fusable: - def forward_with_allreduce_fusion(self, *args, **kwargs): - pass - - -def communicator(layout, context, post_attention_layernorm, cls=comm.LayerCommunicator): - """A communicator with these layer layouts, built without the process-wide - parallel state; returns it with its chosen prepare_mlp steps.""" - c = cls.__new__(cls) - c.layer_scatter_modes = modes(*layout) - c._context = context - c.post_attention_layernorm = post_attention_layernorm - return c, c._select_mlp_input() - - -GATHER_LAYOUT = (TP_ATTN_FULL, TP_ATTN_FULL, FULL, TP_ATTN_FULL) - - -class TestMlpInputOrder(CustomTestCase): - def assert_order(self, order, func, residual_ops=comm.ADD_AND_NORM, **keywords): - self.assertIsInstance(order, partial) - self.assertIs(order.func, func) - bound = dict(order.keywords) - self.assertIs(bound.pop("residual_ops", comm.ADD_AND_NORM), residual_ops) - self.assertEqual(bound, keywords) - - def test_facts_fixed_at_construction_pick_the_order(self): - fusions = (object(),) - for mode, gathers in ((TP_ATTN_FULL, False), (SCATTERED, True)): - with self.subTest(residual_input_mode=mode): - self.assert_order( - comm._mlp_input_order(make_context(tp=2), mode, fusions), - comm._mlp_input_without_dp, - gathers_residual=gathers, - fusions=fusions, - ) - # Attention TP 1 has no partial sum to carry through the gather. - self.assert_order( - comm._mlp_input_order(make_context(dp=2), TP_ATTN_FULL, fusions), - comm._mlp_input_dp_replicate, - gathers_residual=False, - reduces_attention_tp=False, - ) - dp_tp = make_context(dp=2, tp=2) - self.assert_order( - comm._mlp_input_order(dp_tp, SCATTERED, fusions), - comm._mlp_input_dp_partial, - gathers_residual=True, - ) - dp_tp.force_layernorm_before_dp_gather = True - self.assert_order( - comm._mlp_input_order(dp_tp, SCATTERED, fusions), - comm._mlp_input_dp_replicate, - gathers_residual=True, - reduces_attention_tp=True, - ) - - def test_a_gather_runs_the_order_chosen_at_construction(self): - for mlp_mode in (FULL, MOE_FULL): - with self.subTest(mlp_mode=mlp_mode): - c, (steps, fused) = communicator( - (TP_ATTN_FULL, TP_ATTN_FULL, mlp_mode, TP_ATTN_FULL), - make_context(tp=2), - Fusable(), - ) - if mlp_mode is MOE_FULL: - self.assertIs(steps.func, comm._mlp_input_gather_moe_cp) - steps = steps.keywords["gather"] - self.assertIs(steps.func, comm._mlp_input_gather) - entry = c._mlp_input_reduce_output_and_update_and_read_residual - self.assert_order( - steps.keywords["order"], - comm._mlp_input_without_dp, - gathers_residual=False, - fusions=(entry,), - ) - self.assertEqual([f.run for f in fused], [entry]) - - def test_no_fused_entry_without_a_fusable_norm_or_under_attention_dp(self): - for context, norm in ( - (make_context(tp=2), object()), - (make_context(dp=2, tp=2), Fusable()), - ): - with self.subTest(dp=context.attn_dp_size): - _, (steps, fused) = communicator(GATHER_LAYOUT, context, norm) - order = steps.keywords["order"] - self.assertEqual(order.keywords.get("fusions", ()), ()) - self.assertEqual(fused, ()) - - def test_other_kinds_take_their_steps(self): - tp = make_context(tp=2) - for layout, context, expected in ( - ( - (TP_ATTN_FULL, TP_ATTN_FULL, FULL, TP_ATTN_FULL), - make_context(), - comm._mlp_input_norm, - ), - ( - (TP_ATTN_FULL, TP_ATTN_FULL, TP_ATTN_FULL, TP_ATTN_FULL), - tp, - comm._mlp_input_attn_tp_all_reduce, - ), - ): - with self.subTest(expected=expected.__name__): - _, (steps, fused) = communicator(layout, context, Fusable()) - self.assert_order(steps, expected) - self.assertEqual(fused, ()) - _, (steps, _) = communicator( - (TP_ATTN_FULL, TP_ATTN_FULL, SCATTERED, SCATTERED), tp, Fusable() - ) - self.assert_order(steps, comm._mlp_input_scatter, scatters_residual=True) - - def test_mhc_runs_the_shared_steps_with_its_residual(self): - from sglang.srt.layers.communicator_mhc import ( - MHCLayerCommunicator, - MHCState, - ) - - mhc = MHCState(hc_mult=2, hc_attn_pre=None, hc_ffn_pre=None, hc_post=None) - for layout, func, order, keywords in ( - ( - GATHER_LAYOUT, - comm._mlp_input_gather, - comm._mlp_input_without_dp, - # The fused add + RMSNorm kernels do not write hc_post. - dict(gathers_residual=False, fusions=()), - ), - ( - (TP_ATTN_FULL, TP_ATTN_FULL, SCATTERED, SCATTERED), - comm._mlp_input_scatter, - None, - dict(scatters_residual=True), - ), - ): - with self.subTest(func=func.__name__): - c = MHCLayerCommunicator.__new__(MHCLayerCommunicator) - c._residual_ops = mhc - c.layer_scatter_modes = modes(*layout) - c._context = make_context(tp=2) - c.post_attention_layernorm = Fusable() - steps, fused = c._select_mlp_input() - if order is None: - self.assert_order(steps, func, residual_ops=mhc, **keywords) - else: - self.assertIs(steps.func, func) - self.assert_order( - steps.keywords["order"], order, residual_ops=mhc, **keywords - ) - self.assertEqual(fused, ()) - - def test_mhc_gathers_over_dp_after_its_write_back(self): - from sglang.srt.layers.communicator_mhc import MHCState - - mhc = MHCState(hc_mult=2, hc_attn_pre=None, hc_ffn_pre=None, hc_post=None) - self.assert_order( - comm._mlp_input_order(make_context(dp=2, tp=2), SCATTERED, (), mhc), - comm._mlp_input_dp_replicate, - residual_ops=mhc, - gathers_residual=True, - reduces_attention_tp=True, - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/test/registered/unit/layers/test_declared_decoder_boundary.py b/test/registered/unit/layers/test_declared_decoder_boundary.py index 563c2a0b1a..ec7f249729 100644 --- a/test/registered/unit/layers/test_declared_decoder_boundary.py +++ b/test/registered/unit/layers/test_declared_decoder_boundary.py @@ -34,7 +34,6 @@ from sglang.srt.layers.communicator import ( LayerScatterModes, ScatterMode, UnreducedOutput, - scatter_mode_layouts, ) from sglang.srt.layers.communicator_mhc import MHCLayerCommunicator from sglang.srt.layers.moe import utils as moe_utils @@ -228,92 +227,6 @@ def planned_modes( ) -class TestDeclarationsMatchScatterModes(CustomTestCase): - """The layouts derived from the groups each side computes over are the - layouts of the modes layer planning gives the same layers.""" - - def test_every_attention_dp_and_tp(self): - for ( - attn_dp, - attn_tp, - layer_id, - sparse, - previous_sparse, - a2a, - ) in itertools.product( - (1, 2, 4, 8), - (1, 2, 4), - (0, 1, 3), - (False, True), - (False, True), - (False, True), - ): - with self.subTest( - attn_dp=attn_dp, - attn_tp=attn_tp, - layer_id=layer_id, - sparse=sparse, - previous_sparse=previous_sparse, - a2a=a2a, - ): - parallel = parallel_of(attn_dp=attn_dp, attn_tp=attn_tp) - modes = planned_modes( - layer_id, - 4, - sparse=sparse, - previous_sparse=previous_sparse, - parallel=parallel, - a2a=a2a, - ) - sizes = { - TokenAxis.ATTN_DP: attn_dp, - TokenAxis.ATTN_CP: 1, - TokenAxis.ATTN_TP_SCATTER: attn_tp, - } - layouts = scatter_mode_layouts( - attn_dp_size=attn_dp, attn_cp_size=1, attn_tp_size=attn_tp - ) - sides = sides_of( - sizes, - sparse=sparse, - a2a=a2a, - previous_a2a=layer_id > 0 and previous_sparse and a2a, - last=layer_id == 3, - leaves_next=True, - leaves_rs=True, - ) - self.assertEqual(sides.input_rows, layouts[modes.layer_input_mode]) - self.assertEqual( - sides.ffn_residual_rows, layouts[modes.middle_residual_mode] - ) - self.assertEqual(sides.output_rows, layouts[modes.layer_output_mode]) - self.assertEqual(sides.attention.layout, layouts[modes.attn_mode]) - self.assertEqual(sides.ffn.layout, layouts[modes.mlp_mode]) - self.assertEqual( - sides.attention_output.layout, layouts[modes.attn_mode] - ) - self.assertEqual(sides.ffn_output.layout, sides.ffn.layout) - # A MoE dispatched per DP shard hands on a complete output. - self.assertIs(sides.ffn_output.group is None, sparse and a2a) - - def test_the_attention_output_owes_the_attention_tp_sum(self): - for attn_tp in (1, 2): - sides = sides_of( - { - TokenAxis.ATTN_DP: 2, - TokenAxis.ATTN_CP: 1, - TokenAxis.ATTN_TP_SCATTER: attn_tp, - } - ) - owed = attn_tp > 1 - self.assertIs( - sides.attention_output.group, SumGroup.ATTN_TP if owed else None - ) - self.assertIs(sides.attention_output.always_leaves, owed) - self.assertIs(sides.ffn_output.group, SumGroup.TP) - self.assertFalse(sides.ffn_output.always_leaves) - - class TestWhichLayersUseDeclarations(CustomTestCase): """The layer is the unit: all three boundary sides it owns follow the declarations, or none do.""" @@ -501,30 +414,27 @@ class TestWhichLayersUseDeclarations(CustomTestCase): layer = build(modes, parallel, a2a=True) self.assertIs(layer._steps.attention_input, comm.CommunicateSimpleFn._trivial) - def test_layers_that_keep_the_scatter_modes(self): - dp = parallel_of(attn_dp=2, attn_tp=2) - cases = { - "attention CP": ( - layer_facts(1, 3), - parallel_of(attn_dp=2, attn_tp=1, attn_cp=2), - ), - } - two_batch_overlap = SimpleNamespace( - overlap=SimpleNamespace(enable_two_batch_overlap=True) + def test_cp_without_prefill_cp(self): + """No batch shards its tokens, so no CP steps. Under attention DP the + reduce-scatter back runs over the TP group, which spans the CP ranks, + so the FFN completes its own sum; without attention DP it may leave it + and input-scattered attention runs, as without CP.""" + under_dp = build( + layer_facts(1, 3), parallel_of(attn_dp=2, attn_tp=1, attn_cp=2) ) - for name, (facts, parallel) in cases.items(): - with self.subTest(name): - with ( - planning(parallel), - patch.object(comm, "get_exec", lambda: two_batch_overlap), - ): - communicator = LayerCommunicator.__new__(LayerCommunicator) - communicator.layer_scatter_modes = facts - communicator.allow_deferred_ffn_reduction = True - communicator.allow_reduce_scatter = False - self.assertIsNone(communicator._declared_sides()) - # Modes given directly (Nemotron-H's stages) do not say which rows the - # layer takes. + self.assertIsNone(under_dp._cp_steps) + self.assertFalse(under_dp._steps.ffn_output.leaves_for_next_layer) + self.assertFalse(under_dp._steps.ffn_output.leaves_for_reduce_scatterv) + without_dp = build( + layer_facts(1, 3), + parallel_of( + attn_dp=1, attn_tp=2, attn_cp=2, enable_attn_tp_input_scattered=True + ), + ) + self.assertTrue(without_dp._steps.ffn_output.leaves_for_next_layer) + self.assertIsNotNone(without_dp._input_scattered_steps) + + def test_a_layer_without_the_previous_layer_s_facts_is_refused(self): direct = LayerScatterModes( layer_input_mode=ScatterMode.TP_ATTN_FULL, attn_mode=ScatterMode.TP_ATTN_FULL, @@ -532,7 +442,8 @@ class TestWhichLayersUseDeclarations(CustomTestCase): middle_residual_mode=ScatterMode.TP_ATTN_FULL, layer_output_mode=ScatterMode.TP_ATTN_FULL, ) - self.assertFalse(self.declared(build(direct, dp))) + with self.assertRaises(NotImplementedError): + build(direct, parallel_of(attn_dp=2, attn_tp=2)) def build_mhc( @@ -809,6 +720,48 @@ class TestMhcOnTheDeclarations(CustomTestCase): ) build_mhc(modes, parallel, dsa_cp=True, allow_reduce_scatter=True) + def test_input_scattered_attention_under_attention_cp_is_rejected(self): + scattered = dict(enable_attn_tp_input_scattered=True) + parallel = parallel_of(attn_dp=1, attn_tp=2, attn_cp=2, **scattered) + with self.assertRaisesRegex(NotImplementedError, "input-scattered"): + build_mhc( + planned_modes( + 1, 3, sparse=False, previous_sparse=False, parallel=parallel + ), + parallel, + ) + parallel = parallel_of(attn_dp=1, attn_tp=2, **scattered) + build_mhc( + planned_modes(1, 3, sparse=False, previous_sparse=False, parallel=parallel), + parallel, + ) + + def test_a_moe_gathered_over_moe_cp_is_rejected(self): + # A MoE-CP gather (MoE DP narrower than CP), which MHC has not been run with. + for prefill_cp in (True, False): + with self.subTest(prefill_cp=prefill_cp): + parallel = parallel_of( + attn_dp=1, attn_tp=2, attn_cp=2, enable_prefill_cp=prefill_cp + ) + modes = planned_modes( + 1, 3, sparse=True, previous_sparse=True, parallel=parallel + ) + with self.assertRaisesRegex(NotImplementedError, "MoE-CP group"): + build_mhc(modes, parallel) + # A dense layer builds, and so does a MoE on its own CP shard (MoE DP = CP). + parallel = parallel_of(attn_dp=1, attn_tp=2, attn_cp=2) + build_mhc( + planned_modes(1, 3, sparse=False, previous_sparse=False, parallel=parallel), + parallel, + ) + parallel = parallel_of( + attn_dp=1, attn_tp=2, attn_cp=2, moe_dp_size=2, moe_tp_size=2 + ) + build_mhc( + planned_modes(1, 3, sparse=True, previous_sparse=True, parallel=parallel), + parallel, + ) + class TestTwoBatchOverlap(CustomTestCase): """Two-batch overlap splits the attention's rows. A dense MLP on every rank @@ -860,14 +813,6 @@ class TestTwoBatchOverlap(CustomTestCase): after._steps.attention_input, comm.CommunicateSimpleFn._scattered_to_tp_attn_full, ) - # The declarations give the rows the scatter modes planned. - for layer in (first, before, after, last): - layouts = layer._context.layouts - modes = layer.layer_scatter_modes - self.assertEqual(layer.input_rows, layouts[modes.layer_input_mode]) - self.assertEqual( - layer._declared.output_rows, layouts[modes.layer_output_mode] - ) def test_the_split_moves_from_the_first_layer_s_rows(self): pair = comm.CommunicateSummableTensorPairFn @@ -884,54 +829,13 @@ class TestTwoBatchOverlap(CustomTestCase): with self.assertRaises(NotImplementedError): comm.tbo_split_moves(comm.Layout(frozenset())) - def test_a_layer_off_the_declarations_takes_its_rows_from_its_modes(self): - dp = parallel_of(attn_dp=2, attn_tp=2) - direct = LayerScatterModes( - layer_input_mode=ScatterMode.SCATTERED, - attn_mode=ScatterMode.TP_ATTN_FULL, - mlp_mode=ScatterMode.FULL, - middle_residual_mode=ScatterMode.TP_ATTN_FULL, - layer_output_mode=ScatterMode.TP_ATTN_FULL, - ) - communicator = build(direct, dp) - self.assertIsNone(communicator._declared) - self.assertEqual( - communicator.input_rows, - comm.Layout(frozenset({TokenAxis.ATTN_DP, TokenAxis.ATTN_TP_SCATTER})), - ) - # Under attention CP the rows are those of a batch that holds every - # token on each CP rank, which the two-batch-overlap split takes. - cp = parallel_of(attn_dp=1, attn_tp=2, attn_cp=2) - communicator = build(direct, cp) - self.assertIsNone(communicator._declared) - slice_ = comm.Layout(frozenset({TokenAxis.ATTN_TP_SCATTER})) - self.assertEqual(communicator.input_rows, slice_) - pair = comm.CommunicateSummableTensorPairFn - with planning(cp): - self.assertEqual( - comm.tbo_split_moves(communicator.input_rows), - (pair._gather, pair._scatter), - ) - - -class TestTheLoraLayoutFollowsTheFfnInput(CustomTestCase): - """Under attention DP, LoRA routes the FFN's rows TP-global exactly when the - batch's FFN input step gathers them over attention DP.""" - - def build(self, modes, parallel, *, a2a, two_batch_overlap, declared): + def build(self, modes, parallel, *, a2a, two_batch_overlap): overlap = SimpleNamespace( overlap=SimpleNamespace(enable_two_batch_overlap=two_batch_overlap) ) with ( planning(parallel, a2a=a2a), patch.object(comm, "get_exec", lambda: overlap), - # Otherwise the scatter-mode steps, which a layer off the - # declarations runs. - nullcontext() - if declared - else patch.object( - LayerCommunicator, "_declared_sides", lambda self, **kwargs: None - ), ): communicator = LayerCommunicator( layer_scatter_modes=modes, @@ -951,7 +855,6 @@ class TestTheLoraLayoutFollowsTheFfnInput(CustomTestCase): a2a, dense_fully_dp, two_batch_overlap, - declared, ) in itertools.product( (1, 2, 4), (1, 2), @@ -961,7 +864,6 @@ class TestTheLoraLayoutFollowsTheFfnInput(CustomTestCase): (False, True), (False, True), (False, True), - (True, False), ): if two_batch_overlap and attn_dp == 1: continue @@ -974,7 +876,6 @@ class TestTheLoraLayoutFollowsTheFfnInput(CustomTestCase): a2a=a2a, dense_fully_dp=dense_fully_dp, two_batch_overlap=two_batch_overlap, - declared=declared, ): parallel = parallel_of( attn_dp=attn_dp, @@ -994,11 +895,10 @@ class TestTheLoraLayoutFollowsTheFfnInput(CustomTestCase): parallel, a2a=a2a, two_batch_overlap=two_batch_overlap, - declared=declared, ) steps = communicator._steps self.assertEqual( - steps.ffn_input_rows, communicator._context.layouts[modes.mlp_mode] + steps.ffn_input_rows, communicator._declared.ffn.layout ) published = {} with patch.object( @@ -1024,6 +924,23 @@ class TestTheAttentionOutputDecidesItsSum(CustomTestCase): """prepare_mlp completes the attention-TP sum exactly when the attention output's declaration says it is owed, whatever the attention-TP size.""" + def test_the_attention_output_owes_the_attention_tp_sum(self): + for attn_tp in (1, 2): + sides = sides_of( + { + TokenAxis.ATTN_DP: 2, + TokenAxis.ATTN_CP: 1, + TokenAxis.ATTN_TP_SCATTER: attn_tp, + } + ) + owed = attn_tp > 1 + self.assertIs( + sides.attention_output.group, SumGroup.ATTN_TP if owed else None + ) + self.assertIs(sides.attention_output.always_leaves, owed) + self.assertIs(sides.ffn_output.group, SumGroup.TP) + self.assertFalse(sides.ffn_output.always_leaves) + def run_steps(self, produced, *, force=False): sizes = { TokenAxis.ATTN_DP: 2, @@ -1492,52 +1409,13 @@ class TestOneRepresentation(CustomTestCase): "_mlp_input_may_return_new_residual", ) - def test_both_entries_hold_steps_only(self): - parallel = parallel_of(attn_dp=2, attn_tp=2) - direct = LayerScatterModes( - layer_input_mode=ScatterMode.TP_ATTN_FULL, - attn_mode=ScatterMode.TP_ATTN_FULL, - mlp_mode=ScatterMode.FULL, - middle_residual_mode=ScatterMode.TP_ATTN_FULL, - layer_output_mode=ScatterMode.TP_ATTN_FULL, - ) - for name, communicator in ( - ("declarations", build(layer_facts(1, 3), parallel)), - ("scatter modes", build(direct, parallel)), - ): - with self.subTest(name): - self.assertIsInstance(communicator._steps, comm.BoundarySteps) - for attribute in self.SEPARATE: - self.assertFalse(hasattr(communicator, attribute), attribute) - self.assertIs(communicator._batch_steps(None), communicator._steps) - # Under attention DP both bring the FFN output back to this - # rank's tokens with the step the FFN exit chooses. - self.assertTrue(communicator._steps.returns_over_dp) - - def test_the_scatter_mode_steps_complete_a_scattered_input(self): - # The scatter-mode path's input completion reads the batch fact itself. - steps = build( - LayerScatterModes( - layer_input_mode=ScatterMode.TP_ATTN_FULL, - attn_mode=ScatterMode.TP_ATTN_FULL, - mlp_mode=ScatterMode.FULL, - middle_residual_mode=ScatterMode.TP_ATTN_FULL, - layer_output_mode=ScatterMode.TP_ATTN_FULL, - ), - parallel_of(attn_dp=1, attn_tp=2), - )._steps - self.assertIs( - steps.attention_prepare.keywords["layer_input"], - comm._complete_scattered_input, - ) - hidden, residual = torch.ones(2, HIDDEN), torch.ones(2, HIDDEN) - with patch.object( - comm, "get_attn_tp_context", lambda: SimpleNamespace(input_scattered=False) - ): - self.assertEqual( - steps.attention_prepare.keywords["layer_input"](hidden, residual, None), - (hidden, residual), - ) + def test_the_layer_holds_steps_only(self): + communicator = build(layer_facts(1, 3), parallel_of(attn_dp=2, attn_tp=2)) + self.assertIsInstance(communicator._steps, comm.BoundarySteps) + for attribute in self.SEPARATE: + self.assertFalse(hasattr(communicator, attribute), attribute) + self.assertIs(communicator._batch_steps(None), communicator._steps) + self.assertTrue(communicator._steps.returns_over_dp) class TestTheAttentionInputHalf(CustomTestCase): @@ -1733,15 +1611,6 @@ class TestPrefillCP(CustomTestCase): dense = build(layer_facts(1, 3), parallel, allow_deferred_ffn_reduction=True) self.assertIs(dense._cp_steps.ffn_input.func, comm._mlp_input_gather_moe_cp) self.assertFalse(dense._cp_steps.ffn_output.leaves_for_next_layer) - # DSA and MLA CP gather the MoE's input over attention CP whatever its DP. - dsa = self.dsa_parallel(moe_dp_size=2, moe_tp_size=1) - modes = planned_modes( - 1, 3, sparse=True, previous_sparse=False, parallel=dsa, dsa_cp=True - ) - communicator = build(modes, dsa, dsa_cp=True, allow_reduce_scatter=True) - self.assertIs( - communicator._cp_steps.ffn_input.func, comm._mlp_input_gather_attention_cp - ) def test_the_fused_kernels_run_on_each_chunk(self): with planning(self.cp_parallel()): @@ -1756,97 +1625,97 @@ class TestPrefillCP(CustomTestCase): (communicator._mlp_input_reduce_output_and_update_and_read_residual,), ) - def test_which_cp_the_declarations_cover(self): - dp_cp = parallel_of(attn_dp=2, attn_tp=1, attn_cp=2, enable_prefill_cp=True) - dsa_dp_cp = parallel_of( - attn_dp=2, attn_tp=1, attn_cp=2, enable_prefill_cp=True, moe_dense_tp_size=1 + def cp_outcome(self, parallel, *, sparse=True, a2a=False, dsa_cp=False): + """A layer under attention CP: "cp steps" for the batches that shard + their tokens, "ordinary" when no batch does, or "refused".""" + modes = planned_modes( + 1, + 3, + sparse=sparse, + previous_sparse=sparse, + parallel=parallel, + a2a=a2a, + dsa_cp=dsa_cp, ) - for name, parallel, sparse, declared, dsa_cp, a2a in ( - ("DSA or MLA CP", self.dsa_parallel(), True, True, True, False), + try: + layer = build( + modes, parallel, a2a=a2a, dsa_cp=dsa_cp, allow_reduce_scatter=True + ) + except NotImplementedError: + return "refused" + return "ordinary" if layer._cp_steps is None else "cp steps" + + def test_which_cp_the_steps_cover(self): + dp_cp = dict(attn_dp=2, attn_tp=1, attn_cp=2, enable_prefill_cp=True) + moe_dp_eq_cp = dict(attn_cp=2, enable_prefill_cp=True, moe_dp_size=2) + dense = dict(sparse=False) + for expected, name, parallel, kwargs in ( + ("cp steps", "DSA or MLA CP", self.dsa_parallel(), dict(dsa_cp=True)), ( - "DSA or MLA CP, a MoE under attention DP", - dsa_dp_cp, - True, - True, - True, - True, + "cp steps", + "DSA or MLA CP, an a2a MoE under attention DP", + parallel_of(**dp_cp, moe_dense_tp_size=1), + dict(dsa_cp=True, a2a=True), ), - ("prefill CP", self.cp_parallel(), False, True, False, False), + ("cp steps", "prefill CP", self.cp_parallel(), dense), + ("cp steps", "CP under attention DP", parallel_of(**dp_cp), dense), ( + "cp steps", + "a MoE under attention DP and GQA CP", + parallel_of(**dp_cp), + {}, + ), + ( + "cp steps", + "MoE DP equal to CP", + parallel_of(attn_dp=1, attn_tp=2, **moe_dp_eq_cp), + {}, + ), + ( + "cp steps", + "a dense layer beside it under attention DP", + parallel_of(attn_dp=2, attn_tp=1, **moe_dp_eq_cp), + dense, + ), + ( + "ordinary", "CP without prefill CP", parallel_of(attn_dp=1, attn_tp=2, attn_cp=2), - False, - False, - False, - False, - ), - ("CP under attention DP", dp_cp, False, True, False, False), - ("a MoE under attention DP and GQA CP", dp_cp, True, True, False, False), - ( - "a MoE dispatched per DP shard under attention DP and GQA CP", - dp_cp, - True, - False, - False, - True, + {}, ), ( - "a MoE whose data-parallel groups are the CP ranks", - parallel_of( - attn_dp=1, - attn_tp=2, - attn_cp=2, - enable_prefill_cp=True, - moe_dp_size=2, - ), - True, - True, - False, - False, + "ordinary", + "an a2a MoE under attention DP, CP without prefill CP", + parallel_of(attn_dp=2, attn_tp=1, attn_cp=2), + dict(a2a=True), ), ( - "a MoE whose data-parallel groups are the CP ranks, under attention DP", - parallel_of( - attn_dp=2, - attn_tp=1, - attn_cp=2, - enable_prefill_cp=True, - moe_dp_size=2, - ), - True, - False, - False, - False, + "refused", + "DSA or MLA CP, MoE DP equal to CP", + self.dsa_parallel(moe_dp_size=2, moe_tp_size=1), + dict(dsa_cp=True), ), ( - "a MoE-CP group narrower than CP", - parallel_of( - attn_dp=1, - attn_tp=1, - attn_cp=4, - enable_prefill_cp=True, - moe_dp_size=2, - ), - False, - False, - False, - False, + "refused", + "an a2a MoE under attention DP and GQA CP", + parallel_of(**dp_cp), + dict(a2a=True), + ), + ( + "refused", + "MoE DP equal to CP under attention DP", + parallel_of(attn_dp=2, attn_tp=1, **moe_dp_eq_cp), + {}, + ), + ( + "refused", + "the same without prefill CP", + parallel_of(attn_dp=2, attn_tp=1, attn_cp=2, moe_dp_size=2), + {}, ), ): with self.subTest(name): - modes = planned_modes( - 1, - 3, - sparse=sparse, - previous_sparse=sparse, - parallel=parallel, - a2a=a2a, - dsa_cp=dsa_cp, - ) - communicator = build( - modes, parallel, a2a=a2a, dsa_cp=dsa_cp, allow_reduce_scatter=True - ) - self.assertIs(communicator._cp_steps is not None, declared) + self.assertEqual(self.cp_outcome(parallel, **kwargs), expected) def test_under_attention_dp_one_dp_sum_gathers_both_axes(self): parallel = parallel_of(attn_dp=2, attn_tp=2, attn_cp=2, enable_prefill_cp=True) diff --git a/test/registered/unit/model_executor/test_num_token_non_padded_localization.py b/test/registered/unit/model_executor/test_num_token_non_padded_localization.py index 3262fe7100..4a918b47c4 100644 --- a/test/registered/unit/model_executor/test_num_token_non_padded_localization.py +++ b/test/registered/unit/model_executor/test_num_token_non_padded_localization.py @@ -14,7 +14,7 @@ tensor and host-int twin agree, so a change to the sharding math fails loudly. a sparse MoE's input at all. Masking a gathered buffer with it truncates every peer rank's rows, which collapsed gsm8k accuracy under DP attention with ``--moe-a2a-backend none``, where each rank sums its own partial output. The -second table locks that scatter mode x layout x CP decision. +second table locks that MoE input x layout x CP decision. """ from sglang.test.ci.ci_register import register_cpu_ci @@ -22,13 +22,14 @@ from sglang.test.ci.ci_register import register_cpu_ci register_cpu_ci(est_time=10, suite="base-a-test-cpu") import unittest +from contextlib import contextmanager from types import SimpleNamespace from unittest.mock import patch import torch from sglang.srt.layers import communicator as comm -from sglang.srt.layers.communicator import ScatterMode +from sglang.srt.layers.moe import utils as moe_utils from sglang.srt.model_executor.forward_batch_info import ( ForwardBatch, ForwardMode, @@ -39,6 +40,25 @@ from sglang.srt.runtime_context import get_parallel from sglang.test.test_utils import CustomTestCase +@contextmanager +def sparse_moe_input(rows): + """Where a sparse MoE's input is: on each rank's local rows ("local": a2a + dispatch, FP4 all-gather or dwdp), on rows a GQA prefill CP gathers over the + MoE-CP group ("moe_cp", on a CP extend only), or on the FFN's rows ("ffn").""" + local = rows == "local" + with ( + patch.object( + moe_utils, "is_moe_input_scattered_across_dp_ranks", return_value=local + ), + patch.object( + comm, "is_moe_input_scattered_across_dp_ranks", return_value=local + ), + patch.object(comm, "is_enable_moe_cp_allgather", return_value=rows == "moe_cp"), + patch.object(comm, "_gathers_over_attention_cp", return_value=False), + ): + yield + + class TestNumTokenNonPaddedLayoutTable(CustomTestCase): # (label, attn_tp_size, sharded, padded_bucket, real-per-dp-group, # expected [per attn-tp rank] per dp group) @@ -116,20 +136,20 @@ def _value(batch: ForwardBatch): class TestMoeNumTokenNonPaddedTable(CustomTestCase): - # (label, mode, sharded, attn_dp_size, expected). Decode forwards, so the - # MOE_FULL rows follow the FULL layout: that config all-gathers over the - # MoE-CP group on a context-parallel extend only, which is the case below. + # (label, MoE input, sharded, attn_dp_size, expected). Decode forwards, so + # the MoE-CP rows follow the FFN rows' layout: that config all-gathers over + # the MoE-CP group on a context-parallel extend only, which is the case below. _TABLE = [ - ("scattered.replicated", ScatterMode.SCATTERED, False, 1, LOCAL), - ("scattered.sharded", ScatterMode.SCATTERED, True, 1, LOCAL), - ("scattered.dp", ScatterMode.SCATTERED, True, 4, LOCAL), - ("moe_full.decode.replicated", ScatterMode.MOE_FULL, False, 1, LOCAL), - ("moe_full.decode.sharded", ScatterMode.MOE_FULL, True, 1, GLOBAL), - ("moe_full.decode.dp", ScatterMode.MOE_FULL, True, 4, None), - ("full.dp_gathered", ScatterMode.FULL, True, 4, None), - ("full.dp_gathered.replicated", ScatterMode.FULL, False, 4, None), - ("full.tp_sharded", ScatterMode.FULL, True, 1, GLOBAL), - ("full.replicated", ScatterMode.FULL, False, 1, LOCAL), + ("scattered.replicated", "local", False, 1, LOCAL), + ("scattered.sharded", "local", True, 1, LOCAL), + ("scattered.dp", "local", True, 4, LOCAL), + ("moe_full.decode.replicated", "moe_cp", False, 1, LOCAL), + ("moe_full.decode.sharded", "moe_cp", True, 1, GLOBAL), + ("moe_full.decode.dp", "moe_cp", True, 4, None), + ("full.dp_gathered", "ffn", True, 4, None), + ("full.dp_gathered.replicated", "ffn", False, 4, None), + ("full.tp_sharded", "ffn", True, 1, GLOBAL), + ("full.replicated", "ffn", False, 1, LOCAL), ] def test_table(self): @@ -137,21 +157,19 @@ class TestMoeNumTokenNonPaddedTable(CustomTestCase): with ( self.subTest(case=label), get_parallel().override(attn_dp_size=attn_dp_size, attn_cp_size=1), - patch.object(comm, "sparse_mlp_scatter_mode", return_value=mode), + sparse_moe_input(mode), ): self.assertEqual(_value(_forward_batch(sharded=sharded)), expected) def test_cp_gathered_full_is_unmasked(self): - """A CP prefill must route every row: DSA / MLA CP take the FULL mode - but all-gather across CP, which zigzag-permutes the real rows.""" + """A CP prefill must route every row: DSA / MLA CP put a MoE on the + FFN rows but all-gather across CP, which zigzag-permutes the real rows.""" for sharded in (False, True): for dsa_cp, mla_cp in ((True, False), (False, True)): with ( self.subTest(sharded=sharded, dsa_cp=dsa_cp), get_parallel().override(attn_dp_size=1, attn_cp_size=2), - patch.object( - comm, "sparse_mlp_scatter_mode", return_value=ScatterMode.FULL - ), + sparse_moe_input("ffn"), patch( "sglang.srt.layers.attention.dsa.utils.dsa_use_prefill_cp", return_value=dsa_cp, @@ -177,9 +195,7 @@ class TestMoeNumTokenNonPaddedTable(CustomTestCase): with ( self.subTest(case=label), get_parallel().override(attn_dp_size=1, attn_cp_size=2), - patch.object( - comm, "sparse_mlp_scatter_mode", return_value=ScatterMode.MOE_FULL - ), + sparse_moe_input("moe_cp"), patch.object(comm, "get_moe_cp_size", return_value=2), patch( "sglang.srt.layers.attention.dsa.utils.dsa_use_prefill_cp", @@ -200,23 +216,21 @@ class TestMoeNumTokenNonPaddedTable(CustomTestCase): def test_graph_replay_without_global_count_skips_masking(self): """A captured batch carries the LOCAL buffer alone, so the attn-TP - sharded FULL case has no bound to mask with and must not use it.""" + sharded case on the FFN rows has no bound to mask with and must not use it.""" with ( get_parallel().override(attn_dp_size=1, attn_cp_size=1), - patch.object( - comm, "sparse_mlp_scatter_mode", return_value=ScatterMode.FULL - ), + sparse_moe_input("ffn"), ): self.assertIsNone(_value(_forward_batch(sharded=True, glob=None))) def test_absent_local_count_stays_absent(self): """Without expert parallelism the count is never filled, and routing must not mask every row against the unfilled buffer.""" - for mode in (ScatterMode.SCATTERED, ScatterMode.FULL, ScatterMode.MOE_FULL): + for mode in ("local", "ffn", "moe_cp"): with ( self.subTest(mode=mode), get_parallel().override(attn_dp_size=1, attn_cp_size=1), - patch.object(comm, "sparse_mlp_scatter_mode", return_value=mode), + sparse_moe_input(mode), ): self.assertIsNone(_value(_forward_batch(sharded=False, local=None)))