diff --git a/python/sglang/srt/layers/communicator.py b/python/sglang/srt/layers/communicator.py index e9f5ce2e06..c3e681bb86 100644 --- a/python/sglang/srt/layers/communicator.py +++ b/python/sglang/srt/layers/communicator.py @@ -741,6 +741,8 @@ class LayerCommunicator: self._communicate_summable_tensor_pair_fn is CommunicateSummableTensorPairFn._scatter_hidden_states ) + self._mlp_input, self._mlp_input_may_fuse = self._select_mlp_input() + self._attn_input_fusions = self._select_attn_input_fusions() self._speculative_algo = SpeculativeAlgorithm.from_string( get_spec().speculative_algorithm ) @@ -779,15 +781,6 @@ class LayerCommunicator: output_mode=self.layer_scatter_modes.attn_mode, context=self._context, ) - self._communicate_with_all_reduce_and_layer_norm_fn = ( - CommunicateWithAllReduceAndLayerNormFn.get_fn( - hidden_states_input_mode=self.layer_scatter_modes.attn_mode, - residual_input_mode=self.layer_scatter_modes.layer_input_mode, - hidden_states_output_mode=self.layer_scatter_modes.mlp_mode, - residual_output_mode=self.layer_scatter_modes.middle_residual_mode, - context=self._context, - ) - ) self._communicate_summable_tensor_pair_fn = ( CommunicateSummableTensorPairFn.get_fn( hidden_states_input_mode=self.layer_scatter_modes.mlp_mode, @@ -833,23 +826,14 @@ class LayerCommunicator: """True if ``prepare_mlp``'s post-attention RMSNorm leaves ``residual`` untouched, so Eagle3 aux capture can keep its reference and skip the clone. - Only the flashinfer all-reduce-fusion path writes a fresh ``residual_out`` - (see ``flashinfer_allreduce_residual_rmsnorm``); the aiter fused kernel and - every plain norm fold into ``residual`` in place. That path is reachable - only from the ``_gather_*`` communicate-fns, and only when they fall past - their input-scattered branch. + Only the flashinfer all-reduce-fusion kernel writes a fresh + ``residual_out`` (see ``flashinfer_allreduce_residual_rmsnorm``); the aiter + fused kernel and every plain norm fold into ``residual`` in place. It runs + when ``prepare_mlp`` selected its fused entry and the batch is not + input-scattered. """ - norm_fn = getattr( - self._communicate_with_all_reduce_and_layer_norm_fn, - "func", - self._communicate_with_all_reduce_and_layer_norm_fn, - ) - uses_gather_norm = norm_fn in ( - CommunicateWithAllReduceAndLayerNormFn._gather_hidden_states_and_residual, - CommunicateWithAllReduceAndLayerNormFn._gather_hidden_states_and_residual_moe, - ) return ( - uses_gather_norm + self._mlp_input_may_fuse and not get_attn_tp_context().input_scattered and apply_flashinfer_allreduce_fusion(residual.shape[0]) ) @@ -881,22 +865,28 @@ class LayerCommunicator: ): self.publish_attn_lora_layout() - if isinstance(hidden_states, UnreducedOutput) and residual is None: - raise RuntimeError("an UnreducedOutput requires residual input") + # What the previous layer left to complete: an UnreducedOutput, or a + # producer's handoff that one of the fused entries consumes. + owed = None if isinstance(hidden_states, torch.Tensor) else hidden_states + if owed is not None and residual is None: + raise RuntimeError(f"{type(owed).__name__} requires residual input") if ( - isinstance(hidden_states, UnreducedOutput) - and hidden_states.reduce_and_redistribute is not None + isinstance(owed, UnreducedOutput) + and owed.reduce_and_redistribute is not None ): # No fused kernel runs under attention DP: the reduce-scatter back to # this rank's tokens comes first. - hidden_states = reduce_output(hidden_states) - unreduced = ( - hidden_states if isinstance(hidden_states, UnreducedOutput) else None - ) - pending = unreduced is not None - if pending: - hidden_states = unreduced.partial - + hidden_states, owed = reduce_output(owed), None + if owed is not None: + for fused in self._attn_input_fusions: + result = fused(owed, residual, forward_batch, post_residual_addition) + if result is not None: + return self._finish_prepare_attn( + hidden_states=result[0], + residual=result[1], + forward_batch=forward_batch, + ) + hidden_states = owed.partial # residual is None marks the first decoder layer, where the SP region # opens: re-evaluated per forward so a crash mid-loop cannot leak into # the next one. @@ -922,23 +912,11 @@ class LayerCommunicator: ) if hidden_states.shape[0] == 0: residual = hidden_states - elif residual is not None and pending: - fused = ( - self._reduce_output_and_update_and_read_residual( - hidden_states, residual, forward_batch - ) - if post_residual_addition is None - # The fused kernel reduces over the MoE output's group. - and unreduced.group is post_experts_reduction_group() - else None - ) - if fused is not None: - hidden_states, residual = fused - else: - hidden_states = reduce_output(unreduced) - hidden_states, residual = _attn_input_update_and_read_residual( - quant_format - )(self.input_layernorm, hidden_states, residual, post_residual_addition) + elif owed is not None: + hidden_states = reduce_output(owed) + hidden_states, residual = _attn_input_update_and_read_residual( + quant_format + )(self.input_layernorm, hidden_states, residual, post_residual_addition) else: hidden_states, residual = _attn_input_update_and_read_residual( quant_format @@ -964,43 +942,63 @@ class LayerCommunicator: get_attn_tp_context().set_attn_inputs(attn_inputs) return hidden_states, residual + def _select_attn_input_fusions(self) -> Tuple[Callable, ...]: + """The fused kernels that complete what the previous layer left together + with the residual update and the input norm, in the order they are tried. + Each takes (owed, residual, forward_batch, post_residual_addition) and + returns None when it does not take the batch.""" + if not hasattr(self.input_layernorm, "forward_with_allreduce_fusion"): + return () + self._attn_input_fuses_quant = ( + self.enable_fused_ar_quant + and _use_aiter + and hasattr( + self.input_layernorm, "forward_with_allreduce_fusion_quant_per_group" + ) + ) + return (self._reduce_output_and_update_and_read_residual,) + def _reduce_output_and_update_and_read_residual( self, - hidden_states: torch.Tensor, + owed: UnreducedOutput, residual: torch.Tensor, forward_batch: ForwardBatch, + post_residual_addition: Optional[torch.Tensor], ) -> Optional[Tuple[torch.Tensor, torch.Tensor]]: """Complete the sum the previous layer left, add it to the residual and - apply the input norm in one fused kernel; None when the kernel does not - take this batch. The result is not quantized for ``quant_format``.""" + apply the input norm in one aiter or flashinfer kernel; None when the + kernel does not take this batch. The result is not quantized for + ``quant_format``.""" if ( + not isinstance(owed, UnreducedOutput) + # The kernel does not add it. + or post_residual_addition is not None + # The kernel reduces over the MoE output's group. + or owed.group is not post_experts_reduction_group() + ): + return None + hidden_states = owed.partial + if not ( apply_aiter_all_reduce_fusion(hidden_states, forward_batch) or apply_flashinfer_allreduce_fusion(hidden_states.shape[0]) - ) and hasattr(self.input_layernorm, "forward_with_allreduce_fusion"): - if ( - self.enable_fused_ar_quant - and _use_aiter - and hasattr( - self.input_layernorm, - "forward_with_allreduce_fusion_quant_per_group", + ): + return None + if self._attn_input_fuses_quant: + # Falls back to AR+RMSNorm + separate quant internally when the + # fully-fused kernel cannot service the shape. + quant_result = ( + self.input_layernorm.forward_with_allreduce_fusion_quant_per_group( + hidden_states, + residual, + use_attn_tp_group=False, + keep_bf16=self.fused_ar_quant_keep_bf16, ) - ): - # Falls back to AR+RMSNorm + separate quant internally when the - # fully-fused kernel cannot service the shape. - quant_result = ( - self.input_layernorm.forward_with_allreduce_fusion_quant_per_group( - hidden_states, - residual, - use_attn_tp_group=False, - keep_bf16=self.fused_ar_quant_keep_bf16, - ) - ) - if quant_result is not None: - return quant_result - return self.input_layernorm.forward_with_allreduce_fusion( - hidden_states, residual, use_attn_tp_group=False ) - return None + if quant_result is not None: + return quant_result + return self.input_layernorm.forward_with_allreduce_fusion( + hidden_states, residual, use_attn_tp_group=False + ) def _tp_reduce_scatter( self, @@ -1009,6 +1007,69 @@ class LayerCommunicator: ) -> Tuple[torch.Tensor, torch.Tensor]: return tp_reduce_scatter(hidden_states, residual, self._context) + def _select_mlp_input(self) -> Tuple[Callable, bool]: + """The attention-TP -> FFN boundary's steps, chosen from the layouts, and + whether they include the aiter / flashinfer fused entry. The MHC and + DSA-CP communicators pick their own implementations.""" + kind = mlp_input_kind(self.layer_scatter_modes, self._context) + residual_input_mode = self.layer_scatter_modes.layer_input_mode + if kind is MlpInputKind.NORM: + return _mlp_input_norm, False + if kind is MlpInputKind.ATTN_TP_ALL_REDUCE: + return _mlp_input_attn_tp_all_reduce, False + if kind is MlpInputKind.SCATTER: + return ( + partial( + _mlp_input_scatter, + scatters_residual=residual_input_mode == ScatterMode.TP_ATTN_FULL, + ), + False, + ) + # Neither fused kernel runs under attention DP. + fusions = ( + self._select_mlp_input_fusions(residual_input_mode) + if self._context.attn_dp_size == 1 + else () + ) + steps = partial( + _mlp_input_gather, + order=_mlp_input_order(self._context, residual_input_mode, fusions), + ) + if kind is MlpInputKind.GATHER_MOE_CP: + steps = partial(_mlp_input_gather_moe_cp, gather=steps) + return ( + steps, + self._mlp_input_reduce_output_and_update_and_read_residual in fusions, + ) + + def _select_mlp_input_fusions( + self, residual_input_mode: ScatterMode + ) -> Tuple[Callable, ...]: + """The fused kernels that complete the attention output's all-reduce + together with the residual update and the post-attention norm, in the + order they are tried. Each takes (hidden_states, residual, forward_batch) + and returns None when it does not take the batch.""" + if not hasattr(self.post_attention_layernorm, "forward_with_allreduce_fusion"): + return () + return (self._mlp_input_reduce_output_and_update_and_read_residual,) + + def _mlp_input_reduce_output_and_update_and_read_residual( + self, + hidden_states: torch.Tensor, + residual: torch.Tensor, + forward_batch: ForwardBatch, + ) -> Optional[Tuple[torch.Tensor, torch.Tensor]]: + """The attention-TP all-reduce, residual add and norm in one aiter or + flashinfer kernel; None when neither takes the batch.""" + if not ( + apply_aiter_all_reduce_fusion(hidden_states, forward_batch) + or apply_flashinfer_allreduce_fusion(hidden_states.shape[0]) + ): + return None + return self.post_attention_layernorm.forward_with_allreduce_fusion( + hidden_states, residual, use_attn_tp_group=True + ) + def prepare_mlp( self, hidden_states: torch.Tensor, @@ -1024,12 +1085,12 @@ class LayerCommunicator: if cache is not None: self._context.cache = cache - return self._communicate_with_all_reduce_and_layer_norm_fn( - hidden_states=hidden_states, - residual=residual, - forward_batch=forward_batch, - layernorm=self.post_attention_layernorm, - context=self._context, + return self._mlp_input( + hidden_states, + residual, + forward_batch, + self.post_attention_layernorm, + self._context, ) def maybe_prefetch_next_full_attention_kv( @@ -1515,25 +1576,6 @@ def _mlp_input_reduce_output( return attention_tensor_model_parallel_all_reduce(hidden_states) -def _mlp_input_reduce_output_and_update_and_read_residual( - hidden_states: torch.Tensor, - residual: torch.Tensor, - forward_batch: ForwardBatch, - layernorm: torch.nn.Module, -) -> Optional[Tuple[torch.Tensor, torch.Tensor]]: - """The attention-TP all-reduce, residual add and norm in one fused kernel; - None when no fused kernel takes the batch.""" - if ( - apply_aiter_all_reduce_fusion(hidden_states, forward_batch) - or apply_flashinfer_allreduce_fusion(hidden_states.shape[0]) - ) and hasattr(layernorm, "forward_with_allreduce_fusion"): - hidden_states, residual = layernorm.forward_with_allreduce_fusion( - hidden_states, residual, use_attn_tp_group=True - ) - return hidden_states, residual - return None - - def _redistribute_input_to_dp( hidden_states: torch.Tensor, forward_batch: ForwardBatch ) -> torch.Tensor: @@ -1550,259 +1592,284 @@ def _reduce_and_redistribute_output_to_dp( return global_hidden_states -class CommunicateWithAllReduceAndLayerNormFn: - """Besides communication, needs to - 1. All reduce in tp_attn_group on hidden_states - 2. Apply layer norm +def _mlp_input_without_dp( + hidden_states: torch.Tensor, + residual: torch.Tensor, + forward_batch: ForwardBatch, + layernorm: torch.nn.Module, + context: CommunicateContext, + *, + gathers_residual: bool, + fusions: Tuple[Callable, ...], +): + if gathers_residual: + residual = _redistribute_from_attn_tp_shards(residual) + for fused in fusions: + result = fused(hidden_states, residual, forward_batch) + if result is not None: + return result + hidden_states = _mlp_input_reduce_output(hidden_states, forward_batch) + if _is_npu and context.cache is not None: + _ = prepare_weight_cache(hidden_states, context.cache) + return layernorm(hidden_states, residual) + + +def _mlp_input_dp_replicate( + hidden_states: torch.Tensor, + residual: torch.Tensor, + forward_batch: ForwardBatch, + layernorm: torch.nn.Module, + context: CommunicateContext, + *, + gathers_residual: bool, +): + """Attention DP: reduce, add and normalize locally, then gather.""" + if gathers_residual: + residual = _redistribute_from_attn_tp_shards(residual) + if hidden_states.shape[0] != 0: + if context.attn_tp_size > 1: + hidden_states = attention_tensor_model_parallel_all_reduce(hidden_states) + with use_symmetric_memory( + get_parallel().tp_group, + disabled=not is_allocation_symmetric(), + ): + hidden_states, residual = layernorm(hidden_states, residual) + return _redistribute_input_to_dp(hidden_states, forward_batch), residual + + +def _mlp_input_dp_partial( + hidden_states: torch.Tensor, + residual: torch.Tensor, + forward_batch: ForwardBatch, + layernorm: torch.nn.Module, + context: CommunicateContext, + *, + gathers_residual: bool, +): + """Attention DP: one rank adds the residual, the gather sums it, then + normalize.""" + if gathers_residual: + residual = _redistribute_from_attn_tp_shards(residual) + if context.attn_tp_rank == 0: + hidden_states += residual + hidden_states = _reduce_and_redistribute_output_to_dp(hidden_states, forward_batch) + dp_scatter(residual, hidden_states, forward_batch) + if hidden_states.shape[0] != 0: + hidden_states = layernorm(hidden_states) + return hidden_states, residual + + +def _mlp_input_order( + context: CommunicateContext, + residual_input_mode: ScatterMode, + fusions: Tuple[Callable, ...], +) -> Callable: + """The steps from the attention output to the FFN input, chosen from facts + fixed at construction.""" + gathers_residual = ( + residual_input_mode == ScatterMode.SCATTERED and context.attn_tp_size > 1 + ) + if context.attn_dp_size == 1: + return partial( + _mlp_input_without_dp, gathers_residual=gathers_residual, fusions=fusions + ) + if context.force_layernorm_before_dp_gather or context.attn_tp_size == 1: + return partial(_mlp_input_dp_replicate, gathers_residual=gathers_residual) + return partial(_mlp_input_dp_partial, gathers_residual=gathers_residual) + + +class MlpInputKind(Enum): + """What the attention-TP -> FFN boundary does, given the two sides' layouts.""" + + # The layouts agree: add the residual and normalize. + NORM = auto() + # All-reduce over attention TP, normalize, and gather for the FFN group. + GATHER = auto() + # The same, then gather over the MoE-CP group (moe_dp_size < attn_cp_size). + GATHER_MOE_CP = auto() + # Reduce-scatter over attention TP to this rank's tokens, then normalize. + SCATTER = auto() + # All-reduce over attention TP for a dense MLP on that group. + ATTN_TP_ALL_REDUCE = auto() + + +def mlp_input_kind( + modes: LayerScatterModes, context: CommunicateContext +) -> MlpInputKind: + hidden_in, residual_in = modes.attn_mode, modes.layer_input_mode + hidden_out, residual_out = modes.mlp_mode, modes.middle_residual_mode + if ( + context.is_same_layout(hidden_in, hidden_out) + and context.is_same_layout(residual_in, residual_out) + and context.attn_tp_size == 1 + ): + return MlpInputKind.NORM + if ( + hidden_in == ScatterMode.SCATTERED + and residual_in == ScatterMode.SCATTERED + and hidden_out == ScatterMode.SCATTERED + and residual_out == ScatterMode.SCATTERED + ): + # Megatron LayerNorm sequence parallelism (layers/layernorm_sp.py): + # activations stay sequence-sharded across the attn->mlp boundary, so + # there is nothing to gather or scatter here -- just the residual add + # plus LayerNorm on the local shard. The row-parallel o_proj already + # issued the reduce-scatter (g-bar) that the all-reduce would have + # done, and the g all-gather is fused into the next column-parallel + # linear. Distinct from the branch above because under pure TP + # attn_tp_size == tp_size > 1, so that gate does not fire. + return MlpInputKind.NORM + if hidden_in == ScatterMode.TP_ATTN_FULL and residual_in in ( + ScatterMode.SCATTERED, + ScatterMode.TP_ATTN_FULL, + ): + kind = { + (ScatterMode.FULL, ScatterMode.TP_ATTN_FULL): MlpInputKind.GATHER, + ( + ScatterMode.MOE_FULL, + ScatterMode.TP_ATTN_FULL, + ): MlpInputKind.GATHER_MOE_CP, + (ScatterMode.SCATTERED, ScatterMode.SCATTERED): MlpInputKind.SCATTER, + }.get((hidden_out, residual_out)) + if kind is not None: + return kind + if ( + hidden_out == ScatterMode.TP_ATTN_FULL + and residual_out == ScatterMode.TP_ATTN_FULL + and context.attn_tp_size > 1 + ): + # Used when the dense MLP is tensor-parallelized along the + # attention TP group (``moe_dense_tp_size > 1``): hidden states + # need an all-reduce inside the attention TP group before the + # next layernorm, while staying in TP_ATTN_FULL on both sides. + return MlpInputKind.ATTN_TP_ALL_REDUCE + raise NotImplementedError( + f"{hidden_in=} {residual_in=} {hidden_out=} {residual_out=}" + ) + + +def _mlp_input_norm( + hidden_states: torch.Tensor, + residual: torch.Tensor, + forward_batch: ForwardBatch, + layernorm: torch.nn.Module, + context: CommunicateContext, +): + # TODO move these `if shape != 0` into LayerNorm itself + if hidden_states.shape[0] != 0: + hidden_states, residual = layernorm(hidden_states, residual) + return hidden_states, residual + + +def _mlp_input_attn_tp_all_reduce( + hidden_states: torch.Tensor, + residual: torch.Tensor, + forward_batch: ForwardBatch, + layernorm: torch.nn.Module, + context: CommunicateContext, +): + """All-reduce hidden states inside the attention TP group, then layernorm. + + Used when the dense MLP shares the attention TP group + (``moe_dense_tp_size > 1``): both hidden states and residual stay in + ``TP_ATTN_FULL`` across the boundary. """ + hidden_states = get_parallel().attn_tp_group.all_reduce(hidden_states) + if hidden_states.shape[0] != 0: + hidden_states, residual = layernorm(hidden_states, residual) + return hidden_states, residual - @staticmethod - def get_fn( - hidden_states_input_mode: ScatterMode, - residual_input_mode: ScatterMode, - hidden_states_output_mode: ScatterMode, - residual_output_mode: ScatterMode, - context: CommunicateContext, + +def _mlp_input_scatter( + hidden_states: torch.Tensor, + residual: torch.Tensor, + forward_batch: ForwardBatch, + layernorm: torch.nn.Module, + context: CommunicateContext, + *, + scatters_residual: bool, +): + hidden_states = _reduce_and_redistribute_output_to_attn_tp_shards( + hidden_states, context + ) + if scatters_residual: + residual = _redistribute_to_attn_tp_shards(residual, context) + if hidden_states.shape[0] != 0: + hidden_states, residual = layernorm(hidden_states, residual) + return hidden_states, residual + + +def _tp_all_reduce_with_scattered_residual( + hidden_states: torch.Tensor, + residual: torch.Tensor, + layernorm: torch.nn.Module, + context: CommunicateContext, +): + if hidden_states.shape[0] == 0: + return hidden_states, hidden_states + + scattered_states = hidden_states.tensor_split(context.tp_size)[context.tp_rank] + scattered_states += residual + residual = tensor_model_parallel_all_reduce(hidden_states) + hidden_states = layernorm(residual) + return hidden_states, residual + + +def _mlp_input_gather( + hidden_states: torch.Tensor, + residual: torch.Tensor, + forward_batch: ForwardBatch, + layernorm: torch.nn.Module, + context: CommunicateContext, + *, + order: Callable, +): + """Run ``order``, the steps ``_mlp_input_order`` chose at construction, + unless this batch's attention input is scattered.""" + if get_attn_tp_context().input_scattered: + return _tp_all_reduce_with_scattered_residual( + hidden_states, residual, layernorm, context + ) + return order(hidden_states, residual, forward_batch, layernorm, context) + + +def _mlp_input_gather_moe_cp( + hidden_states: torch.Tensor, + residual: torch.Tensor, + forward_batch: ForwardBatch, + layernorm: torch.nn.Module, + context: CommunicateContext, + *, + gather: Callable, +): + """Gather for the FFN, then over the MoE-CP group so each rank holds all + tokens of its MoE group (moe_dp_size < attn_cp_size). The residual stays at + TP_ATTN_FULL.""" + # Early return on empty tensor is safe for MOE_CP because: + # - During CP extend: zigzag split guarantees all CP ranks have non-zero tokens, + # so no rank hits this path while others proceed to the allgather. + # - During decode: moe_cp allgather is skipped (guarded by is_context_parallel_extend). + # - CUDA graph warmup: not applicable when --cuda-graph-backend-prefill=disabled is used. + if hidden_states.shape[0] == 0: + return hidden_states, residual + + hidden_states, residual = gather( + hidden_states, residual, forward_batch, layernorm, context + ) + + # Only active during prefill (context-parallel extend); decode keeps existing path. + moe_cp_size = get_moe_cp_size() + if ( + moe_cp_size > 1 + and hidden_states.shape[0] > 0 + and forward_batch.forward_mode.is_context_parallel_extend() + and forward_batch.attn_cp_metadata is not None ): - - if ( - context.is_same_layout(hidden_states_input_mode, hidden_states_output_mode) - and context.is_same_layout(residual_input_mode, residual_output_mode) - and context.attn_tp_size == 1 - ): - return CommunicateWithAllReduceAndLayerNormFn._simple - - if ( - hidden_states_input_mode == ScatterMode.SCATTERED - and residual_input_mode == ScatterMode.SCATTERED - and hidden_states_output_mode == ScatterMode.SCATTERED - and residual_output_mode == ScatterMode.SCATTERED - ): - # Megatron LayerNorm sequence parallelism (layers/layernorm_sp.py): - # activations stay sequence-sharded across the attn->mlp boundary, so - # there is nothing to gather or scatter here -- just the residual add - # plus LayerNorm on the local shard. The row-parallel o_proj already - # issued the reduce-scatter (g-bar) that the all-reduce would have - # done, and the g all-gather is fused into the next column-parallel - # linear. Distinct from the branch above because under pure TP - # attn_tp_size == tp_size > 1, so that gate does not fire. - return CommunicateWithAllReduceAndLayerNormFn._simple - - if hidden_states_input_mode == ScatterMode.TP_ATTN_FULL and ( - residual_input_mode in (ScatterMode.SCATTERED, ScatterMode.TP_ATTN_FULL) - ): - fn = { - ( - ScatterMode.FULL, - ScatterMode.TP_ATTN_FULL, - ): CommunicateWithAllReduceAndLayerNormFn._gather_hidden_states_and_residual, - ( - ScatterMode.MOE_FULL, - ScatterMode.TP_ATTN_FULL, - ): CommunicateWithAllReduceAndLayerNormFn._gather_hidden_states_and_residual_moe, - ( - ScatterMode.SCATTERED, - ScatterMode.SCATTERED, - ): CommunicateWithAllReduceAndLayerNormFn._scatter_hidden_states_and_residual, - }.get((hidden_states_output_mode, residual_output_mode)) - if fn is not None: - return partial(fn, residual_input_mode=residual_input_mode) - - if ( - hidden_states_output_mode == ScatterMode.TP_ATTN_FULL - and residual_output_mode == ScatterMode.TP_ATTN_FULL - and context.attn_tp_size > 1 - ): - # Used when the dense MLP is tensor-parallelized along the - # attention TP group (``moe_dense_tp_size > 1``): hidden states - # need an all-reduce inside the attention TP group before the - # next layernorm, while staying in TP_ATTN_FULL on both sides. - return CommunicateWithAllReduceAndLayerNormFn._tp_attn_all_reduce_and_layernorm - - raise NotImplementedError( - f"{hidden_states_input_mode=} {residual_input_mode=} {hidden_states_output_mode=} {residual_output_mode=}" + hidden_states = _redistribute_input_to_moe_cp( + hidden_states, forward_batch, moe_cp_size ) - @staticmethod - def _simple( - hidden_states: torch.Tensor, - residual: torch.Tensor, - forward_batch: ForwardBatch, - layernorm: torch.nn.Module, - context: CommunicateContext, - ): - # TODO move these `if shape != 0` into LayerNorm itself - if hidden_states.shape[0] != 0: - hidden_states, residual = layernorm(hidden_states, residual) - return hidden_states, residual - - @staticmethod - def _tp_attn_all_reduce_and_layernorm( - hidden_states: torch.Tensor, - residual: torch.Tensor, - forward_batch: ForwardBatch, - layernorm: torch.nn.Module, - context: CommunicateContext, - ): - """All-reduce hidden states inside the attention TP group, then layernorm. - - Used when the dense MLP shares the attention TP group - (``moe_dense_tp_size > 1``): both hidden states and residual stay in - ``TP_ATTN_FULL`` across the boundary. - """ - hidden_states = get_parallel().attn_tp_group.all_reduce(hidden_states) - if hidden_states.shape[0] != 0: - hidden_states, residual = layernorm(hidden_states, residual) - return hidden_states, residual - - @staticmethod - def _gather_hidden_states_and_residual( - hidden_states: torch.Tensor, - residual: torch.Tensor, - forward_batch: ForwardBatch, - layernorm: torch.nn.Module, - context: CommunicateContext, - *, - residual_input_mode, - ): - if get_attn_tp_context().input_scattered: - return CommunicateWithAllReduceAndLayerNormFn._tp_all_reduce_with_scattered_residual( - hidden_states, - residual, - layernorm, - context, - ) - - if residual_input_mode == ScatterMode.SCATTERED and context.attn_tp_size > 1: - residual = _redistribute_from_attn_tp_shards(residual) - if context.attn_dp_size == 1: - fused = _mlp_input_reduce_output_and_update_and_read_residual( - hidden_states, residual, forward_batch, layernorm - ) - if fused is not None: - return fused - hidden_states = _mlp_input_reduce_output(hidden_states, forward_batch) - if _is_npu and context.cache is not None: - _ = prepare_weight_cache(hidden_states, context.cache) - return layernorm(hidden_states, residual) - - # Attention DP. Replicate: reduce, add and normalize locally, then gather. - # Partial: one rank adds the residual, the gather sums it, then normalize. - replicate = ( - context.force_layernorm_before_dp_gather or context.attn_tp_size == 1 - ) - if replicate and hidden_states.shape[0] != 0: - if context.attn_tp_size > 1: - hidden_states = attention_tensor_model_parallel_all_reduce( - hidden_states - ) - with use_symmetric_memory( - get_parallel().tp_group, - disabled=not is_allocation_symmetric(), - ): - hidden_states, residual = layernorm(hidden_states, residual) - elif context.attn_tp_rank == 0: - hidden_states += residual - if replicate: - return _redistribute_input_to_dp(hidden_states, forward_batch), residual - hidden_states = _reduce_and_redistribute_output_to_dp( - hidden_states, forward_batch - ) - dp_scatter(residual, hidden_states, forward_batch) - if hidden_states.shape[0] != 0: - hidden_states = layernorm(hidden_states) - return hidden_states, residual - - @staticmethod - def _scatter_hidden_states_and_residual( - hidden_states: torch.Tensor, - residual: torch.Tensor, - forward_batch: ForwardBatch, - layernorm: torch.nn.Module, - context: CommunicateContext, - *, - residual_input_mode, - ): - hidden_states = _reduce_and_redistribute_output_to_attn_tp_shards( - hidden_states, context - ) - if residual_input_mode == ScatterMode.TP_ATTN_FULL: - residual = _redistribute_to_attn_tp_shards(residual, context) - if hidden_states.shape[0] != 0: - hidden_states, residual = layernorm(hidden_states, residual) - return hidden_states, residual - - @staticmethod - def _tp_all_reduce_with_scattered_residual( - hidden_states: torch.Tensor, - residual: torch.Tensor, - layernorm: torch.nn.Module, - context: CommunicateContext, - ): - if hidden_states.shape[0] == 0: - return hidden_states, hidden_states - - scattered_states = hidden_states.tensor_split(context.tp_size)[context.tp_rank] - scattered_states += residual - residual = tensor_model_parallel_all_reduce(hidden_states) - hidden_states = layernorm(residual) - return hidden_states, residual - - @staticmethod - def _gather_hidden_states_and_residual_moe( - hidden_states: torch.Tensor, - residual: torch.Tensor, - forward_batch, - layernorm: torch.nn.Module, - context: CommunicateContext, - *, - residual_input_mode, - ): - """Allgather tokens for MoE when moe_dp_size < attn_cp_size. - - Steps: - 1. Standard attn-TP all-reduce + optional DP allgather + layernorm (same as - _gather_hidden_states_and_residual for the dp>1 case, or simple all-reduce - + layernorm for dp==1). - 2. moe_cp allgather: gather tokens from cp_per_moe CP ranks so each rank holds - all tokens for its MoE group. - - Residual is left at TP_ATTN_FULL throughout. - """ - # Early return on empty tensor is safe for MOE_CP because: - # - During CP extend: zigzag split guarantees all CP ranks have non-zero tokens, - # so no rank hits this path while others proceed to the allgather. - # - During decode: moe_cp allgather is skipped (guarded by is_context_parallel_extend). - # - CUDA graph warmup: not applicable when --cuda-graph-backend-prefill=disabled is used. - if hidden_states.shape[0] == 0: - return hidden_states, residual - - # Step 1: Standard all-reduce/DP-allgather + layernorm (reuse existing logic). - hidden_states, residual = ( - CommunicateWithAllReduceAndLayerNormFn._gather_hidden_states_and_residual( - hidden_states=hidden_states, - residual=residual, - forward_batch=forward_batch, - layernorm=layernorm, - context=context, - residual_input_mode=residual_input_mode, - ) - ) - - # Step 2: moe_cp allgather — gather across cp_per_moe CP ranks. - # Only active during prefill (context-parallel extend); decode keeps existing path. - moe_cp_size = get_moe_cp_size() - if ( - moe_cp_size > 1 - and hidden_states.shape[0] > 0 - and forward_batch.forward_mode.is_context_parallel_extend() - and forward_batch.attn_cp_metadata is not None - ): - hidden_states = _redistribute_input_to_moe_cp( - hidden_states, forward_batch, moe_cp_size - ) - - return hidden_states, residual + return hidden_states, residual def _dp_scatter_group() -> GroupCoordinator: @@ -2022,7 +2089,7 @@ class CommunicateSummableTensorPairFn: If DP>1, further scatter back to the local DP slice. """ # Only scatter back during prefill; decode was never allgathered so no-op. - # Safe w.r.t. empty tensors: same reasoning as _gather_hidden_states_and_residual_moe + # Safe w.r.t. empty tensors: same reasoning as _mlp_input_gather_moe_cp # — CP extend always has non-zero tokens per rank, and decode skips this path. moe_cp_size = get_moe_cp_size() if ( diff --git a/python/sglang/srt/layers/communicator_dsa_cp.py b/python/sglang/srt/layers/communicator_dsa_cp.py index 1b2e5c3f42..ee683c118d 100644 --- a/python/sglang/srt/layers/communicator_dsa_cp.py +++ b/python/sglang/srt/layers/communicator_dsa_cp.py @@ -25,10 +25,10 @@ from sglang.srt.layers.communicator import ( CommunicateContext, CommunicateSimpleFn, CommunicateSummableTensorPairFn, - CommunicateWithAllReduceAndLayerNormFn, LayerCommunicator, LayerScatterModes, ScatterMode, + _mlp_input_norm, ) from sglang.srt.layers.cp.utils import is_mla_cp_active from sglang.srt.layers.dp_attention import ( @@ -113,13 +113,6 @@ class DSACPLayerCommunicator(LayerCommunicator): output_mode=ScatterMode.SCATTERED, context=self._context, ) - self._communicate_with_all_reduce_and_layer_norm_fn = DSACPCommunicateWithAllReduceAndLayerNormFn.get_fn( - hidden_states_input_mode=ScatterMode.SCATTERED, - residual_input_mode=ScatterMode.SCATTERED, - hidden_states_output_mode=self.layer_scatter_modes.mlp_mode, # SCATTERED, FULL - residual_output_mode=ScatterMode.SCATTERED, - context=self._context, - ) self._communicate_summable_tensor_pair_fn = DSACPCommunicateSummableTensorPairFn.get_fn( hidden_states_input_mode=self.layer_scatter_modes.mlp_mode, # SCATTERED, FULL residual_input_mode=ScatterMode.SCATTERED, @@ -127,6 +120,16 @@ class DSACPLayerCommunicator(LayerCommunicator): context=self._context, ) + def _select_mlp_input(self): + fn = DSACPCommunicateWithAllReduceAndLayerNormFn.get_fn( + hidden_states_input_mode=ScatterMode.SCATTERED, + residual_input_mode=ScatterMode.SCATTERED, + hidden_states_output_mode=self.layer_scatter_modes.mlp_mode, # SCATTERED, FULL + residual_output_mode=ScatterMode.SCATTERED, + context=self._context, + ) + return fn, False + class DSACPCommunicateSimpleFn(CommunicateSimpleFn): @staticmethod @@ -141,9 +144,7 @@ class DSACPCommunicateSimpleFn(CommunicateSimpleFn): raise NotImplementedError(f"{input_mode=} {output_mode=}") -class DSACPCommunicateWithAllReduceAndLayerNormFn( - CommunicateWithAllReduceAndLayerNormFn -): +class DSACPCommunicateWithAllReduceAndLayerNormFn: """Besides communication, needs to 1. All reduce in tp_attn_group on hidden_states 2. Apply layer norm @@ -161,7 +162,7 @@ class DSACPCommunicateWithAllReduceAndLayerNormFn( assert residual_input_mode == ScatterMode.SCATTERED assert residual_output_mode == ScatterMode.SCATTERED if hidden_states_output_mode == ScatterMode.SCATTERED: - return DSACPCommunicateWithAllReduceAndLayerNormFn._simple + return _mlp_input_norm if hidden_states_output_mode == ScatterMode.FULL: return partial( diff --git a/python/sglang/srt/layers/communicator_mhc.py b/python/sglang/srt/layers/communicator_mhc.py index 6d1b4ec1d4..e8c3052b60 100644 --- a/python/sglang/srt/layers/communicator_mhc.py +++ b/python/sglang/srt/layers/communicator_mhc.py @@ -29,11 +29,12 @@ from sglang.srt.layers.communicator import ( CommunicateContext, CommunicateSimpleFn, CommunicateSummableTensorPairFn, - CommunicateWithAllReduceAndLayerNormFn, LayerCommunicator, LayerScatterModes, + MlpInputKind, ScatterMode, get_attn_tp_context, + mlp_input_kind, tp_reduce_scatter, ) from sglang.srt.layers.dp_attention import ( @@ -130,35 +131,7 @@ class MHCState: self.h_post = None -class MHCCommunicateWithAllReduceAndLayerNormFn(CommunicateWithAllReduceAndLayerNormFn): - @staticmethod - def get_fn( - hidden_states_input_mode: ScatterMode, - residual_input_mode: ScatterMode, - hidden_states_output_mode: ScatterMode, - residual_output_mode: ScatterMode, - context: CommunicateContext, - ): - fn = CommunicateWithAllReduceAndLayerNormFn.get_fn( - hidden_states_input_mode, - residual_input_mode, - hidden_states_output_mode, - residual_output_mode, - context, - ) - replacements = { - CommunicateWithAllReduceAndLayerNormFn._simple: MHCCommunicateWithAllReduceAndLayerNormFn._simple, - CommunicateWithAllReduceAndLayerNormFn._gather_hidden_states_and_residual: MHCCommunicateWithAllReduceAndLayerNormFn._gather_hidden_states_and_residual, - CommunicateWithAllReduceAndLayerNormFn._scatter_hidden_states_and_residual: MHCCommunicateWithAllReduceAndLayerNormFn._scatter_hidden_states_and_residual, - } - if isinstance(fn, partial): - return partial( - replacements.get(fn.func, fn.func), - *fn.args, - **(fn.keywords or {}), - ) - return replacements.get(fn, fn) - +class MHCCommunicateWithAllReduceAndLayerNormFn: @staticmethod def _scatter_hidden_states_and_residual( hidden_states: torch.Tensor, @@ -456,15 +429,6 @@ class MHCLayerCommunicator(LayerCommunicator): output_mode=self.layer_scatter_modes.attn_mode, context=self._context, ) - self._communicate_with_all_reduce_and_layer_norm_fn = ( - MHCCommunicateWithAllReduceAndLayerNormFn.get_fn( - hidden_states_input_mode=self.layer_scatter_modes.attn_mode, - residual_input_mode=self.layer_scatter_modes.layer_input_mode, - hidden_states_output_mode=self.layer_scatter_modes.mlp_mode, - residual_output_mode=self.layer_scatter_modes.middle_residual_mode, - context=self._context, - ) - ) self._communicate_summable_tensor_pair_fn = ( MHCCommunicateSummableTensorPairFn.get_fn( hidden_states_input_mode=self.layer_scatter_modes.mlp_mode, @@ -519,27 +483,21 @@ class MHCLayerCommunicator(LayerCommunicator): return hidden_states, residual - def prepare_mlp( - self, - hidden_states: torch.Tensor, - residual: torch.Tensor, - forward_batch: ForwardBatch, - cache=None, - ): - self.publish_mlp_lora_layout() - if cache is not None: - self._context.cache = cache - - hidden_states, residual = self._communicate_with_all_reduce_and_layer_norm_fn( - hidden_states=hidden_states, - residual=residual, - forward_batch=forward_batch, - layernorm=self.post_attention_layernorm, - context=self._context, - mhc=self.mhc, - ) - - return hidden_states, residual + def _select_mlp_input(self): + """MHC's own implementation of each boundary kind, applied to this + layer's MHC state.""" + kind = mlp_input_kind(self.layer_scatter_modes, self._context) + residual_input_mode = self.layer_scatter_modes.layer_input_mode + fns = MHCCommunicateWithAllReduceAndLayerNormFn + if kind is MlpInputKind.NORM: + return partial(fns._simple, mhc=self.mhc), False + if kind is MlpInputKind.GATHER: + fn = fns._gather_hidden_states_and_residual + elif kind is MlpInputKind.SCATTER: + fn = fns._scatter_hidden_states_and_residual + else: + raise NotImplementedError(f"MHCLayerCommunicator does not support {kind}") + return partial(fn, residual_input_mode=residual_input_mode, mhc=self.mhc), False def postprocess_layer(self, hidden_states, residual, forward_batch): hidden_states, residual = self._communicate_summable_tensor_pair_fn( diff --git a/python/sglang/srt/layers/moe/cutedsl_ar_fusion.py b/python/sglang/srt/layers/moe/cutedsl_ar_fusion.py index b3cd34f919..3e53da15b6 100644 --- a/python/sglang/srt/layers/moe/cutedsl_ar_fusion.py +++ b/python/sglang/srt/layers/moe/cutedsl_ar_fusion.py @@ -7,7 +7,6 @@ shared-expert add folded in when the runner hands back a MoeFinalizeHandoff. from __future__ import annotations -import functools import logging from typing import Callable, Optional, Sequence @@ -15,7 +14,6 @@ import msgspec import torch from sglang.srt.layers.communicator import ( - CommunicateWithAllReduceAndLayerNormFn, LayerCommunicator, ScatterMode, UnreducedOutput, @@ -187,112 +185,87 @@ class CuteDSLFusionLayerCommunicator(LayerCommunicator): # A replicated output follows the reduction; moving it would scale that by tp. owes_local_reduction: bool = False - def prepare_attn( - self, - hidden_states, - residual, - forward_batch, - quant_format: str = "", - post_residual_addition=None, - ): - if isinstance(hidden_states, MoeFinalizeHandoff): - if not self._should_use_finalize(forward_batch, hidden_states.m): - raise RuntimeError( - "received deferred MoE output on an ineligible path " - f"(M={hidden_states.m}, mode={forward_batch.forward_mode})" - ) - if residual is None: - raise RuntimeError("deferred MoE finalize requires residual input") - gamma = _fused_norm_gamma(self.input_layernorm) - if gamma is None: - raise RuntimeError( - "deferred MoE finalize requires a fusable RMSNorm flavour" - ) - if post_residual_addition is not None: - residual = residual + post_residual_addition - assert self.fusion_service is not None - hidden_states, residual = self.fusion_service.finalize( - handoff=hidden_states, residual=residual, gamma=gamma - ) - return self._finish_prepare_attn( - hidden_states=hidden_states, - residual=residual, - forward_batch=forward_batch, - ) - - if ( - residual is not None - and isinstance(hidden_states, UnreducedOutput) - and self._can_consume_post_moe_all_reduce( - forward_batch, int(hidden_states.partial.shape[0]) - ) - ): - if post_residual_addition is not None: - residual = residual + post_residual_addition - assert self.fusion_service is not None - hidden_states, residual = self.fusion_service.all_reduce_residual_rms_norm( - local_contribution=hidden_states.partial, - residual=residual, - gamma=_fused_norm_gamma(self.input_layernorm), - ) - return self._finish_prepare_attn( - hidden_states=hidden_states, - residual=residual, - forward_batch=forward_batch, - ) - - return super().prepare_attn( - hidden_states, - residual, - forward_batch, - quant_format=quant_format, - post_residual_addition=post_residual_addition, + def _select_attn_input_fusions(self): + return ( + self._finalize_output_and_update_and_read_residual_cutedsl, + self._reduce_output_and_update_and_read_residual_cutedsl, + *super()._select_attn_input_fusions(), ) - def prepare_mlp( - self, - hidden_states: torch.Tensor, - residual: torch.Tensor, - forward_batch: ForwardBatch, - cache=None, + def _finalize_output_and_update_and_read_residual_cutedsl( + self, owed, residual, forward_batch, post_residual_addition ): - if cache is not None: - self._context.cache = cache - if self._should_use_all_reduce_rms_norm( - forward_batch, int(hidden_states.shape[0]), residual - ): - assert self.fusion_service is not None and residual is not None - return self.fusion_service.all_reduce_residual_rms_norm( - local_contribution=hidden_states, - residual=residual, - gamma=_fused_norm_gamma(self.post_attention_layernorm), + """Finish a deferred MoE finalize with the all-reduce, residual add and + input norm.""" + if not isinstance(owed, MoeFinalizeHandoff): + return None + if not self._should_use_finalize(forward_batch, owed.m): + raise RuntimeError( + "received deferred MoE output on an ineligible path " + f"(M={owed.m}, mode={forward_batch.forward_mode})" ) - return super().prepare_mlp(hidden_states, residual, forward_batch, cache=cache) + gamma = _fused_norm_gamma(self.input_layernorm) + if gamma is None: + raise RuntimeError( + "deferred MoE finalize requires a fusable RMSNorm flavour" + ) + if post_residual_addition is not None: + residual = residual + post_residual_addition + assert self.fusion_service is not None + return self.fusion_service.finalize( + handoff=owed, residual=residual, gamma=gamma + ) - def _should_use_all_reduce_rms_norm( - self, - forward_batch: ForwardBatch, - m: int, - residual: Optional[torch.Tensor], - ) -> bool: - communicate_fn = self._communicate_with_all_reduce_and_layer_norm_fn - if isinstance(communicate_fn, functools.partial): - norm_fn = communicate_fn.func - residual_input_mode = communicate_fn.keywords.get("residual_input_mode") - else: - norm_fn = communicate_fn - residual_input_mode = None + def _reduce_output_and_update_and_read_residual_cutedsl( + self, owed, residual, forward_batch, post_residual_addition + ): + """Complete the all-reduce the previous layer left with the residual add + and input norm.""" + if not isinstance(owed, UnreducedOutput) or not ( + self._can_consume_post_moe_all_reduce( + forward_batch, int(owed.partial.shape[0]) + ) + ): + return None + if post_residual_addition is not None: + residual = residual + post_residual_addition + assert self.fusion_service is not None + return self.fusion_service.all_reduce_residual_rms_norm( + local_contribution=owed.partial, + residual=residual, + gamma=_fused_norm_gamma(self.input_layernorm), + ) + + def _select_mlp_input_fusions(self, residual_input_mode): + fusions = super()._select_mlp_input_fusions(residual_input_mode) parallel = get_parallel() - return ( - self._common_eligible(forward_batch, m) - and residual is not None - and _fused_norm_gamma(self.post_attention_layernorm) is not None - and norm_fn - is CommunicateWithAllReduceAndLayerNormFn._gather_hidden_states_and_residual - and residual_input_mode is ScatterMode.TP_ATTN_FULL - and self._context.attn_dp_size == 1 + if ( + residual_input_mode is ScatterMode.TP_ATTN_FULL and parallel.attn_tp_size == parallel.tp_size + and _fused_norm_gamma(self.post_attention_layernorm) is not None + ): + return ( + self._mlp_input_reduce_output_and_update_and_read_residual_cutedsl, + *fusions, + ) + return fusions + + def _mlp_input_reduce_output_and_update_and_read_residual_cutedsl( + self, hidden_states, residual, forward_batch + ): + """The attention output's all-reduce with the residual add and the + post-attention norm.""" + if not ( + self._common_eligible(forward_batch, int(hidden_states.shape[0])) + and residual is not None and not get_exec().comm.enable_quant_communications + ): + return None + assert self.fusion_service is not None + return self.fusion_service.all_reduce_residual_rms_norm( + local_contribution=hidden_states, + residual=residual, + gamma=_fused_norm_gamma(self.post_attention_layernorm), ) def _should_use_finalize(self, forward_batch: ForwardBatch, m: int) -> bool: diff --git a/python/sglang/srt/models/deepseek_v2.py b/python/sglang/srt/models/deepseek_v2.py index 896fcb7979..3c87e70516 100644 --- a/python/sglang/srt/models/deepseek_v2.py +++ b/python/sglang/srt/models/deepseek_v2.py @@ -23,7 +23,7 @@ from __future__ import annotations import logging from contextlib import contextmanager, nullcontext from functools import cached_property -from typing import Any, Dict, Iterable, List, Optional, Tuple, Union +from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple, Union import torch import torch.nn.functional as F @@ -2616,9 +2616,24 @@ class DeepseekV2DecoderLayer(nn.Module): self._gfx95_quant_format = self._detect_gfx95_quant_format() + self.layer_communicator = self._build_layer_communicator( + input_layernorm=self.input_layernorm, + post_attention_layernorm=self.post_attention_layernorm, + qkv_latent_func=self.self_attn.prepare_qkv_latent, + ) + + def _build_layer_communicator( + self, + *, + input_layernorm: nn.Module, + post_attention_layernorm: nn.Module, + qkv_latent_func: Optional[Callable], + ): + """The communicator for this layer's norms; it chooses its boundary + steps from them at construction.""" if get_parallel().enable_prefill_cp: communicator_cls = DSACPLayerCommunicator - elif not is_nextn and _use_mnnvl_cutedsl_fusion(): + elif not self.is_nextn and _use_mnnvl_cutedsl_fusion(): # Dense layers too: selecting cutedsl turns the legacy fusion off. from sglang.srt.layers.moe.cutedsl_ar_fusion import ( CuteDSLFusionLayerCommunicator, @@ -2627,12 +2642,12 @@ class DeepseekV2DecoderLayer(nn.Module): communicator_cls = CuteDSLFusionLayerCommunicator else: communicator_cls = LayerCommunicator - self.layer_communicator = communicator_cls( + return communicator_cls( layer_scatter_modes=self.layer_scatter_modes, - input_layernorm=self.input_layernorm, - post_attention_layernorm=self.post_attention_layernorm, + input_layernorm=input_layernorm, + post_attention_layernorm=post_attention_layernorm, allow_reduce_scatter=True, - qkv_latent_func=self.self_attn.prepare_qkv_latent, + qkv_latent_func=qkv_latent_func, ) def _detect_gfx95_quant_format(self) -> str: diff --git a/python/sglang/srt/models/gigachat35.py b/python/sglang/srt/models/gigachat35.py index 0fa4c460d0..93bcbfe49d 100644 --- a/python/sglang/srt/models/gigachat35.py +++ b/python/sglang/srt/models/gigachat35.py @@ -438,8 +438,12 @@ class GigaChat35DecoderLayer(deepseek_v2.DeepseekV2DecoderLayer): pre_layernorm=self.post_attention_layernorm if self._use_pre else None, post_layernorm=self.post_self_attn_layernorm, ) - self.layer_communicator.input_layernorm = attn_prepare_layernorm - self.layer_communicator.post_attention_layernorm = mlp_prepare_layernorm + # The communicator chooses its fused steps from its norms when built. + self.layer_communicator = self._build_layer_communicator( + input_layernorm=attn_prepare_layernorm, + post_attention_layernorm=mlp_prepare_layernorm, + qkv_latent_func=self.layer_communicator.qkv_latent_func, + ) def _is_layer_sparse(self, layer_id: int, is_nextn: bool) -> bool: if is_nextn: diff --git a/test/registered/unit/layers/moe/test_cutedsl_ar_fusion.py b/test/registered/unit/layers/moe/test_cutedsl_ar_fusion.py index d41a2b0fdb..9dbd1fcae2 100644 --- a/test/registered/unit/layers/moe/test_cutedsl_ar_fusion.py +++ b/test/registered/unit/layers/moe/test_cutedsl_ar_fusion.py @@ -4,7 +4,11 @@ from unittest.mock import patch import pytest import torch -from sglang.srt.layers.communicator import LayerCommunicator, UnreducedOutput +from sglang.srt.layers.communicator import ( + LayerCommunicator, + ScatterMode, + UnreducedOutput, +) from sglang.srt.layers.flashinfer_mnnvl_cutedsl import ( FlashInferMNNVLCuteDSLARFusion, _retargeted_config, @@ -33,6 +37,8 @@ def _communicator(): comm = CuteDSLFusionLayerCommunicator.__new__(CuteDSLFusionLayerCommunicator) comm.input_layernorm = RMSNorm(8, eps=1e-6) comm.post_attention_layernorm = RMSNorm(8, eps=1e-6) + comm.enable_fused_ar_quant = False + comm._attn_input_fusions = comm._select_attn_input_fusions() return comm @@ -89,9 +95,8 @@ def test_last_layer_consumes_but_does_not_skip_the_pending_all_reduce(eligible): residual, ), ), - patch.object( - LayerCommunicator, - "prepare_attn", + patch( + "sglang.srt.layers.communicator.reduce_output", lambda *a, **k: pytest.fail("fell through to the unfused path"), ), ): @@ -101,6 +106,27 @@ def test_last_layer_consumes_but_does_not_skip_the_pending_all_reduce(eligible): assert last._can_absorb_post_moe_all_reduce(_DECODE, 8) is False +def test_cutedsl_entries_come_before_the_base_fused_kernel(): + comm = _communicator() + assert comm._attn_input_fusions == ( + comm._finalize_output_and_update_and_read_residual_cutedsl, + comm._reduce_output_and_update_and_read_residual_cutedsl, + comm._reduce_output_and_update_and_read_residual, + ) + base = comm._mlp_input_reduce_output_and_update_and_read_residual + cutedsl = comm._mlp_input_reduce_output_and_update_and_read_residual_cutedsl + with patch( + f"{_MODULE}.get_parallel", + return_value=SimpleNamespace(attn_tp_size=2, tp_size=2), + ): + assert comm._select_mlp_input_fusions(ScatterMode.TP_ATTN_FULL) == ( + cutedsl, + base, + ) + # A scattered residual is gathered first, which the workspace does not do. + assert comm._select_mlp_input_fusions(ScatterMode.SCATTERED) == (base,) + + def test_a_replicated_output_producer_keeps_its_own_all_reduce(eligible): """A TP1 shared expert (or TP1 dense MLP) is not partial; handing its layer's reduction onward would scale the replicated output by tp_size.""" diff --git a/test/registered/unit/layers/test_communicator_layout.py b/test/registered/unit/layers/test_communicator_layout.py index 0dbad9ea36..06989a939e 100644 --- a/test/registered/unit/layers/test_communicator_layout.py +++ b/test/registered/unit/layers/test_communicator_layout.py @@ -1,5 +1,6 @@ import unittest from functools import partial +from types import SimpleNamespace from unittest.mock import patch from sglang.srt.layers import communicator as comm @@ -8,7 +9,6 @@ from sglang.srt.layers.communicator import ( CommunicateContext, CommunicateSimpleFn, CommunicateSummableTensorPairFn, - CommunicateWithAllReduceAndLayerNormFn, ScatterMode, ) from sglang.test.ci.ci_register import register_cpu_ci @@ -20,7 +20,6 @@ SCATTERED = ScatterMode.SCATTERED TP_ATTN_FULL = ScatterMode.TP_ATTN_FULL FULL = ScatterMode.FULL MOE_FULL = ScatterMode.MOE_FULL -Norm = CommunicateWithAllReduceAndLayerNormFn Pair = CommunicateSummableTensorPairFn @@ -83,54 +82,55 @@ class TestScatterModeLayouts(CustomTestCase): self.assertFalse(context.is_same_layout(a, b)) -class TestBoundarySelection(CustomTestCase): - def assert_partial(self, fn, func, residual_input_mode): - self.assertIsInstance(fn, partial) - self.assertIs(fn.func, func) - self.assertEqual(fn.keywords, {"residual_input_mode": residual_input_mode}) +def modes(attn, layer_input, mlp, middle_residual): + return SimpleNamespace( + attn_mode=attn, + layer_input_mode=layer_input, + mlp_mode=mlp, + middle_residual_mode=middle_residual, + ) + +class TestBoundarySelection(CustomTestCase): def test_prepare_mlp(self): dp_tp = make_context(dp=2, tp=2) - self.assert_partial( - Norm.get_fn(TP_ATTN_FULL, SCATTERED, FULL, TP_ATTN_FULL, dp_tp), - Norm._gather_hidden_states_and_residual, - SCATTERED, - ) - self.assert_partial( - Norm.get_fn( - TP_ATTN_FULL, TP_ATTN_FULL, MOE_FULL, TP_ATTN_FULL, make_context(cp=2) + Kind = comm.MlpInputKind + for layout, context, kind in ( + ((TP_ATTN_FULL, SCATTERED, FULL, TP_ATTN_FULL), dp_tp, Kind.GATHER), + ( + (TP_ATTN_FULL, TP_ATTN_FULL, MOE_FULL, TP_ATTN_FULL), + make_context(cp=2), + Kind.GATHER_MOE_CP, ), - Norm._gather_hidden_states_and_residual_moe, - TP_ATTN_FULL, - ) - self.assert_partial( - Norm.get_fn(TP_ATTN_FULL, TP_ATTN_FULL, SCATTERED, SCATTERED, dp_tp), - Norm._scatter_hidden_states_and_residual, - TP_ATTN_FULL, - ) + ((TP_ATTN_FULL, TP_ATTN_FULL, SCATTERED, SCATTERED), dp_tp, Kind.SCATTER), + ): + with self.subTest(layout=layout): + self.assertIs(comm.mlp_input_kind(modes(*layout), context), kind) def test_prepare_mlp_without_attention_dp(self): # FULL and TP_ATTN_FULL share a layout here but select different paths. tp = make_context(tp=2) - self.assert_partial( - Norm.get_fn(TP_ATTN_FULL, TP_ATTN_FULL, FULL, TP_ATTN_FULL, tp), - Norm._gather_hidden_states_and_residual, - TP_ATTN_FULL, - ) - self.assertIs( - Norm.get_fn(TP_ATTN_FULL, TP_ATTN_FULL, TP_ATTN_FULL, TP_ATTN_FULL, tp), - Norm._tp_attn_all_reduce_and_layernorm, - ) - self.assertIs( - Norm.get_fn(TP_ATTN_FULL, TP_ATTN_FULL, FULL, TP_ATTN_FULL, make_context()), - Norm._simple, - ) - self.assertIs( - Norm.get_fn(SCATTERED, SCATTERED, SCATTERED, SCATTERED, tp), - Norm._simple, - ) + Kind = comm.MlpInputKind + for layout, context, kind in ( + ((TP_ATTN_FULL, TP_ATTN_FULL, FULL, TP_ATTN_FULL), tp, Kind.GATHER), + ( + (TP_ATTN_FULL, TP_ATTN_FULL, TP_ATTN_FULL, TP_ATTN_FULL), + tp, + Kind.ATTN_TP_ALL_REDUCE, + ), + ( + (TP_ATTN_FULL, TP_ATTN_FULL, FULL, TP_ATTN_FULL), + make_context(), + Kind.NORM, + ), + ((SCATTERED, SCATTERED, SCATTERED, SCATTERED), tp, Kind.NORM), + ): + with self.subTest(layout=layout): + self.assertIs(comm.mlp_input_kind(modes(*layout), context), kind) with self.assertRaises(NotImplementedError): - Norm.get_fn(SCATTERED, SCATTERED, TP_ATTN_FULL, TP_ATTN_FULL, tp) + comm.mlp_input_kind( + modes(SCATTERED, SCATTERED, TP_ATTN_FULL, TP_ATTN_FULL), tp + ) def test_postprocess(self): self.assertIs( @@ -163,5 +163,142 @@ class TestBoundarySelection(CustomTestCase): ) +class Fusable: + def forward_with_allreduce_fusion(self, *args, **kwargs): + pass + + +def communicator(layout, context, post_attention_layernorm, cls=comm.LayerCommunicator): + """A communicator with these layer layouts, built without the process-wide + parallel state; returns it with its chosen prepare_mlp steps.""" + c = cls.__new__(cls) + c.layer_scatter_modes = modes(*layout) + c._context = context + c.post_attention_layernorm = post_attention_layernorm + return c, c._select_mlp_input() + + +GATHER_LAYOUT = (TP_ATTN_FULL, TP_ATTN_FULL, FULL, TP_ATTN_FULL) + + +class TestMlpInputOrder(CustomTestCase): + def assert_order(self, order, func, **keywords): + self.assertIsInstance(order, partial) + self.assertIs(order.func, func) + self.assertEqual(order.keywords, keywords) + + def test_facts_fixed_at_construction_pick_the_order(self): + fusions = (object(),) + for mode, gathers in ((TP_ATTN_FULL, False), (SCATTERED, True)): + with self.subTest(residual_input_mode=mode): + self.assert_order( + comm._mlp_input_order(make_context(tp=2), mode, fusions), + comm._mlp_input_without_dp, + gathers_residual=gathers, + fusions=fusions, + ) + # Attention TP 1 has no partial sum to carry through the gather. + self.assert_order( + comm._mlp_input_order(make_context(dp=2), TP_ATTN_FULL, fusions), + comm._mlp_input_dp_replicate, + gathers_residual=False, + ) + dp_tp = make_context(dp=2, tp=2) + self.assert_order( + comm._mlp_input_order(dp_tp, SCATTERED, fusions), + comm._mlp_input_dp_partial, + gathers_residual=True, + ) + dp_tp.force_layernorm_before_dp_gather = True + self.assert_order( + comm._mlp_input_order(dp_tp, SCATTERED, fusions), + comm._mlp_input_dp_replicate, + gathers_residual=True, + ) + + def test_a_gather_runs_the_order_chosen_at_construction(self): + for mlp_mode in (FULL, MOE_FULL): + with self.subTest(mlp_mode=mlp_mode): + c, (steps, may_fuse) = communicator( + (TP_ATTN_FULL, TP_ATTN_FULL, mlp_mode, TP_ATTN_FULL), + make_context(tp=2), + Fusable(), + ) + if mlp_mode is MOE_FULL: + self.assertIs(steps.func, comm._mlp_input_gather_moe_cp) + steps = steps.keywords["gather"] + self.assertIs(steps.func, comm._mlp_input_gather) + entry = c._mlp_input_reduce_output_and_update_and_read_residual + self.assert_order( + steps.keywords["order"], + comm._mlp_input_without_dp, + gathers_residual=False, + fusions=(entry,), + ) + self.assertTrue(may_fuse) + + def test_no_fused_entry_without_a_fusable_norm_or_under_attention_dp(self): + for context, norm in ( + (make_context(tp=2), object()), + (make_context(dp=2, tp=2), Fusable()), + ): + with self.subTest(dp=context.attn_dp_size): + _, (steps, may_fuse) = communicator(GATHER_LAYOUT, context, norm) + order = steps.keywords["order"] + self.assertEqual(order.keywords.get("fusions", ()), ()) + self.assertFalse(may_fuse) + + def test_other_kinds_take_their_steps(self): + tp = make_context(tp=2) + for layout, context, expected in ( + ( + (TP_ATTN_FULL, TP_ATTN_FULL, FULL, TP_ATTN_FULL), + make_context(), + comm._mlp_input_norm, + ), + ( + (TP_ATTN_FULL, TP_ATTN_FULL, TP_ATTN_FULL, TP_ATTN_FULL), + tp, + comm._mlp_input_attn_tp_all_reduce, + ), + ): + with self.subTest(expected=expected.__name__): + _, (steps, may_fuse) = communicator(layout, context, Fusable()) + self.assertIs(steps, expected) + self.assertFalse(may_fuse) + _, (steps, _) = communicator( + (TP_ATTN_FULL, TP_ATTN_FULL, SCATTERED, SCATTERED), tp, Fusable() + ) + self.assert_order(steps, comm._mlp_input_scatter, scatters_residual=True) + + def test_mhc_picks_its_own_implementation_of_the_kind(self): + from sglang.srt.layers.communicator_mhc import ( + MHCCommunicateWithAllReduceAndLayerNormFn as MHC, + ) + from sglang.srt.layers.communicator_mhc import ( + MHCLayerCommunicator, + ) + + mhc = object() + for layout, func in ( + (GATHER_LAYOUT, MHC._gather_hidden_states_and_residual), + ( + (TP_ATTN_FULL, TP_ATTN_FULL, SCATTERED, SCATTERED), + MHC._scatter_hidden_states_and_residual, + ), + ): + with self.subTest(func=func.__name__): + c = MHCLayerCommunicator.__new__(MHCLayerCommunicator) + c.mhc = mhc + c.layer_scatter_modes = modes(*layout) + c._context = make_context(tp=2) + c.post_attention_layernorm = Fusable() + steps, may_fuse = c._select_mlp_input() + self.assert_order( + steps, func, residual_input_mode=TP_ATTN_FULL, mhc=mhc + ) + self.assertFalse(may_fuse) + + if __name__ == "__main__": unittest.main() diff --git a/test/registered/unit/layers/test_prepare_attn_steps.py b/test/registered/unit/layers/test_prepare_attn_steps.py index f6c2f433ef..21fbf4f7d7 100644 --- a/test/registered/unit/layers/test_prepare_attn_steps.py +++ b/test/registered/unit/layers/test_prepare_attn_steps.py @@ -48,6 +48,8 @@ def communicator(norm): c.enable_fused_ar_quant = False c.fused_ar_quant_keep_bf16 = False c._communicate_simple_fn = lambda hidden_states, **_: hidden_states + # Construction picks the fused entries; call it under the platform patches. + c._attn_input_fusions = c._select_attn_input_fusions() return c diff --git a/test/registered/unit/lora/test_triton_dp_attention_unit.py b/test/registered/unit/lora/test_triton_dp_attention_unit.py index 0aecaea182..ac8ea4f9d9 100644 --- a/test/registered/unit/lora/test_triton_dp_attention_unit.py +++ b/test/registered/unit/lora/test_triton_dp_attention_unit.py @@ -227,10 +227,11 @@ def test_communicator_publishes_layout_at_each_transition( communicator.input_layernorm = lambda x: x communicator.qkv_latent_func = None communicator._communicate_simple_fn = lambda **kwargs: kwargs["hidden_states"] - communicator._communicate_with_all_reduce_and_layer_norm_fn = lambda **kwargs: ( - kwargs["hidden_states"], - kwargs["residual"], + communicator._mlp_input = lambda hidden_states, residual, *args: ( + hidden_states, + residual, ) + communicator._attn_input_fusions = () monkeypatch.setattr( "sglang.srt.layers.communicator.get_attn_tp_context", lambda: SimpleNamespace(input_scattered=False),