diff --git a/python/sglang/srt/batch_overlap/two_batch_overlap.py b/python/sglang/srt/batch_overlap/two_batch_overlap.py index 05015459a6..0786b22ea1 100644 --- a/python/sglang/srt/batch_overlap/two_batch_overlap.py +++ b/python/sglang/srt/batch_overlap/two_batch_overlap.py @@ -14,9 +14,9 @@ from sglang.srt.batch_overlap.operations import ( ) from sglang.srt.batch_overlap.operations_strategy import OperationsStrategy from sglang.srt.layers import deep_gemm_wrapper -from sglang.srt.layers.boundary_layout import Layout from sglang.srt.layers.communicator import ( CommunicateContext, + Layout, reduce_output, tbo_split_moves, ) diff --git a/python/sglang/srt/layers/boundary_layout.py b/python/sglang/srt/layers/boundary_layout.py deleted file mode 100644 index b85b36d8aa..0000000000 --- a/python/sglang/srt/layers/boundary_layout.py +++ /dev/null @@ -1,403 +0,0 @@ -# Copyright 2026 SGLang Team -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Token layouts of the tensors handed across layer communication boundaries.""" - -from enum import Enum, auto -from typing import FrozenSet, Mapping, Optional, Tuple - -import msgspec - - -class TokenAxis(Enum): - """A parallel dimension across which ranks hold different tokens.""" - - ATTN_DP = auto() - ATTN_CP = auto() - # Each attention-TP rank holds a slice of its group's tokens. - ATTN_TP_SCATTER = auto() - - -class Layout(msgspec.Struct, frozen=True): - """The token axes a rank's rows are sharded over. Single-rank axes are left - out, so two layouts are equal exactly when every rank holds the same tokens.""" - - sharded: FrozenSet[TokenAxis] - - @classmethod - def sharded_over( - cls, *axes: TokenAxis, axis_sizes: Mapping[TokenAxis, int] - ) -> "Layout": - return cls(frozenset(axis for axis in axes if axis_sizes[axis] > 1)) - - -class SumGroup(Enum): - """The group a stage's output is summed over, by name: the handle comes - from get_parallel() when the sum runs.""" - - ATTN_TP = auto() - TP = auto() - # The group one all-reduce of a MoE output runs over. - MOE_OUTPUT = auto() - - -class StageInput(msgspec.Struct, frozen=True): - """The rows a stage's consumer needs: sharded over the token axes its - compute group does not span.""" - - layout: Layout - # Token axes the consumer gathers over itself when its input arrives - # sharded over them. - gathers_itself: FrozenSet[TokenAxis] = frozenset() - - -class StageOutput(msgspec.Struct, frozen=True): - """What a stage's producer hands the boundary after it, fixed at - construction: its rows, the group its output is summed over, and when it - leaves that sum to the boundary instead of completing it. - - What one output actually owes is carried with the value: a plain - tensor handed across layers is complete, an ``UnreducedOutput`` names what - is left. A raw compute output entering a boundary is read with this - declaration and the boundary's decision for the batch.""" - - layout: Layout - # None when there is nothing to sum. - group: Optional[SumGroup] = None - # The producer never reduces its output, e.g. a row-parallel projection - # built with reduce_results=False. - always_leaves: bool = False - # Otherwise it leaves the sum only when the boundary publishes the flag for - # it: fuse_mlp_allreduce, or mlp_reduce_scatter. - leaves_for_next_layer: bool = False - leaves_for_reduce_scatter: bool = False - # Whether it leaves the sum to the attention-DP reduce_scatterv whenever that - # combine applies, as a MoE block does without any flag. - leaves_for_reduce_scatterv: bool = False - - -class DecoderLayerSides(msgspec.Struct, frozen=True): - """What a decoder layer declares about its attention, its FFN and the rows - between them; ``decoder_layer_edges`` turns it into the layer's - boundaries.""" - - # The rows the layer's input and residual arrive on. - input_rows: Layout - attention: StageInput - attention_output: StageOutput - ffn: StageInput - ffn_output: StageOutput - # The rows the residual is on while the FFN runs. - ffn_residual_rows: Layout - # The rows the layer hands on. - output_rows: Layout - # The group the layer's input arrives as a partial sum over, if any. - input_owes: Optional[SumGroup] = None - # Whether the residual is added into one rank's share of the attention - # output's sum before that sum completes, instead of after it. - residual_joins_attention_sum: bool = False - - -class StageDecl(msgspec.Struct, frozen=True): - """A computing stage's two sides: the rows its input must be on, and what - its output is.""" - - input: StageInput - output: StageOutput - - -class EdgeDecl(msgspec.Struct, frozen=True): - """One boundary between two stages, as the layer that runs one side of it - sees it: what arrives from the producer, what the consumer needs, and the - rows the residual is on before and after the boundary.""" - - produced: StageOutput - need: StageInput - residual: Layout - residual_to: Layout - # Whether the residual is added into one rank's share of the produced sum - # before that sum completes, instead of after it. - residual_joins_sum: bool = False - - -class DecoderLayerEdges(msgspec.Struct, frozen=True): - """A decoder layer's three boundaries: from the rows the previous layer - handed on into the attention, from the attention into the FFN, and from - the FFN onto the rows the layer hands on. The next layer runs the rest of - the last one.""" - - into_attention: EdgeDecl - into_ffn: EdgeDecl - out_of_ffn: EdgeDecl - - -def decoder_layer_edges(sides: DecoderLayerSides) -> DecoderLayerEdges: - """The boundaries of a decoder layer, each with the declarations of its two - sides. What arrives at the layer is complete unless it owes a sum by - construction (``input_owes``); a sum the previous layer leaves for a batch - comes with the value.""" - owes = sides.input_owes is not None - arrived = StageOutput(sides.input_rows, group=sides.input_owes, always_leaves=owes) - # Completing what the input owes leaves it and the residual on each - # attention-TP rank's slice. - attention_rows = ( - Layout(sides.input_rows.sharded | {TokenAxis.ATTN_TP_SCATTER}) - if owes - else sides.input_rows - ) - return DecoderLayerEdges( - into_attention=EdgeDecl( - produced=arrived, - need=sides.attention, - residual=sides.input_rows, - residual_to=attention_rows, - ), - into_ffn=EdgeDecl( - produced=sides.attention_output, - need=sides.ffn, - residual=attention_rows, - residual_to=sides.ffn_residual_rows, - residual_joins_sum=sides.residual_joins_attention_sum, - ), - out_of_ffn=EdgeDecl( - produced=sides.ffn_output, - need=StageInput(sides.output_rows), - residual=sides.ffn_residual_rows, - residual_to=sides.output_rows, - ), - ) - - -def stage_edges( - *, previous: Optional[StageOutput], stage: StageDecl, rows: Layout -) -> Tuple[EdgeDecl, EdgeDecl]: - """The two boundaries of a layer that is one stage of a sequence of stages: - into it from the previous stage's output (None at the start of the layer - stack) and out of it onto ``rows``, the rows every layer hands on and the - residual is on between stages. The previous output arrives on those rows: a - stage whose output is elsewhere (an FFN on the TP group) moves it back there - first, and what it may leave of its sum comes with the value; otherwise the - value is complete. The residual follows the input onto a finer slice and - stays where it is when the input is gathered.""" - arrived = ( - StageOutput( - rows, - group=previous.group, - always_leaves=previous.always_leaves, - leaves_for_next_layer=previous.leaves_for_next_layer, - ) - if previous is not None - and (previous.always_leaves or previous.leaves_for_next_layer) - else StageOutput(rows) - ) - during = stage.input.layout if rows.sharded <= stage.input.layout.sharded else rows - return ( - EdgeDecl(produced=arrived, need=stage.input, residual=rows, residual_to=during), - EdgeDecl( - produced=stage.output, - need=StageInput(rows), - residual=during, - residual_to=rows, - ), - ) - - -def sequence_parallel_layer_sides( - *, axis_sizes: Mapping[TokenAxis, int] -) -> DecoderLayerSides: - """A decoder layer while a LayerNorm SP region is active. Its linears - all-gather their input and reduce-scatter their output themselves, so the - layer's rows, its residual and both outputs stay on each TP rank's slice.""" - attention = Layout.sharded_over( - TokenAxis.ATTN_DP, TokenAxis.ATTN_CP, axis_sizes=axis_sizes - ) - local = Layout.sharded_over( - TokenAxis.ATTN_DP, - TokenAxis.ATTN_CP, - TokenAxis.ATTN_TP_SCATTER, - axis_sizes=axis_sizes, - ) - gathers = frozenset({TokenAxis.ATTN_TP_SCATTER}) - return DecoderLayerSides( - input_rows=local, - # qkv and gate_up gather the slices into their GEMM. - attention=StageInput(attention, gathers_itself=gathers), - # o_proj and down reduce-scatter out of theirs, whether fused or not. - attention_output=StageOutput(local), - ffn=StageInput( - Layout.sharded_over(axis_sizes=axis_sizes), gathers_itself=gathers - ), - ffn_output=StageOutput(local), - ffn_residual_rows=local, - output_rows=local, - ) - - -def scattered_residual_layer_sides( - *, - axis_sizes: Mapping[TokenAxis, int], - ffn_group: SumGroup, - is_first_layer: bool, - is_last_layer: bool, - leaves_for_reduce_scatter: bool, -) -> DecoderLayerSides: - """A decoder layer on an input-scattered batch whose residual stays on - each attention-TP rank's slice (MHC). The attention and the FFN compute on - the full rows. The attention takes the slice and gathers it itself (its - QKV hook, or the handoff before an attention that needs the rows); the - boundary brings the FFN its rows and moves both outputs back onto the - slice: the attention output's TP sum by a reduce-scatter, the FFN's too - when it leaves it (``leaves_for_reduce_scatter``). The first layer's input - is the embedding's partial sum on the full rows; the last layer hands on - the full rows again.""" - attention = Layout.sharded_over( - TokenAxis.ATTN_DP, TokenAxis.ATTN_CP, axis_sizes=axis_sizes - ) - local = Layout.sharded_over( - TokenAxis.ATTN_DP, - TokenAxis.ATTN_CP, - TokenAxis.ATTN_TP_SCATTER, - axis_sizes=axis_sizes, - ) - return DecoderLayerSides( - input_rows=attention if is_first_layer else local, - attention=StageInput( - attention, gathers_itself=frozenset({TokenAxis.ATTN_TP_SCATTER}) - ), - attention_output=StageOutput( - attention, group=SumGroup.ATTN_TP, always_leaves=True - ), - ffn=StageInput(Layout.sharded_over(axis_sizes=axis_sizes)), - ffn_output=StageOutput( - attention, - group=ffn_group, - leaves_for_reduce_scatter=leaves_for_reduce_scatter, - ), - ffn_residual_rows=local, - output_rows=attention if is_last_layer else local, - input_owes=SumGroup.TP if is_first_layer else None, - ) - - -def input_scattered_layer_sides( - *, - axis_sizes: Mapping[TokenAxis, int], - ffn_group: SumGroup, - hands_on_partial: bool, -) -> DecoderLayerSides: - """A decoder layer on a batch whose attention input is scattered over - attention TP. Each layer's input is a partial sum over TP on the full rows: - the embedding and every FFN before it leave that sum, and a reduce-scatter - completes it onto each rank's slice, which the attention gathers itself. - The residual comes back to the full rows inside the attention output's sum.""" - attention = Layout.sharded_over( - TokenAxis.ATTN_DP, TokenAxis.ATTN_CP, axis_sizes=axis_sizes - ) - return DecoderLayerSides( - input_rows=attention, - attention=StageInput( - attention, gathers_itself=frozenset({TokenAxis.ATTN_TP_SCATTER}) - ), - attention_output=StageOutput( - attention, group=SumGroup.ATTN_TP, always_leaves=True - ), - ffn=StageInput(Layout.sharded_over(axis_sizes=axis_sizes)), - # The next layer's input completes the sum; the last layer's FFN does. - ffn_output=StageOutput( - attention, group=ffn_group, leaves_for_reduce_scatter=hands_on_partial - ), - ffn_residual_rows=attention, - output_rows=attention, - input_owes=SumGroup.TP, - residual_joins_attention_sum=True, - ) - - -def decoder_layer_sides( - *, - axis_sizes: Mapping[TokenAxis, int], - ffn_on_local_rows: bool, - previous_on_local_rows: bool, - is_last_layer: bool, - attention_gathers_local_rows: bool, - ffn_group: SumGroup, - leaves_for_next_layer: bool, - leaves_for_reduce_scatter: bool, - leaves_for_reduce_scatterv: bool, - hands_on_attention_rows: bool = False, - ffn_shards_over_cp: bool = False, -) -> DecoderLayerSides: - """An attention followed by an FFN, derived from the groups each computes - over. The FFN runs either on the TP group (a dense MLP, or a MoE not - dispatched per DP shard) or on this rank's local rows (a MoE dispatched per - DP shard, which completes its own combine, or a dense MLP on every rank). - A MoE whose data-parallel groups are the CP ranks (``ffn_shards_over_cp``) - computes each CP shard on its own ranks: its rows stay sharded over CP. - A layer on local rows hands on the attention's rows when it is the last, or - with ``hands_on_attention_rows``.""" - # Attention computes over the attention-TP ranks of one DP (and CP) shard. - attention = Layout.sharded_over( - TokenAxis.ATTN_DP, TokenAxis.ATTN_CP, axis_sizes=axis_sizes - ) - # Each attention-TP rank's own slice of those rows. - local = Layout.sharded_over( - TokenAxis.ATTN_DP, - TokenAxis.ATTN_CP, - TokenAxis.ATTN_TP_SCATTER, - axis_sizes=axis_sizes, - ) - # The TP group spans every token axis, so an FFN on it needs every row. - ffn = ( - local - if ffn_on_local_rows - else Layout.sharded_over( - *((TokenAxis.ATTN_CP,) if ffn_shards_over_cp else ()), - axis_sizes=axis_sizes, - ) - ) - attention_tp = axis_sizes[TokenAxis.ATTN_TP_SCATTER] > 1 - return DecoderLayerSides( - input_rows=local if previous_on_local_rows else attention, - attention=StageInput( - attention, - gathers_itself=( - frozenset({TokenAxis.ATTN_TP_SCATTER}) - if attention_gathers_local_rows - else frozenset() - ), - ), - # The output projection leaves the attention-TP sum to prepare_mlp. - attention_output=StageOutput( - attention, - group=SumGroup.ATTN_TP if attention_tp else None, - always_leaves=attention_tp, - ), - ffn=StageInput(ffn), - ffn_output=( - StageOutput(ffn) - if ffn_on_local_rows - else StageOutput( - ffn, - group=ffn_group, - leaves_for_next_layer=leaves_for_next_layer, - leaves_for_reduce_scatter=leaves_for_reduce_scatter, - leaves_for_reduce_scatterv=leaves_for_reduce_scatterv, - ) - ), - ffn_residual_rows=local if ffn_on_local_rows else attention, - output_rows=( - local - if ffn_on_local_rows and not (is_last_layer or hands_on_attention_rows) - else attention - ), - ) diff --git a/python/sglang/srt/layers/communicator.py b/python/sglang/srt/layers/communicator.py deleted file mode 100644 index fcf3cca238..0000000000 --- a/python/sglang/srt/layers/communicator.py +++ /dev/null @@ -1,3244 +0,0 @@ -# Copyright 2023-2024 SGLang Team -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -import logging -from contextlib import contextmanager -from dataclasses import dataclass -from enum import Enum, auto -from functools import partial -from typing import Callable, Dict, List, Optional, Protocol, Tuple, Union - -import msgspec -import torch - -from sglang.srt.distributed import ( - GroupCoordinator, - attention_tensor_model_parallel_all_reduce, - attention_tensor_model_parallel_quant_all_reduce, - tensor_model_parallel_all_reduce, -) -from sglang.srt.distributed.device_communicators.pynccl_allocator import ( - use_symmetric_memory, -) -from sglang.srt.environ import envs -from sglang.srt.layers import layernorm_sp -from sglang.srt.layers.attention.dsa.utils import ( - dsa_use_prefill_cp, - is_dsa_enable_prefill_cp, -) -from sglang.srt.layers.aux_hidden_states import AuxHiddenStateAccumulator -from sglang.srt.layers.boundary_layout import ( - DecoderLayerSides, - EdgeDecl, - Layout, - StageInput, - StageOutput, - SumGroup, - TokenAxis, - decoder_layer_edges, - decoder_layer_sides, - input_scattered_layer_sides, - scattered_residual_layer_sides, - sequence_parallel_layer_sides, -) -from sglang.srt.layers.communicator_dsa_cp import ( - dsa_cp_gather_hidden_states, - dsa_cp_reduce_scatter_hidden_states, -) -from sglang.srt.layers.cp.utils import ( - is_mla_cp_active, - is_mla_cp_enabled, -) -from sglang.srt.layers.dp_attention import ( - attn_tp_all_gather_into_tensor, - attn_tp_reduce_scatter_tensor, - can_use_dp_reduce_scatter, - dp_gather_partial, - dp_gather_replicate, - dp_reduce_scatter_tensor, - dp_scatter, - get_dp_global_num_tokens, - get_global_dp_buffer, - get_local_dp_buffer, - get_moe_cp_rank, - get_moe_cp_size, - is_allocation_symmetric, - is_dp_attention_enabled, - is_enable_moe_cp_allgather, - moe_cp_all_gather_into_tensor, -) -from sglang.srt.layers.flashinfer_comm_fusion import ( - is_flashinfer_allreduce_unavailable, - uses_cutedsl_ar_fusion, -) -from sglang.srt.layers.moe import ( - can_merge_post_experts_all_reduce, - get_moe_a2a_backend, - is_moe_input_scattered_across_dp_ranks, - post_experts_reduction_group, - post_experts_sum_is_one_all_reduce, - should_use_dp_reduce_scatterv, - should_use_flashinfer_cutlass_moe_fp4_allgather, -) -from sglang.srt.layers.quantization.fp8_utils import ( - _use_aiter_bpreshuffle_gfx95, - materialize_bpreshuffle_fp8_scale_tuple, -) -from sglang.srt.model_executor.cuda_graph_config import ( - Backend, - Phase, - check_cuda_graph_backend, -) -from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode -from sglang.srt.runtime_context import ( - LoRABatchLayout, - get_exec, - get_forward, - get_lora, - get_parallel, - get_platform, - get_spec, -) -from sglang.srt.speculative.spec_info import SpeculativeAlgorithm -from sglang.srt.utils import ( - get_bool_env_var, - is_cuda, - is_flashinfer_available, - is_gfx95_supported, - is_gfx1250_supported, - is_hip, - is_npu, -) - -_is_cuda = is_cuda() -_is_flashinfer_available = is_flashinfer_available() -_is_sm90_supported = _is_cuda and get_platform().is_sm90 -_is_sm100_supported = _is_cuda and get_platform().is_sm100 -_use_aiter = get_bool_env_var("SGLANG_USE_AITER") and is_hip() -_is_gfx95_supported = is_gfx95_supported() -_is_gfx1250_supported = is_gfx1250_supported() -_is_npu = is_npu() -_use_ag_after_qlora = envs.SGLANG_USE_AG_AFTER_QLORA.get() - -if _use_aiter: - from sglang.kernels.ops.quantization.fp8_kernel import fp8_dtype as _aiter_fp8_dtype - - if _is_gfx1250_supported: - from aiter.ops.triton.fused_fp8_quant import fused_rms_fp8_group_quant - else: - from aiter.ops.rmsnorm import add_rmsnorm_quant as _aiter_add_rmsnorm_quant - from aiter.ops.rmsnorm import rmsnorm_quant as _aiter_rmsnorm_quant - - if _is_gfx95_supported: - from aiter.ops.triton.fused_fp8_quant import fused_rms_fp8_group_quant - - from sglang.srt.layers.quantization.rocm_mxfp4_utils import ( - fused_rms_mxfp4_quant, - ) -elif _is_npu: - from sglang.srt.hardware_backend.npu.cmo import prepare_weight_cache - - -def _fused_rmsnorm_fp8_per_token_quant( - hidden_states: torch.Tensor, - weight: torch.Tensor, - epsilon: float, - residual: Optional[torch.Tensor] = None, -): - """Fused (optional residual-add +) RMSNorm + FP8 per-token quantization. - - Only used with the aiter (ROCm) backend. - - Args: - residual: if provided, computes hidden_states + residual before RMSNorm - and returns updated residual_out as second element. - - Returns: - If residual is None: (out_fp8, scale) - If residual provided: ((out_fp8, scale), residual_out) - """ - if _is_gfx1250_supported: - # per-token quant == group quant with group_size == hidden size, giving - # an (M, 1) scale. - N = hidden_states.shape[-1] - (out_fp8, scale), _out1, _out2, residual_out = fused_rms_fp8_group_quant( - hidden_states, - weight, - epsilon, - group_size=N, - dtype_quant=_aiter_fp8_dtype, - res1=residual, - ) - if residual is not None: - return (out_fp8, scale), residual_out - return (out_fp8, scale) - - M, N = hidden_states.shape - out_fp8 = torch.empty((M, N), dtype=_aiter_fp8_dtype, device=hidden_states.device) - scale = torch.empty(M, dtype=torch.float32, device=hidden_states.device) - if residual is not None: - residual_out = torch.empty_like(hidden_states) - _aiter_add_rmsnorm_quant( - out_fp8, - hidden_states, - residual, - residual_out, - scale, - weight, - epsilon, - 0, # group_size=0 → per-token - ) - return (out_fp8, scale.unsqueeze(1)), residual_out - else: - _aiter_rmsnorm_quant( - out_fp8, - hidden_states, - scale, - weight, - epsilon, - 0, # group_size=0 → per-token - ) - return (out_fp8, scale.unsqueeze(1)) - - -# TODO: According to the discussion in https://github.com/flashinfer-ai/flashinfer/issues/1223#issuecomment-3047256465 -# We set the max token num to 128 for allreduce fusion with min-latency case(use_oneshot=True). -FUSE_ALLREDUCE_MAX_BATCH_SIZE = 2048 - - -def apply_flashinfer_allreduce_fusion(batch_size: int): - return ( - # NOTE: flashinfer 0.6.1 caused performance regression on sm100 for allreduce fusion - # Ref: https://github.com/sgl-project/sglang/issues/17237 - (_is_sm90_supported or _is_sm100_supported) - and _is_flashinfer_available - and not is_dp_attention_enabled() - and get_exec().comm.flashinfer_allreduce_fusion_backend is not None - # cutedsl runs its own fused path from the fusion communicator. - and not uses_cutedsl_ar_fusion() - and not is_flashinfer_allreduce_unavailable() - # Symbolic size checks stay last: under Dynamo tracing they guard on - # the dynamic token dim, so statically-off configs must short-circuit - # before reaching them. - and batch_size > 0 - and batch_size <= FUSE_ALLREDUCE_MAX_BATCH_SIZE - ) - - -def aiter_all_reduce_fusion_enabled_for(forward_mode: ForwardMode) -> bool: - comm = get_exec().comm - if not comm.enable_aiter_allreduce_fusion: - return False - if forward_mode.is_extend_or_draft_extend_or_mixed(): - return not comm.disable_aiter_allreduce_fusion_in_prefill - return not comm.disable_aiter_allreduce_fusion_in_decode - - -def apply_aiter_all_reduce_fusion( - input_tensor: torch.Tensor, forward_batch: ForwardBatch -): - n = input_tensor.shape[-1] - total_bytes = input_tensor.numel() * input_tensor.element_size() - # Aiter's should_custom_ar uses <= max_size/2 (64 MB); match that boundary. - return ( - _use_aiter - and total_bytes > 0 - and n <= 16384 - and total_bytes <= 8 * 1024 * 8192 - and get_parallel().tp_size != 6 - and not is_dp_attention_enabled() - and aiter_all_reduce_fusion_enabled_for(forward_batch.forward_mode) - ) - - -class ScatterMode(Enum): - """ - Suppose we have TP=4, DP=2, enable-dp-attention, and the system handles seq a,b,c,d - Model input/output: [ab, ab, cd, cd] for four ranks respectively - SCATTERED: [a, b, c, d] - TP_ATTN_FULL: [ab, ab, cd, cd], i.e. all ranks inside a TP attn group have full data of the group - FULL: [abcd, abcd, abcd, abcd] - MOE_FULL: full within the MoE group (cp_per_moe CP chunks), used when moe_dp_size < attn_cp_size - """ - - SCATTERED = auto() - TP_ATTN_FULL = auto() - FULL = auto() - MOE_FULL = auto() - - @staticmethod - def model_input_output(): - """The scatter mode for model forward pass input and output data""" - if is_dsa_enable_prefill_cp() or is_mla_cp_enabled(): - return ScatterMode.SCATTERED - - return ScatterMode.TP_ATTN_FULL - - -class AttentionInputs: - def __init__( - self, - hidden_states: torch.Tensor, - forward_batch: ForwardBatch, - qkv_latent_func: Callable, - *, - is_pre_gathered: bool = False, - ): - self.hidden_states_local = hidden_states - self.forward_batch = forward_batch - self.qkv_latent_func = qkv_latent_func - self.hidden_states_ = None - self.qkv_latent_ = None - # When True, hidden_states_local is already attn_tp-gathered upstream - # (e.g. by MHC's prepare_attn for DSA). fetch_* must NOT gather again. - self.is_pre_gathered = is_pre_gathered - - def tp_all_gather_hidden_states(self, hidden_states, forward_batch): - total_tokens = forward_batch.input_ids.shape[0] - output = hidden_states.new_empty((total_tokens, hidden_states.shape[-1])) - get_parallel().tp_group.all_gather_into_tensor(output, hidden_states) - return output - - def fetch_qkv_latent(self): - if self.qkv_latent_ is not None: - return self.qkv_latent_ - assert self.qkv_latent_func is not None - self.qkv_latent_ = self.qkv_latent_func( - self.hidden_states_local, self.forward_batch - ) - if get_attn_tp_context().input_scattered and not self.is_pre_gathered: - self.qkv_latent_ = self.tp_all_gather_hidden_states( - self.qkv_latent_, self.forward_batch - ) - return self.qkv_latent_ - - def fetch_hidden_states(self): - if self.hidden_states_ is not None: - return self.hidden_states_ - self.hidden_states_ = self.hidden_states_local - if get_attn_tp_context().input_scattered and not self.is_pre_gathered: - self.hidden_states_ = self.tp_all_gather_hidden_states( - self.hidden_states_, self.forward_batch - ) - return self.hidden_states_ - - -class AttnTpContext: - def __init__(self): - self.allow_input_scattered = False - self.is_dsa = False - - def init_context(self, q_lora_rank, is_dsa, is_mhc=False): - # Only MHC pre-gathers hidden states before DSA attention, so non-MHC DSA - # cannot use scattered inputs. - self.is_dsa = is_dsa - self.allow_input_scattered = ( - get_parallel().enable_attn_tp_input_scattered - and (_is_cuda or _is_npu) - and q_lora_rank is not None - and (is_mhc or not is_dsa) - and get_parallel().tp_size > 1 - and not is_dp_attention_enabled() - and get_moe_a2a_backend().is_none() - and not enable_moe_dense_fully_dp() - and not check_cuda_graph_backend(Phase.PREFILL, Backend.TC_PIECEWISE) - and get_spec().speculative_algorithm != "EAGLE3" - ) - if get_parallel().enable_attn_tp_input_scattered: - if not self.allow_input_scattered: - logging.info( - "attn_tp_input_scattered is not enabled while other conditions are not met" - ) - else: - logging.info("attn_tp_input_scattered is enabled") - - def use_input_scattered(self, forward_batch: ForwardBatch): - return ( - self.allow_input_scattered - and forward_batch.forward_mode.is_extend() - and not forward_batch.forward_mode.is_target_verify() - and forward_batch.input_ids is not None - and not forward_batch.can_run_tbo - ) - - @property - def input_scattered(self): - return get_forward().attn_input_scattered - - def set_attn_inputs(self, attn_inputs: AttentionInputs): - get_forward().set("attn_inputs", attn_inputs) - - def set_hidden_states_local(self, hidden_states: torch.Tensor) -> None: - attn_inputs = get_forward().attn_inputs - if attn_inputs is not None: - attn_inputs.hidden_states_local = hidden_states - - def fetch_qkv_latent(self): - attn_inputs = get_forward().attn_inputs - assert attn_inputs is not None - return attn_inputs.fetch_qkv_latent() - - def fetch_hidden_states(self): - attn_inputs = get_forward().attn_inputs - assert attn_inputs is not None - return attn_inputs.fetch_hidden_states() - - def clear_attn_inputs(self) -> None: - get_forward().set("attn_inputs", None) - - @contextmanager - def maybe_input_scattered(self, forward_batch: ForwardBatch): - flag = self.use_input_scattered(forward_batch) - forward = get_forward() - # scoped() also restores when the forward raises — the old in-place - # swap leaked the flag on exceptions. - with forward.scoped(attn_input_scattered=flag): - try: - yield - finally: - forward.set("attn_inputs", None) - - -ATTN_TP_CONTEXT = AttnTpContext() - - -def get_attn_tp_context(): - return ATTN_TP_CONTEXT - - -@dataclass -class _LayerModeComputationContext: - num_layers: int - layer_id: int - is_layer_sparse: bool - is_previous_layer_sparse: Optional[bool] - is_next_layer_sparse: Optional[bool] - - def previous_layer(self): - assert self.is_previous_layer_sparse is not None - return _LayerModeComputationContext( - num_layers=self.num_layers, - layer_id=self.layer_id - 1, - is_layer_sparse=self.is_previous_layer_sparse, - is_previous_layer_sparse=None, - is_next_layer_sparse=self.is_layer_sparse, - ) - - -def sparse_mlp_scatter_mode() -> ScatterMode: - """SCATTERED hands a sparse MLP this rank's own token shard; FULL and - MOE_FULL hand it a buffer gathered over the attn-TP or MoE-CP group.""" - if ( - # Token dispatch/combine will be handled outside of LayerCommunicator for these modes. - not get_moe_a2a_backend().is_none() - or should_use_flashinfer_cutlass_moe_fp4_allgather() - or enable_dwdp() - ): - return ScatterMode.SCATTERED - # DSA CP and MLA CP both don't support MOE_FULL yet; fall back to FULL. - if is_enable_moe_cp_allgather() and not ( - is_dsa_enable_prefill_cp() or is_mla_cp_enabled() - ): - return ScatterMode.MOE_FULL - return ScatterMode.FULL - - -@dataclass -class LayerScatterModes: - layer_input_mode: ScatterMode - attn_mode: ScatterMode - # Can be further split into e.g. mlp_input_mode and mlp_output_mode if needed - mlp_mode: ScatterMode - middle_residual_mode: ScatterMode - layer_output_mode: ScatterMode - is_layer_sparse: bool = False - # The model's first layer: its input is the embedding, not a layer output. - is_first_layer: bool = False - # The model's last layer: its output goes to the final norm, not a next layer. - is_last_layer: bool = False - # Whether the layer before this one has a sparse MLP; None when the modes - # were given directly, not planned from the layer sequence. - is_previous_layer_sparse: Optional[bool] = None - # Whether the layer after this one has a sparse MLP; None likewise. - is_next_layer_sparse: Optional[bool] = None - - @classmethod - def init_new(cls, **kwargs): - context = _LayerModeComputationContext(**kwargs) - return cls( - layer_input_mode=cls._compute_layer_input_mode(context), - attn_mode=ScatterMode.TP_ATTN_FULL, - mlp_mode=cls._compute_mlp_mode(context), - middle_residual_mode=cls._compute_middle_residual_mode(context), - layer_output_mode=cls._compute_layer_output_mode(context), - is_layer_sparse=context.is_layer_sparse, - is_first_layer=context.layer_id == 0, - is_last_layer=context.layer_id == context.num_layers - 1, - is_previous_layer_sparse=context.is_previous_layer_sparse, - is_next_layer_sparse=context.is_next_layer_sparse, - ) - - @classmethod - def _compute_layer_input_mode(cls, context: _LayerModeComputationContext): - if context.layer_id == 0: - return ScatterMode.model_input_output() - return cls._compute_layer_output_mode(context.previous_layer()) - - @classmethod - def _compute_mlp_mode(cls, context: _LayerModeComputationContext): - if context.is_layer_sparse: - return sparse_mlp_scatter_mode() - else: - if enable_moe_dense_fully_dp(): - return ScatterMode.SCATTERED - # A TP-sharded dense MLP reduces over the whole TP group, which spans - # every CP rank; a CP-sharded prefill must gather tokens across CP - # first or the all-reduce sums different tokens' partial outputs. - # MLA/DSA CP models gather over the attention-CP group instead. - if _generic_prefill_cp_shards_tokens() and not ( - is_dsa_enable_prefill_cp() or is_mla_cp_enabled() - ): - return ScatterMode.MOE_FULL - return ScatterMode.FULL - - @classmethod - def _should_gather_for_tbo(cls, context: _LayerModeComputationContext): - return ( - not context.is_layer_sparse - and context.is_next_layer_sparse - and enable_moe_dense_fully_dp() - and get_exec().overlap.enable_two_batch_overlap - ) - - @classmethod - def _compute_middle_residual_mode(cls, context: _LayerModeComputationContext): - mlp_mode = cls._compute_mlp_mode(context) - if mlp_mode == ScatterMode.SCATTERED: - return ScatterMode.SCATTERED - if mlp_mode in (ScatterMode.FULL, ScatterMode.MOE_FULL): - return ScatterMode.TP_ATTN_FULL - raise NotImplementedError - - @classmethod - def _compute_layer_output_mode(cls, context: _LayerModeComputationContext): - mlp_mode = cls._compute_mlp_mode(context) - if context.layer_id == context.num_layers - 1: - return ScatterMode.model_input_output() - if mlp_mode == ScatterMode.SCATTERED: - if cls._should_gather_for_tbo(context): - return ScatterMode.TP_ATTN_FULL - return ScatterMode.SCATTERED - if mlp_mode in (ScatterMode.FULL, ScatterMode.MOE_FULL): - return ScatterMode.TP_ATTN_FULL - raise NotImplementedError - - -def enable_moe_dense_fully_dp(): - return get_parallel().moe_dense_tp_size == 1 - - -def _generic_prefill_cp_shards_tokens() -> bool: - """Whether the strategy prefill CP path shards prefill tokens across CP ranks.""" - parallel = get_parallel() - return parallel.attn_cp_size > 1 and parallel.enable_prefill_cp - - -def enable_dwdp(): - return get_parallel().dwdp_size > 1 - - -def tp_reduce_scatter( - hidden_states: torch.Tensor, - residual: Optional[torch.Tensor], - context: "CommunicateContext", -) -> Tuple[torch.Tensor, Optional[torch.Tensor]]: - """Module-level so MHC communicators can reuse it without a - ``LayerCommunicator`` instance.""" - if hidden_states.shape[0] == 0: - return hidden_states, hidden_states - assert hidden_states.shape[0] % context.tp_size == 0, ( - f"Expected total tokens {hidden_states.shape[0]} % tp_size {context.tp_size} to be 0" - ) - local_tokens = hidden_states.shape[0] // context.tp_size - output = hidden_states.new_empty(local_tokens, *hidden_states.shape[1:]) - get_parallel().tp_group.reduce_scatter_tensor(output, hidden_states) - if residual is not None: - residual = residual.tensor_split(context.tp_size)[context.tp_rank] - return output, residual - - -def _update_and_read_residual_plain( - norm, hidden_states, residual, post_residual_addition -): - if residual is None: - return norm(hidden_states), hidden_states - return norm(hidden_states, residual, post_residual_addition) - - -def _update_and_read_residual_aiter_mxfp4( - norm, hidden_states, residual, post_residual_addition -): - # post_residual_addition is not applied on this path. - output, *_, residual_out = fused_rms_mxfp4_quant( - hidden_states, - norm.weight, - norm.variance_epsilon, - None, - None, - None, - residual, - ) - return output, hidden_states if residual is None else residual_out - - -def _update_and_read_residual_aiter_fp8_group( - norm, hidden_states, residual, post_residual_addition -): - """aiter (ROCm gfx95) fused RMSNorm + FP8 group quant. Under DSA the - unquantized bf16 output rides along as a third element, so the DSA indexer - can skip dequantizing. post_residual_addition is not applied on this path.""" - needs_bf16 = get_attn_tp_context().is_dsa - output, unquantized, _, residual_out = fused_rms_fp8_group_quant( - hidden_states, - norm.weight, - norm.variance_epsilon, - inp2=None, - inp2_weight=None, - inp2_epsilon=None, - group_size=128, - dtype_quant=torch.float8_e4m3fn, - res1=residual, - output_unquantized_inp1=needs_bf16, - transpose_scale=False, - ) - if _use_aiter_bpreshuffle_gfx95: - output = materialize_bpreshuffle_fp8_scale_tuple(output) - if needs_bf16: - output = (output[0], output[1], unquantized) - return output, hidden_states if residual is None else residual_out - - -def _update_and_read_residual_aiter_fp8_per_token( - norm, hidden_states, residual, post_residual_addition -): - if residual is None: - output = _fused_rmsnorm_fp8_per_token_quant( - hidden_states, norm.weight.data, norm.variance_epsilon - ) - return output, hidden_states - if post_residual_addition is not None: - residual = residual + post_residual_addition - return _fused_rmsnorm_fp8_per_token_quant( - hidden_states, norm.weight.data, norm.variance_epsilon, residual=residual - ) - - -def _attn_input_update_and_read_residual(quant_format: str): - """Add the previous layer's output to the residual and read the attention - input from it: the input norm, fused with the quantization this format - wants. Without a residual (the first layer, or one already folded into - hidden_states) the input itself becomes the residual.""" - if _use_aiter and _is_gfx95_supported and "mxfp4" in quant_format: - return _update_and_read_residual_aiter_mxfp4 - if _use_aiter and _is_gfx95_supported and quant_format == "fp8": - return _update_and_read_residual_aiter_fp8_group - if _use_aiter and quant_format == "fp8_per_token": - return _update_and_read_residual_aiter_fp8_per_token - return _update_and_read_residual_plain - - -class ResidualOps(Protocol): - """How a layer writes each stage's output into its residual and reads the - next stage's input from it. The boundary steps complete sums and move - tokens around these operations. AddAndNorm adds and normalizes; MHC's - hyper-connection streams implement the same operations (communicator_mhc).""" - - # The write-back is a plain add, which one rank may run before the sum it - # adds into completes: the DP partial order, and a residual that joins the - # attention output's sum. - adds_plainly: bool - # The layer writes its FFN output into the residual itself instead of - # leaving that to the next layer's input. - updates_residual_after_ffn: bool - - def enter(self, hidden_states: torch.Tensor) -> torch.Tensor: - """The residual the layer stack starts from, given its input.""" - - def read_attention_input(self, residual, norm, quant_format: str) -> Tuple: - """The attention input and the residual, from a residual that already - holds the previous layer's output.""" - - def update_and_read_attention_input( - self, hidden_states, residual, norm, quant_format: str, post_residual_addition - ) -> Tuple: - """Write the previous layer's output into the residual, then read the - attention input from it.""" - - def update_and_read_ffn_input(self, hidden_states, residual, norm) -> Tuple: - """Write the attention output into the residual, then read the FFN - input from it.""" - - def update_residual(self, hidden_states, residual) -> torch.Tensor: - """The residual with the FFN output written into it.""" - - def residual_to_attn_tp_shard(self, residual, context) -> torch.Tensor: - """This attention-TP rank's slice of the residual.""" - - def residual_from_attn_tp_shards(self, residual) -> torch.Tensor: - """The residual gathered from every attention-TP rank's slice.""" - - -class AddAndNorm: - """A plain residual: a stage's output is added into it, and the next - stage's input is its norm. An empty batch skips the norms.""" - - adds_plainly = True - updates_residual_after_ffn = False - - def enter(self, hidden_states): - return hidden_states - - def read_attention_input(self, residual, norm, quant_format): - if residual.shape[0] == 0: - return residual, residual - return _attn_input_update_and_read_residual(quant_format)( - norm, residual, None, None - ) - - def update_and_read_attention_input( - self, hidden_states, residual, norm, quant_format, post_residual_addition - ): - if hidden_states.shape[0] == 0: - return hidden_states, hidden_states - return _attn_input_update_and_read_residual(quant_format)( - norm, hidden_states, residual, post_residual_addition - ) - - def update_and_read_ffn_input(self, hidden_states, residual, norm): - if residual is None: - # The layer stack starts at this FFN: its input is the residual. - if hidden_states.shape[0] == 0: - return hidden_states, hidden_states - return norm(hidden_states), hidden_states - if hidden_states.shape[0] == 0: - return hidden_states, residual - return norm(hidden_states, residual) - - def update_residual(self, hidden_states, residual): - hidden_states += residual - return hidden_states - - def residual_to_attn_tp_shard(self, residual, context): - return _redistribute_to_attn_tp_shards(residual, context) - - def residual_from_attn_tp_shards(self, residual): - return _redistribute_from_attn_tp_shards(residual) - - -ADD_AND_NORM = AddAndNorm() - - -class UnreducedOutput(msgspec.Struct, frozen=True): - """A layer output that still owes its sum, left for the next layer's input. - Hand it to the next layer, or pass it through reduce_output() before reading - it any other way. The producer says what is owed: one all-reduce over - ``group`` that keeps the layout, or ``reduce_and_redistribute``.""" - - partial: torch.Tensor - group: Optional[GroupCoordinator] = None - # Under attention DP: the reduction that also brings ``partial`` back to this - # rank's tokens (a reduce-scatter, or an all-reduce then a scatter). - reduce_and_redistribute: Optional[Callable[[torch.Tensor], torch.Tensor]] = None - - -class HandoffOutput(msgspec.Struct, frozen=True): - """A layer output that still owes work only its producer knows how to do (a - MoE's finalize and sum), left for the next layer's input. A fused kernel - there may do that work together with its own; anything else passes it - through reduce_output(), which calls ``complete()``.""" - - def complete(self) -> torch.Tensor: - """Do the owed work, unfused, and return the complete output.""" - raise NotImplementedError - - -def reduce_output( - hidden_states: Union[torch.Tensor, UnreducedOutput, HandoffOutput, None], -) -> Optional[torch.Tensor]: - """Run the work an UnreducedOutput or a HandoffOutput still owes; pass - anything else through.""" - if isinstance(hidden_states, UnreducedOutput): - if hidden_states.reduce_and_redistribute is not None: - return hidden_states.reduce_and_redistribute(hidden_states.partial) - return hidden_states.group.all_reduce(hidden_states.partial) - if isinstance(hidden_states, HandoffOutput): - return hidden_states.complete() - return hidden_states - - -def layer_input_buffer( - hidden_states: Union[torch.Tensor, UnreducedOutput], -) -> torch.Tensor: - """The tensor holding a layer's input, for reusing its memory without reading it.""" - if isinstance(hidden_states, UnreducedOutput): - return hidden_states.partial - return hidden_states - - -def _batch_size(forward_batch: ForwardBatch) -> int: - return ( - forward_batch.input_ids.shape[0] if hasattr(forward_batch, "input_ids") else 0 - ) - - -def _ffn_has_tokens(forward_batch: ForwardBatch) -> bool: - if is_dp_attention_enabled(): - # The FFN runs on every DP rank's tokens, so every rank decides alike. - return (getattr(forward_batch, "global_dp_buffer_len", None) or 0) > 0 - return _batch_size(forward_batch) > 0 - - -def _unfused_completion_matches_the_ffn(forward_batch: ForwardBatch) -> bool: - """Whether the next layer's all-reduce is the one the FFN would have run - itself, as the MoE declares it (post_experts_sum_is_one_all_reduce), on a - batch the FFN runs.""" - return _ffn_has_tokens(forward_batch) and post_experts_sum_is_one_all_reduce() - - -class LayerCommunicator: - # Communicators built without __init__ (e.g. test doubles) publish no LoRA - # layout and try no fused kernel at the FFN exit. - _publish_lora_layout: bool = False - _ffn_exit_fusions: Tuple[Callable, ...] = () - # A plain residual unless the layer is built with its own. - _residual_ops: ResidualOps = ADD_AND_NORM - - def __init__( - self, - layer_scatter_modes: LayerScatterModes, - input_layernorm: torch.nn.Module, - post_attention_layernorm: torch.nn.Module, - # Reduce scatter requires skipping all-reduce in model code after MoE/MLP, so only enable for models which have that implemented. Remove flag once done for all models that use LayerCommunicator. - allow_reduce_scatter: bool = False, - qkv_latent_func: Optional[Callable] = None, - force_layernorm_before_dp_gather: bool = False, - enable_fused_ar_quant: bool = False, - fused_ar_quant_keep_bf16: bool = False, - # False for a layer whose FFN always completes its own reduction. - allow_deferred_ffn_reduction: bool = True, - # How the layer writes its residual and reads its stages' inputs. - residual_ops: ResidualOps = ADD_AND_NORM, - # A layer that is one stage of a sequence of stages, instead of an - # attention followed by an FFN. - stage: Optional["LayerStage"] = None, - ): - self.layer_scatter_modes = layer_scatter_modes - self.input_layernorm = input_layernorm - self.post_attention_layernorm = post_attention_layernorm - self.allow_reduce_scatter = allow_reduce_scatter - self.is_last_layer = layer_scatter_modes.is_last_layer - self.qkv_latent_func = qkv_latent_func - self.force_layernorm_before_dp_gather = force_layernorm_before_dp_gather - self.enable_fused_ar_quant = enable_fused_ar_quant - self.fused_ar_quant_keep_bf16 = fused_ar_quant_keep_bf16 - self.allow_deferred_ffn_reduction = allow_deferred_ffn_reduction - self._residual_ops = residual_ops - - self._context = CommunicateContext.init_new() - self._context.force_layernorm_before_dp_gather = ( - force_layernorm_before_dp_gather - ) - # The fused kernels every batch's attention input tries first. - self._attn_input_fusions = self._select_attn_input_fusions() - # The fused kernels of the next layer's input the FFN exit tries first. - self._ffn_exit_fusions = self._select_ffn_exit_fusions() - self._speculative_algo = SpeculativeAlgorithm.from_string( - get_spec().speculative_algorithm - ) - # LoRA kernels need the per-layer token layout only under DP attention. - self._publish_lora_layout = get_parallel().enable_dp_attention and bool( - get_lora().enable_lora - ) - if stage is not None: - self._init_stage(stage) - return - # Its two boundaries, for a layer that is one stage. - self.stage_edges = None - # The steps the layer's ordinary batches run. - sides = self._declared_sides() - self._declared = sides - self._steps = self._steps_from_declarations( - sides, - fusions=self._select_mlp_input_fusions(), - force_layernorm_before_gather=force_layernorm_before_dp_gather, - ) - # The steps a batch that shards its tokens over attention CP runs; None - # when no batch does. - self._cp_steps = ( - self._steps_from_declarations( - self._declared_sides(cp_active=True), - fusions=self._select_mlp_input_fusions(), - force_layernorm_before_gather=force_layernorm_before_dp_gather, - cp_moves=_cp_moves(), - ) - if _generic_prefill_cp_shards_tokens() - else None - ) - # The steps a batch with input-scattered attention runs; None where it - # cannot run. - self._input_scattered_steps = ( - self._steps_for_input_scattered(sides) - if self._input_can_be_scattered() - else None - ) - - # Under LayerNorm SP, the steps the layer runs while the region is - # active; None without SP. The two are exclusive: SP runs a model without - # q_lora, which input-scattered attention needs. - self._sp_steps = ( - _select_boundary_steps( - sequence_parallel_layer_sides(axis_sizes=token_axis_sizes()), - residual_ops=residual_ops, - attention_fusions=self._attn_input_fusions, - enters_stack=self.layer_scatter_modes.is_first_layer, - ) - if layernorm_sp.layernorm_sp_enabled() - else None - ) - - def _init_stage(self, stage: "LayerStage") -> None: - """A layer that is one stage: every batch runs the two boundaries its - declarations give.""" - if get_parallel().attn_cp_size > 1 or layernorm_sp.layernorm_sp_enabled(): - raise NotImplementedError( - "a layer that is one stage with attention CP or LayerNorm SP" - ) - self._declared = None - self._cp_steps = self._input_scattered_steps = self._sp_steps = None - self.stage_edges = stage.edges - into_edge, out_edge = stage.edges - reads_ffn = stage.reads is InputRead.FFN - into = make_boundary( - into_edge, - reads=stage.reads, - fusions=( - self._select_mlp_input_fusions() - if reads_ffn - else self._attn_input_fusions - ), - force_layernorm_before_gather=self.force_layernorm_before_dp_gather, - residual_ops=self._residual_ops, - enters_stack=stage.enters_stack, - ) - out = make_boundary(out_edge, reads=None, residual_ops=self._residual_ops) - self._steps = BoundarySteps( - attention_prepare=_another_stage if reads_ffn else into.prepare, - attention_input=_another_stage if reads_ffn else into.input_move, - ffn_input=into.prepare if reads_ffn else _another_stage, - ffn_input_rows=into.input_rows, - ffn_output=out_edge.produced, - ffn_output_move=out.output_move, - ffn_output_move_completes_sum=out.output_move_completes_sum, - ffn_sum_is_movable=out_edge.produced.group is not None, - fused=into.fused, - ) - - @property - def input_rows(self) -> Layout: - """The rows the layer's input and residual arrive on in a batch that - runs its ordinary steps.""" - if self._declared is not None: - return self._declared.input_rows - return self.stage_edges[0].residual - - @property - def input_on_attention_tp_slices(self) -> bool: - """Whether the layer's input arrives on each attention-TP rank's slice - of its rows (after a layer on local rows), in a batch that runs its - ordinary steps.""" - return TokenAxis.ATTN_TP_SCATTER in self.input_rows.sharded - - def _input_can_be_scattered(self) -> bool: - """Whether a batch may run this layer with input-scattered attention: - configured, on TP without attention DP, a prefill CP, an a2a backend or - a dense MLP on every rank. The rest of what ``AttnTpContext.init_context`` requires is only - known once the model is built.""" - parallel = get_parallel() - return ( - parallel.enable_attn_tp_input_scattered - and parallel.tp_size > 1 - and parallel.attn_dp_size == 1 - and not _generic_prefill_cp_shards_tokens() - and get_moe_a2a_backend().is_none() - and not enable_moe_dense_fully_dp() - ) - - def _declared_sides(self, *, cp_active: bool = False) -> DecoderLayerSides: - """The declarations this layer's boundaries are chosen from: an - attention and an FFN, in a layer whose previous layer is known. - ``cp_active`` gives a batch that shards its tokens over CP; the others - hold every token on each CP rank. An active LayerNorm SP region and - input-scattered attention have their own. Raises NotImplementedError - for the combinations the steps do not cover.""" - modes = self.layer_scatter_modes - parallel = get_parallel() - if not (modes.is_first_layer or modes.is_previous_layer_sparse is not None): - raise NotImplementedError( - "a layer built without the facts of the layer before it" - ) - # A MoE dispatched per DP shard computes on this rank's local rows and - # hands its layer's output on there; so does a dense MLP on every rank. - moe_on_local_rows = is_moe_input_scattered_across_dp_ranks() - dense_on_local_rows = enable_moe_dense_fully_dp() - - # Some batch shards its tokens over attention CP. - cp_shards = _generic_prefill_cp_shards_tokens() - if parallel.attn_cp_size > 1 and modes.is_layer_sparse: - self._refuse_uncovered_cp_moe(moe_on_local_rows, cp_shards) - # A MoE whose data-parallel groups are the CP ranks computes each CP - # shard of a GQA prefill CP on its own ranks. - ffn_on_cp_shards = ( - modes.is_layer_sparse - and parallel.attn_cp_size > 1 - and parallel.moe_dp_size == parallel.attn_cp_size - and not _gathers_over_attention_cp() - ) - - def on_local_rows(sparse: bool) -> bool: - return moe_on_local_rows if sparse else dense_on_local_rows - - def gathers_for_tbo(sparse: bool, next_sparse: Optional[bool]) -> bool: - # Under two-batch overlap a dense layer on local rows hands the - # sparse layer after it, where the split happens, the attention's - # rows. - return ( - dense_on_local_rows - and get_exec().overlap.enable_two_batch_overlap - and not sparse - and bool(next_sparse) - ) - - if cp_shards and _cp_moves().reduce_scatter is not None: - # A CP extend's FFN may leave its sum to the reduce-scatter that - # takes each rank's shard back (DSA and MLA CP). - may_leave = not cp_active - may_leave_to_reduce_scatter = True - elif cp_shards or parallel.attn_dp_size > 1: - # Otherwise under CP an FFN over every CP shard completes its own sum; - # one on its own CP shard hands on its rows as without CP. - may_leave = may_leave_to_reduce_scatter = ( - parallel.attn_cp_size == 1 or ffn_on_cp_shards - ) - else: - # No batch shards its tokens and there is no attention DP: as - # without CP. - may_leave = may_leave_to_reduce_scatter = True - return decoder_layer_sides( - axis_sizes=token_axis_sizes(cp_active=cp_active), - ffn_on_local_rows=on_local_rows(modes.is_layer_sparse), - ffn_shards_over_cp=ffn_on_cp_shards, - previous_on_local_rows=( - not modes.is_first_layer - and on_local_rows(modes.is_previous_layer_sparse) - and not gathers_for_tbo( - modes.is_previous_layer_sparse, modes.is_layer_sparse - ) - ), - is_last_layer=modes.is_last_layer, - hands_on_attention_rows=gathers_for_tbo( - modes.is_layer_sparse, modes.is_next_layer_sparse - ), - attention_gathers_local_rows=_use_ag_after_qlora, - ffn_group=SumGroup.MOE_OUTPUT if modes.is_layer_sparse else SumGroup.TP, - leaves_for_next_layer=self.allow_deferred_ffn_reduction and may_leave, - leaves_for_reduce_scatter=self.allow_reduce_scatter - and may_leave_to_reduce_scatter, - # A MoE block leaves its sum to reduce_scatterv whenever that combine - # applies (should_skip_post_experts_all_reduce). - leaves_for_reduce_scatterv=( - self.allow_reduce_scatter or modes.is_layer_sparse - ) - and may_leave, - ) - - @staticmethod - def _refuse_uncovered_cp_moe(moe_on_local_rows: bool, cp_shards: bool) -> None: - """A MoE layer under attention CP whose tokens the steps cannot bring - to it: under a prefill CP, one dispatched per DP shard under attention DP - and GQA CP, and one on the TP group whose data-parallel groups are the CP - ranks under DSA or MLA CP; and under attention DP, one on the TP group - whose data-parallel groups are the CP ranks.""" - parallel = get_parallel() - gqa = not _gathers_over_attention_cp() - if moe_on_local_rows: - if cp_shards and gqa and parallel.attn_dp_size > 1: - raise NotImplementedError( - "a MoE dispatched per DP shard under attention DP and GQA " - "prefill CP" - ) - elif parallel.moe_dp_size == parallel.attn_cp_size: - if cp_shards and not gqa: - raise NotImplementedError( - "a MoE on the TP group with moe_dp_size == attn_cp_size under " - "DSA or MLA prefill CP" - ) - if parallel.attn_dp_size > 1: - raise NotImplementedError( - "a MoE on the TP group with moe_dp_size == attn_cp_size under " - "attention DP and attention CP" - ) - - def _steps_for_input_scattered(self, sides: DecoderLayerSides) -> "BoundarySteps": - """The steps a batch with input-scattered attention runs at this layer, - for the layer's ordinary declarations ``sides``. A plain residual comes - back to the full rows inside the attention output's sum; one whose - write-back is not a plain add stays on each rank's slice.""" - if self._residual_ops.adds_plainly: - scattered = input_scattered_layer_sides( - axis_sizes=token_axis_sizes(), - ffn_group=sides.ffn_output.group, - hands_on_partial=self.allow_reduce_scatter and not self.is_last_layer, - ) - handoff = _hand_qkv_hook_its_input - else: - scattered = scattered_residual_layer_sides( - axis_sizes=token_axis_sizes(), - ffn_group=sides.ffn_output.group, - is_first_layer=self.layer_scatter_modes.is_first_layer, - is_last_layer=self.is_last_layer, - leaves_for_reduce_scatter=self.allow_reduce_scatter, - ) - # DSA and hook-less attention take the slice gathered. - handoff = _hand_scattered_input_to_attention - return _select_boundary_steps( - scattered, - residual_ops=self._residual_ops, - attention_handoff=handoff, - attention_fusions=self._attn_input_fusions, - enters_stack=self.layer_scatter_modes.is_first_layer, - ) - - def _steps_from_declarations( - self, sides: DecoderLayerSides, **kwargs - ) -> "BoundarySteps": - """The steps this layer runs for a set of declarations.""" - return _select_boundary_steps( - sides, - residual_ops=self._residual_ops, - attention_fusions=self._attn_input_fusions, - enters_stack=self.layer_scatter_modes.is_first_layer, - **kwargs, - ) - - def prepare_attn_and_capture_last_layer_outputs( - self, - hidden_states: torch.Tensor, - residual: torch.Tensor, - forward_batch: ForwardBatch, - captured_last_layer_outputs: Optional[AuxHiddenStateAccumulator] = None, - post_residual_addition: Optional[torch.Tensor] = None, - quant_format: str = "", - ): - hidden_states, residual = self.prepare_attn( - hidden_states, - residual, - forward_batch, - quant_format=quant_format, - post_residual_addition=post_residual_addition, - ) - if captured_last_layer_outputs is not None: - gathered_last_layer_output = self._batch_steps( - forward_batch - ).attention_input( - hidden_states=residual, - forward_batch=forward_batch, - context=self._context, - ) - if ( - gathered_last_layer_output is residual - # An accumulator that copies on append already holds a snapshot. - and not getattr(captured_last_layer_outputs, "copies_on_append", False) - and not self._post_attn_residual_is_read_only(residual, forward_batch) - ): - gathered_last_layer_output = residual.clone() - captured_last_layer_outputs.append(gathered_last_layer_output) - return hidden_states, residual - - def _post_attn_residual_is_read_only( - self, residual: torch.Tensor, forward_batch: ForwardBatch - ) -> bool: - """True if ``prepare_mlp``'s post-attention RMSNorm leaves ``residual`` - untouched, so Eagle3 aux capture can keep its reference and skip the clone. - - Of the base fused entry's kernels only flashinfer's writes a fresh - ``residual_out`` (see ``flashinfer_allreduce_residual_rmsnorm``); the aiter - kernel and every plain norm fold into ``residual`` in place. It runs when - ``prepare_mlp`` selected a fused entry that may return a new residual and - the batch is not input-scattered. - """ - return ( - any( - f.may_return_new_residual - for f in self._batch_steps(forward_batch).fused - ) - and not get_attn_tp_context().input_scattered - and apply_flashinfer_allreduce_fusion(residual.shape[0]) - ) - - def publish_attn_lora_layout(self) -> None: - """Attention consumes the DP-local token batch.""" - if self._publish_lora_layout: - get_forward().set("lora_batch_layout", LoRABatchLayout.DP_LOCAL) - - def publish_mlp_lora_layout(self, steps: "BoundarySteps") -> None: - """The MLP consumes the TP-global batch only when the batch's FFN input - is gathered over attention DP.""" - if self._publish_lora_layout: - get_forward().set( - "lora_batch_layout", - ( - LoRABatchLayout.DP_LOCAL - if TokenAxis.ATTN_DP in steps.ffn_input_rows.sharded - else LoRABatchLayout.TP_GLOBAL - ), - ) - - def prepare_attn( - self, - hidden_states: Union[torch.Tensor, UnreducedOutput], - residual: torch.Tensor, - forward_batch: ForwardBatch, - quant_format: str = "", - post_residual_addition: Optional[torch.Tensor] = None, - ): - self.publish_attn_lora_layout() - # The SP region opens at the first layer, re-evaluated per forward so a - # crash mid-loop cannot leak into the next one. It sets what the batch's - # steps are chosen from, and the first layer's input owes nothing. - if self._sp_steps is not None and self.layer_scatter_modes.is_first_layer: - get_forward().set( - "sp_active", layernorm_sp.runs_sp(forward_batch.forward_mode) - ) - if get_forward().sp_active: - hidden_states = layernorm_sp.sp_entry_scatter(hidden_states) - hidden_states, residual = self._batch_steps(forward_batch).attention_prepare( - hidden_states, - residual, - forward_batch, - self.input_layernorm, - self._context, - quant_format=quant_format, - post_residual_addition=post_residual_addition, - ) - return self._finish_prepare_attn( - hidden_states=hidden_states, - residual=residual, - forward_batch=forward_batch, - ) - - def _in_sp_region(self) -> bool: - return self._sp_steps is not None and get_forward().sp_active - - def _batch_steps(self, forward_batch: ForwardBatch) -> "BoundarySteps": - """The steps this batch runs: an active LayerNorm SP region's, - input-scattered attention's, a CP extend's, or the layer's ordinary - ones.""" - if self._in_sp_region(): - return self._sp_steps - if ( - self._input_scattered_steps is not None - and get_attn_tp_context().input_scattered - ): - return self._input_scattered_steps - if self._cp_steps is not None and _batch_shards_over_cp(forward_batch): - return self._cp_steps - return self._steps - - def _finish_prepare_attn(self, hidden_states, residual, forward_batch): - """Tail every prepare_attn path must run, or ``attn_inputs`` is unset.""" - steps = self._batch_steps(forward_batch) - hidden_states = steps.attention_input( - hidden_states=hidden_states, - forward_batch=forward_batch, - context=self._context, - ) - hidden_states = steps.attention_handoff( - hidden_states, forward_batch, self.qkv_latent_func - ) - return hidden_states, residual - - def _select_attn_input_fusions(self) -> Tuple[Callable, ...]: - """The fused kernels that complete what the previous layer left together - with the residual update and the input norm, in the order they are tried. - Each takes (owed, residual, forward_batch, post_residual_addition) and - returns None when it does not take the batch. They add the residual - plainly.""" - if not ( - self._residual_ops.adds_plainly - and hasattr(self.input_layernorm, "forward_with_allreduce_fusion") - ): - return () - self._attn_input_fuses_quant = ( - self.enable_fused_ar_quant - and _use_aiter - and hasattr( - self.input_layernorm, "forward_with_allreduce_fusion_quant_per_group" - ) - ) - return (self._reduce_output_and_update_and_read_residual,) - - def _reduce_output_and_update_and_read_residual( - self, - owed: UnreducedOutput, - residual: torch.Tensor, - forward_batch: ForwardBatch, - post_residual_addition: Optional[torch.Tensor], - ) -> Optional[Tuple[torch.Tensor, torch.Tensor]]: - """Complete the sum the previous layer left, add it to the residual and - apply the input norm in one aiter or flashinfer kernel; None when the - kernel does not take this batch. The result is not quantized for - ``quant_format``.""" - if ( - not isinstance(owed, UnreducedOutput) - # The kernel does not add it. - or post_residual_addition is not None - # The kernel reduces over the MoE output's group. - or owed.group is not post_experts_reduction_group() - ): - return None - hidden_states = owed.partial - if not ( - apply_aiter_all_reduce_fusion(hidden_states, forward_batch) - or apply_flashinfer_allreduce_fusion(hidden_states.shape[0]) - ): - return None - if self._attn_input_fuses_quant: - # Falls back to AR+RMSNorm + separate quant internally when the - # fully-fused kernel cannot service the shape. - quant_result = ( - self.input_layernorm.forward_with_allreduce_fusion_quant_per_group( - hidden_states, - residual, - use_attn_tp_group=False, - keep_bf16=self.fused_ar_quant_keep_bf16, - ) - ) - if quant_result is not None: - return quant_result - return self.input_layernorm.forward_with_allreduce_fusion( - hidden_states, residual, use_attn_tp_group=False - ) - - def _select_mlp_input_fusions(self) -> Tuple["FusedMlpInput", ...]: - """The fused kernels that can take the attention -> FFN steps, in the - order they are tried. They add the residual plainly.""" - if not ( - self._residual_ops.adds_plainly - and hasattr(self.post_attention_layernorm, "forward_with_allreduce_fusion") - ): - return () - return ( - FusedMlpInput( - completes=SumGroup.ATTN_TP, - run=self._mlp_input_reduce_output_and_update_and_read_residual, - may_return_new_residual=True, - ), - ) - - def _mlp_input_reduce_output_and_update_and_read_residual( - self, - hidden_states: torch.Tensor, - residual: torch.Tensor, - forward_batch: ForwardBatch, - ) -> Optional[Tuple[torch.Tensor, torch.Tensor]]: - """The attention-TP all-reduce, residual add and norm in one aiter or - flashinfer kernel; None when neither takes the batch.""" - if not ( - apply_aiter_all_reduce_fusion(hidden_states, forward_batch) - or apply_flashinfer_allreduce_fusion(hidden_states.shape[0]) - ): - return None - return self.post_attention_layernorm.forward_with_allreduce_fusion( - hidden_states, residual, use_attn_tp_group=True - ) - - def prepare_mlp( - self, - hidden_states: Union[torch.Tensor, UnreducedOutput], - residual: torch.Tensor, - forward_batch: ForwardBatch, - cache=None, - ): - steps = self._batch_steps(forward_batch) - self.publish_mlp_lora_layout(steps) - if cache is not None: - self._context.cache = cache - - return steps.ffn_input( - hidden_states, - residual, - forward_batch, - self.post_attention_layernorm, - self._context, - ) - - def maybe_prefetch_next_full_attention_kv( - self, - forward_batch: ForwardBatch, - next_full_attention_layer_id: Optional[int], - ) -> None: - return - - def postprocess_layer( - self, - hidden_states: torch.Tensor, - residual: torch.Tensor, - forward_batch: ForwardBatch, - ): - return self._complete_ffn_output_now( - hidden_states, - residual, - forward_batch=forward_batch, - dp_step=self._postprocess_dp_step(forward_batch), - ) - - def _local_token_move_can_go_to_next_layer( - self, forward_batch: ForwardBatch - ) -> bool: - """Whether the next layer's input can run this layer's move of its FFN - output back to this rank's tokens: the base postprocess scatter, when - the next layer's input also writes the output into the residual.""" - return ( - self._batch_steps(forward_batch).returns_over_dp - and not self._residual_ops.updates_residual_after_ffn - ) - - def _postprocess_dp_step( - self, forward_batch: ForwardBatch - ) -> Optional[Callable[[torch.Tensor, torch.Tensor, ForwardBatch], None]]: - """The reduce-scatter that brings this layer's FFN output back to this - rank's tokens under attention DP; None when the base postprocess would - only scatter, or does not move tokens.""" - steps = self._batch_steps(forward_batch) - if not steps.returns_over_dp: - return None - return _reduce_and_redistribute_output_step( - forward_batch, - leaves_for_reduce_scatter=steps.ffn_output.leaves_for_reduce_scatter, - leaves_for_reduce_scatterv=steps.ffn_output.leaves_for_reduce_scatterv, - ) - - def _ffn_leaves_sum_to_reduce_scatter( - self, forward_batch: ForwardBatch, dp_step: Optional[Callable] - ) -> bool: - """Whether the FFN leaves its sum out because a reduce-scatter completes - it: the attention-DP one ``dp_step`` names, or the CP / input-scattered - one.""" - steps = self._batch_steps(forward_batch) - if not steps.ffn_output.leaves_for_reduce_scatter: - return False - if dp_step is not None or steps.ffn_output_move_completes_sum: - return True - return get_attn_tp_context().input_scattered and not self.is_last_layer - - def ffn_reduction_group(self, forward_batch: ForwardBatch) -> GroupCoordinator: - """The group this layer's FFN output owes its sum over: the MoE output's - group on a sparse layer, the TP group a dense MLP reduces over.""" - return _sum_group(self._batch_steps(forward_batch).ffn_output.group) - - def _select_ffn_completion(self, forward_batch: ForwardBatch) -> "FfnCompletion": - """Decide once, before the FFN runs, what it skips and what completes its - output: the next layer's input, or this layer's postprocess step.""" - dp_step = self._postprocess_dp_step(forward_batch) - mlp_reduce_scatter = self._ffn_leaves_sum_to_reduce_scatter( - forward_batch, dp_step - ) - complete_now = partial( - self._complete_ffn_output_now, - forward_batch=forward_batch, - dp_step=dp_step, - ) - steps = self._batch_steps(forward_batch) - if not steps.ffn_output.leaves_for_next_layer: - return FfnCompletion( - defer_moe_finalize=False, - fuse_mlp_allreduce=False, - mlp_reduce_scatter=mlp_reduce_scatter, - complete=complete_now, - ) - fusion = next( - filter(None, (fused(forward_batch) for fused in self._ffn_exit_fusions)), - None, - ) - defer_moe_finalize = fusion is FfnExitFusion.DEFER_MOE_FINALIZE - # A fused kernel that takes the sum, a handoff included, skips the - # post-experts all-reduce. - fuse_mlp_allreduce = fusion is not None or self._ffn_sum_moves_to_next_layer( - forward_batch, mlp_reduce_scatter=mlp_reduce_scatter, dp_step=dp_step - ) - if fuse_mlp_allreduce: - group = self.ffn_reduction_group(forward_batch) - if steps.returns_over_dp: - # Under attention DP the next layer also brings the sum back to - # this rank's tokens. - wrap = partial( - UnreducedOutput, - reduce_and_redistribute=partial( - _all_reduce_then_to_local_tokens, group, forward_batch - ), - ) - else: - wrap = partial(UnreducedOutput, group=group) - complete = partial(_leave_to_next_layer, wrap) - elif ( - dp_step is not None - and not self.is_last_layer - and self._local_token_move_can_go_to_next_layer(forward_batch) - ): - complete = partial( - _leave_to_next_layer, - partial( - UnreducedOutput, - reduce_and_redistribute=partial( - _to_local_tokens, dp_step, forward_batch - ), - ), - ) - else: - complete = complete_now - return FfnCompletion( - defer_moe_finalize=defer_moe_finalize, - fuse_mlp_allreduce=fuse_mlp_allreduce, - mlp_reduce_scatter=mlp_reduce_scatter, - complete=complete, - ) - - def _complete_ffn_output_now( - self, - hidden_states: torch.Tensor, - residual: torch.Tensor, - *, - forward_batch: ForwardBatch, - dp_step: Optional[Callable], - ) -> Tuple[torch.Tensor, torch.Tensor]: - """This layer's postprocess, run with the attention-DP step already - chosen: the move back to where the next layer reads the FFN output, then - the write-back into the residual for a layer that does it itself.""" - steps = self._batch_steps(forward_batch) - if steps.returns_over_dp: - hidden_states = _to_local_tokens( - dp_step or _redistribute_output, forward_batch, hidden_states - ) - else: - hidden_states, residual = steps.ffn_output_move( - hidden_states=hidden_states, - residual=residual, - forward_batch=forward_batch, - context=self._context, - allow_reduce_scatter=self.allow_reduce_scatter, - is_layer_sparse=self.layer_scatter_modes.is_layer_sparse, - ) - if residual is not None and self._residual_ops.updates_residual_after_ffn: - hidden_states = self._residual_ops.update_residual(hidden_states, residual) - residual = None - return hidden_states, residual - - def mixer_exit(self, forward_batch: ForwardBatch) -> "MixerExit": - """Decide once whether this stage's mixer (an attention-like stage) - skips its output all-reduce. Use the result as a context manager around - the mixer, then call ``finish``.""" - return MixerExit(self, forward_batch) - - def ffn_exit(self, forward_batch: ForwardBatch) -> "FfnExit": - """Decide once how this layer's FFN output reduction completes. Use the - result as a context manager around the FFN call, then call ``finish``.""" - return FfnExit(self, forward_batch) - - def _branch_rows( - self, forward_batch: ForwardBatch - ) -> Tuple[Layout, Layout, Layout]: - """The rows of this layer's FFN input, of its residual while the FFN - runs, and of what the layer hands on, for a batch that runs its ordinary - steps.""" - if self._batch_steps(forward_batch) is not self._steps: - raise NotImplementedError( - "a branch on a batch with attention CP, LayerNorm SP or " - "input-scattered attention" - ) - if self._declared is None: - raise NotImplementedError("a branch on a layer that is one stage") - sides = self._declared - return sides.ffn.layout, sides.ffn_residual_rows, sides.output_rows - - def branch_input( - self, - source: "LayerCommunicator", - hidden_states: torch.Tensor, - residual: torch.Tensor, - forward_batch: ForwardBatch, - ) -> Tuple[torch.Tensor, torch.Tensor]: - """The FFN input and residual that ``source``'s boundary read for its - own FFN, for this layer's FFN, which branches from the same input: moved - to the rows this FFN needs and its residual's rows.""" - rows, residual_rows, _ = source._branch_rows(forward_batch) - to, residual_to, _ = self._branch_rows(forward_batch) - return ( - move_rows(hidden_states, rows, to, forward_batch), - move_rows(residual, residual_rows, residual_to, forward_batch), - ) - - def branch_output( - self, hidden_states: torch.Tensor, forward_batch: ForwardBatch - ) -> torch.Tensor: - """This layer's complete FFN output as a branch's contribution, which - adds to the layer's output without writing the residual: moved to the - rows the layer hands on.""" - rows, _, to = self._branch_rows(forward_batch) - return move_rows(hidden_states, rows, to, forward_batch) - - def merge_branch( - self, - contribution: torch.Tensor, - hidden_states: torch.Tensor, - residual: torch.Tensor, - source: "LayerCommunicator", - forward_batch: ForwardBatch, - ) -> Tuple[torch.Tensor, torch.Tensor]: - """A contribution from ``branch_output`` summed with what ``source``'s - layer hands on, ``hidden_states`` and ``residual``, moved to the rows - this layer hands on.""" - _, _, rows = source._branch_rows(forward_batch) - _, _, to = self._branch_rows(forward_batch) - hidden_states = move_rows(hidden_states, rows, to, forward_batch) - residual = move_rows(residual, rows, to, forward_batch) - return contribution + hidden_states, residual - - def finish_layer_stack( - self, - hidden_states: Union[torch.Tensor, UnreducedOutput, HandoffOutput], - residual: Optional[torch.Tensor], - forward_batch: ForwardBatch, - *, - final_norm_takes_handoff: bool = False, - ) -> Tuple[Union[torch.Tensor, HandoffOutput], Optional[torch.Tensor]]: - """Complete what this layer left for a next layer. Call it on the last - layer of this rank before its output reaches the final norm, the next - pipeline rank, or any other consumer outside the layers. A final norm - that does a producer's handoff together with its own work - (``final_norm_takes_handoff``) receives it as it is.""" - if final_norm_takes_handoff and isinstance(hidden_states, HandoffOutput): - return hidden_states, residual - return reduce_output(hidden_states), residual - - def should_use_reduce_scatter(self, forward_batch: ForwardBatch): - """Whether the FFN leaves its sum to a reduce-scatter, for layers that run - their FFN outside ffn_exit.""" - return self._ffn_leaves_sum_to_reduce_scatter( - forward_batch, self._postprocess_dp_step(forward_batch) - ) - - def _select_ffn_exit_fusions( - self, - ) -> Tuple[Callable[[ForwardBatch], Optional["FfnExitFusion"]], ...]: - """The fused kernels of the next layer's input that may take this layer's - FFN sum, in the order they are tried. Each returns what the exit does - when its kernel takes this batch, or None.""" - return (self._next_input_norm_takes_ffn_sum,) - - def _next_input_norm_takes_ffn_sum( - self, forward_batch: ForwardBatch - ) -> Optional["FfnExitFusion"]: - """The aiter / flashinfer AR + add + norm of the next layer's input.""" - if self.should_fuse_mlp_allreduce_with_next_layer(forward_batch): - return FfnExitFusion.NEXT_INPUT - return None - - def _ffn_sum_can_move_to_next_layer(self, forward_batch: ForwardBatch) -> bool: - # Under the MoE-CP all-gather the fusion path would skip postprocess_layer - # and its MoE-CP scatter, leaving hidden_states longer than the residual. - if ( - is_enable_moe_cp_allgather() - or not self._batch_steps(forward_batch).ffn_sum_is_movable - ): - return False - - # The fused residual+LN reduces over a single group. Hybrid EP+TP spans - # two disjoint groups; post_experts_all_reduce() merges them into one - # _TP reduction when moe_dp_size == 1, which the fused kernel can absorb. - # When merging is blocked, no single group covers both, so fusion stays off. - parallel = get_parallel() - if ( - parallel.moe_ep_size > 1 - and parallel.moe_tp_size > 1 - and not can_merge_post_experts_all_reduce() - ): - return False - - if ( - is_dp_attention_enabled() - and self._speculative_algo is not None - and self._speculative_algo.is_eagle() - ): - return False - - return not get_attn_tp_context().input_scattered - - # NOTE: This function will cause torch recompilation - def should_fuse_mlp_allreduce_with_next_layer( - self, forward_batch: ForwardBatch - ) -> bool: - if not self._ffn_sum_can_move_to_next_layer(forward_batch): - return False - batch_size = _batch_size(forward_batch) - return ( - ( - apply_flashinfer_allreduce_fusion(batch_size) - or ( - _use_aiter - and batch_size > 0 - and get_parallel().tp_size != 6 - and not is_dp_attention_enabled() - and get_moe_a2a_backend().is_none() - and aiter_all_reduce_fusion_enabled_for(forward_batch.forward_mode) - ) - ) - and (not self.is_last_layer) - and (self._context.tp_size > 1) - ) - - def _ffn_sum_moves_to_next_layer( - self, - forward_batch: ForwardBatch, - *, - mlp_reduce_scatter: bool, - dp_step: Optional[Callable], - ) -> bool: - """Whether the FFN leaves its output's all-reduce to the next layer's - input when no fused kernel takes it: when the next layer would run the - same all-reduce the FFN itself would have.""" - return ( - self._context.tp_size > 1 - and not self.is_last_layer - and self._ffn_sum_can_move_to_next_layer(forward_batch) - and _unfused_completion_matches_the_ffn(forward_batch) - and not mlp_reduce_scatter - # Under attention DP the next layer must also run postprocess's - # scatter back to this rank's tokens, and nothing more. - and ( - not is_dp_attention_enabled() - or ( - self._local_token_move_can_go_to_next_layer(forward_batch) - and dp_step is None - ) - ) - ) - - -class FfnExitFusion(Enum): - """What an FFN exit does when a fused kernel of the next layer's input takes - its sum.""" - - # The MoE hands its unfinalized output on; the next input finalizes it. - DEFER_MOE_FINALIZE = auto() - # The FFN leaves its all-reduce to the next input norm. - NEXT_INPUT = auto() - - -class FfnCompletion(msgspec.Struct, frozen=True): - """One FFN's reduction decision. The flags are published while the FFN runs; - ``complete(hidden_states, residual)`` then either wraps the output for the - next layer's input to complete, or runs this layer's postprocess step.""" - - defer_moe_finalize: bool - fuse_mlp_allreduce: bool - mlp_reduce_scatter: bool - complete: Callable[[torch.Tensor, torch.Tensor], Tuple] - - -def _leave_to_next_layer( - wrap: Callable[[torch.Tensor], UnreducedOutput], - hidden_states: torch.Tensor, - residual: torch.Tensor, -) -> Tuple[UnreducedOutput, torch.Tensor]: - return wrap(hidden_states), residual - - -class MixerExit: - """The scope that publishes a mixer's decision while it runs: inside the - ``with`` block ``fuse_mlp_allreduce`` on ``get_forward()`` tells its - row-parallel output projection to skip the all-reduce. It skips when the - stage's output always leaves its sum (to an FFN stage, which completes it in - its input), and when it may leave it and the fused kernel takes it into the - next input norm.""" - - __slots__ = ("skips_reduction", "_hands_on", "_scope") - - def __init__(self, communicator: LayerCommunicator, forward_batch: ForwardBatch): - produced = communicator._batch_steps(forward_batch).ffn_output - self._hands_on = ( - produced.leaves_for_next_layer - and communicator.should_fuse_mlp_allreduce_with_next_layer(forward_batch) - ) - self.skips_reduction = produced.always_leaves or self._hands_on - self._scope = get_forward().scoped(fuse_mlp_allreduce=self.skips_reduction) - - def __enter__(self) -> "MixerExit": - self._scope.__enter__() - return self - - def __exit__(self, *exc_info): - return self._scope.__exit__(*exc_info) - - def finish( - self, hidden_states: torch.Tensor - ) -> Union[torch.Tensor, UnreducedOutput]: - """The mixer's output: its partial sum to the FFN stage after it, as a - value that owes the sum to a mixer after it, or complete.""" - if self._hands_on: - return UnreducedOutput(hidden_states, group=get_parallel().tp_group) - return hidden_states - - -class FfnExit: - """The scope that publishes an FfnCompletion while the FFN runs: inside the - ``with`` block it is ``fuse_mlp_allreduce`` / ``mlp_reduce_scatter`` / - ``defer_moe_finalize`` on ``get_forward()``.""" - - __slots__ = ( - "communicator", - "forward_batch", - "defer_moe_finalize", - "fuse_mlp_allreduce", - "mlp_reduce_scatter", - "_complete", - "_scope", - ) - - def __init__(self, communicator: LayerCommunicator, forward_batch: ForwardBatch): - self.communicator = communicator - self.forward_batch = forward_batch - completion = communicator._select_ffn_completion(forward_batch) - self.defer_moe_finalize = completion.defer_moe_finalize - self.fuse_mlp_allreduce = completion.fuse_mlp_allreduce - self.mlp_reduce_scatter = completion.mlp_reduce_scatter - self._complete = completion.complete - self._scope = get_forward().scoped( - fuse_mlp_allreduce=self.fuse_mlp_allreduce, - mlp_reduce_scatter=self.mlp_reduce_scatter, - defer_moe_finalize=self.defer_moe_finalize, - ) - - def __enter__(self) -> "FfnExit": - self._scope.__enter__() - return self - - def __exit__(self, *exc_info): - return self._scope.__exit__(*exc_info) - - def finish( - self, hidden_states: torch.Tensor, residual: torch.Tensor - ) -> Tuple[Union[torch.Tensor, UnreducedOutput], torch.Tensor]: - """Leave the reduction to the next layer's input, or postprocess.""" - if not isinstance(hidden_states, torch.Tensor): - # A deferred MoE finalize handoff, consumed by the next prepare_attn. - assert self.defer_moe_finalize, "unrequested deferred MoE handoff" - return hidden_states, residual - return self._complete(hidden_states, residual) - - -@dataclass -class CommunicateContext: - process_group_sizes: Dict[ScatterMode, int] - attn_tp_rank: int - attn_tp_size: int - attn_dp_size: int - attn_cp_rank: int - attn_cp_size: int - tp_size: int - cache = None - tp_rank: int - force_layernorm_before_dp_gather: bool = False - - @classmethod - def init_new(cls): - attn_tp_rank = get_parallel().attn_tp_rank - attn_tp_size = get_parallel().attn_tp_size - attn_dp_size = get_parallel().attn_dp_size - attn_cp_size = get_parallel().attn_cp_size - attn_cp_rank = get_parallel().attn_cp_rank - tp_size = get_parallel().tp_size - tp_rank = get_parallel().tp_rank - moe_cp_size = get_moe_cp_size() - process_group_sizes = { - ScatterMode.SCATTERED: 1, - ScatterMode.TP_ATTN_FULL: attn_tp_size, - # TODO: support --moe-dense-tp-size > 1 - # With context parallel enabled, we should exclude - # the attn_cp_size from the total tp_size - ScatterMode.FULL: tp_size // attn_cp_size, - ScatterMode.MOE_FULL: tp_size // (attn_cp_size // moe_cp_size), - } - return cls( - process_group_sizes=process_group_sizes, - attn_tp_rank=attn_tp_rank, - attn_tp_size=attn_tp_size, - attn_dp_size=attn_dp_size, - attn_cp_rank=attn_cp_rank, - attn_cp_size=attn_cp_size, - tp_size=tp_size, - tp_rank=tp_rank, - ) - - -class CommunicateSimpleFn: - @staticmethod - def _trivial( - hidden_states: torch.Tensor, - forward_batch: ForwardBatch, - context: CommunicateContext, - ) -> torch.Tensor: - return hidden_states - - @staticmethod - def _scattered_to_tp_attn_full( - hidden_states: Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]], - forward_batch: ForwardBatch, - context: CommunicateContext, - ) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]: - if isinstance(hidden_states, tuple): - gathered_hidden_states = [] - for local_hidden_states in hidden_states: - with use_symmetric_memory( - get_parallel().tp_group, - disabled=not is_allocation_symmetric(), - ): - output = torch.empty( - ( - local_hidden_states.shape[0] * context.attn_tp_size, - *local_hidden_states.shape[1:], - ), - dtype=local_hidden_states.dtype, - device=local_hidden_states.device, - ) - attn_tp_all_gather_into_tensor( - output, - local_hidden_states, - ) - gathered_hidden_states.append(output) - return tuple(gathered_hidden_states) - - return _redistribute_from_attn_tp_shards(hidden_states) - - -def _redistribute_from_attn_tp_shards(tensor: torch.Tensor) -> torch.Tensor: - gathered = get_local_dp_buffer( - get_parallel().attn_tp_group, hidden_size=tensor.shape[-1] - ) - attn_tp_all_gather_into_tensor(gathered, tensor) - return gathered - - -def _redistribute_to_attn_tp_shards( - tensor: torch.Tensor, context: CommunicateContext -) -> torch.Tensor: - return tensor.tensor_split(context.attn_tp_size)[context.attn_tp_rank] - - -def _reduce_and_redistribute_output_to_attn_tp_shards( - hidden_states: torch.Tensor, context: CommunicateContext -) -> torch.Tensor: - local_hidden_states = hidden_states.tensor_split(context.attn_tp_size)[ - context.attn_tp_rank - ] - attn_tp_reduce_scatter_tensor(local_hidden_states, hidden_states) - return local_hidden_states - - -def moe_cp_gathered_rows(forward_batch: ForwardBatch) -> Optional[List[int]]: - """The real rows each MoE-CP rank contributes when this batch's FFN input is - gathered across the MoE-CP group, or None when it is not: only a context - parallel extend with CP metadata gathers, and only when the group has more - than one rank. The batch is read first, so a batch that is not a CP extend - never reads the group.""" - if ( - forward_batch.forward_mode.is_context_parallel_extend() - and forward_batch.attn_cp_metadata is not None - and get_moe_cp_size() > 1 - ): - return forward_batch.attn_cp_metadata.per_rank_actual_token - return None - - -def sparse_moe_gathers_over_moe_cp() -> bool: - """Whether a sparse MoE's input is gathered over the MoE-CP group on a CP - extend: a MoE on the TP group under GQA CP whose MoE-CP group is wider than - the MoE's data-parallel groups. DSA and MLA CP gather over attention CP - instead.""" - return ( - not is_moe_input_scattered_across_dp_ranks() - and is_enable_moe_cp_allgather() - and not _gathers_over_attention_cp() - ) - - -def moe_cp_gathers_sparse_moe_input(forward_batch: ForwardBatch) -> bool: - """Whether a sparse MoE's input is gathered over the MoE-CP group on this - batch: a CP extend, for a MoE that gathers there.""" - return ( - sparse_moe_gathers_over_moe_cp() - and moe_cp_gathered_rows(forward_batch) is not None - ) - - -def _redistribute_input_to_moe_cp( - hidden_states: torch.Tensor, rows: List[int], moe_cp_size: int -) -> torch.Tensor: - # Zigzag split can produce unequal token counts across CP ranks - # (when seq_len % (cp_size * 2) != 0). NCCL allgather requires - # equal input sizes, so pad to the max per-rank token count. - max_tokens = max(rows) - pad_size = max_tokens - hidden_states.shape[0] - if pad_size > 0: - hidden_states = torch.nn.functional.pad(hidden_states, [0, 0, 0, pad_size]) - - output = torch.empty( - (max_tokens * moe_cp_size, hidden_states.shape[1]), - dtype=hidden_states.dtype, - device=hidden_states.device, - ) - moe_cp_all_gather_into_tensor(output, hidden_states) - return output - - -def _mlp_input_reduce_output( - hidden_states: torch.Tensor, forward_batch: ForwardBatch, may_quantize: bool = True -) -> torch.Tensor: - if ( - may_quantize - and not forward_batch.forward_mode.is_decode_or_idle() - and get_exec().comm.enable_quant_communications - ): - return attention_tensor_model_parallel_quant_all_reduce(hidden_states) - return attention_tensor_model_parallel_all_reduce(hidden_states) - - -def _cp_shard_token_rows(forward_batch: ForwardBatch) -> List[int]: - """Rows of each CP rank's shard that hold tokens; the shards are padded to - one length after them.""" - metadata = forward_batch.attn_cp_metadata - return metadata.per_rank_logical_token or metadata.per_rank_actual_token - - -def _redistribute_input_to_dp( - hidden_states: torch.Tensor, - forward_batch: ForwardBatch, - cp_shard_counts: Optional[List[int]] = None, -) -> torch.Tensor: - global_hidden_states = get_global_dp_buffer(get_parallel().tp_group) - dp_gather_replicate( - global_hidden_states, hidden_states, forward_batch, cp_shard_counts - ) - return global_hidden_states - - -def _reduce_and_redistribute_output_to_dp( - hidden_states: torch.Tensor, - forward_batch: ForwardBatch, - cp_shard_counts: Optional[List[int]] = None, -) -> torch.Tensor: - global_hidden_states = get_global_dp_buffer(get_parallel().tp_group) - dp_gather_partial( - global_hidden_states, hidden_states, forward_batch, cp_shard_counts - ) - return global_hidden_states - - -def move_rows( - hidden_states: torch.Tensor, - rows: Layout, - to: Layout, - forward_batch: ForwardBatch, -) -> torch.Tensor: - """A complete value from the rows it is on to ``to``: gathered over the - token axes ``to`` does not shard (attention TP, then attention DP), or cut to - this rank's share of those it does (attention DP, then attention TP).""" - gathered, cut = rows.sharded - to.sharded, to.sharded - rows.sharded - if (gathered and cut) or TokenAxis.ATTN_CP in gathered | cut: - raise NotImplementedError(f"{rows=} {to=}") - if TokenAxis.ATTN_TP_SCATTER in gathered: - hidden_states = _redistribute_from_attn_tp_shards(hidden_states) - if TokenAxis.ATTN_DP in gathered: - hidden_states = _redistribute_input_to_dp(hidden_states, forward_batch) - if TokenAxis.ATTN_DP in cut: - hidden_states = _to_local_tokens( - _redistribute_output, forward_batch, hidden_states - ) - if TokenAxis.ATTN_TP_SCATTER in cut: - parallel = get_parallel() - hidden_states = hidden_states.tensor_split(parallel.attn_tp_size)[ - parallel.attn_tp_rank - ] - return hidden_states - - -class FusedMlpInput(msgspec.Struct, frozen=True): - """A kernel that completes the sum the attention output owes together with - the residual add and the post-attention norm, in that order. - - ``run(hidden_states, residual, forward_batch)`` returns the FFN's input and - the residual, or None when it does not take the batch; it returns None only - before touching its inputs or starting a collective.""" - - # The group whose sum it completes. - completes: SumGroup - run: Callable[..., Optional[Tuple[torch.Tensor, torch.Tensor]]] - # It may hand back a new residual and leave the one it took unchanged. - may_return_new_residual: bool - - -def _mlp_input_without_dp( - hidden_states: torch.Tensor, - residual: torch.Tensor, - forward_batch: ForwardBatch, - layernorm: torch.nn.Module, - context: CommunicateContext, - *, - gathers_residual: bool, - fusions: Tuple[Callable, ...], - residual_ops: ResidualOps = ADD_AND_NORM, -): - if gathers_residual: - residual = residual_ops.residual_from_attn_tp_shards(residual) - for fused in fusions: - result = fused(hidden_states, residual, forward_batch) - if result is not None: - return result - # MHC sums its streams in full precision. - hidden_states = _mlp_input_reduce_output( - hidden_states, forward_batch, may_quantize=residual_ops.adds_plainly - ) - if _is_npu and context.cache is not None: - _ = prepare_weight_cache(hidden_states, context.cache) - return residual_ops.update_and_read_ffn_input(hidden_states, residual, layernorm) - - -def _mlp_input_dp_replicate( - hidden_states: torch.Tensor, - residual: torch.Tensor, - forward_batch: ForwardBatch, - layernorm: torch.nn.Module, - context: CommunicateContext, - *, - gathers_residual: bool, - reduces_attention_tp: bool, - places_cp_shards: bool = False, - residual_ops: ResidualOps = ADD_AND_NORM, -): - """Attention DP: complete the attention-TP sum if it is owed, write the - output into the residual and read the FFN input locally, then gather. With - ``places_cp_shards`` each CP rank puts its shard of the DP group's tokens - beside the others' in the group's slot.""" - if gathers_residual: - residual = residual_ops.residual_from_attn_tp_shards(residual) - if hidden_states.shape[0] != 0: - if reduces_attention_tp: - hidden_states = attention_tensor_model_parallel_all_reduce(hidden_states) - with use_symmetric_memory( - get_parallel().tp_group, - disabled=not is_allocation_symmetric(), - ): - hidden_states, residual = residual_ops.update_and_read_ffn_input( - hidden_states, residual, layernorm - ) - else: - hidden_states, residual = residual_ops.update_and_read_ffn_input( - hidden_states, residual, layernorm - ) - cp_shard_counts = _cp_shard_token_rows(forward_batch) if places_cp_shards else None - return ( - _redistribute_input_to_dp(hidden_states, forward_batch, cp_shard_counts), - residual, - ) - - -def _mlp_input_dp_partial( - hidden_states: torch.Tensor, - residual: torch.Tensor, - forward_batch: ForwardBatch, - layernorm: torch.nn.Module, - context: CommunicateContext, - *, - gathers_residual: bool, - places_cp_shards: bool = False, -): - """Attention DP: one rank adds the residual, the gather sums it, then - normalize. With ``places_cp_shards`` each CP rank puts its shard of the DP - group's tokens beside the others' in the group's slot.""" - if gathers_residual: - residual = _redistribute_from_attn_tp_shards(residual) - if context.attn_tp_rank == 0: - hidden_states += residual - cp_shard_counts = _cp_shard_token_rows(forward_batch) if places_cp_shards else None - hidden_states = _reduce_and_redistribute_output_to_dp( - hidden_states, forward_batch, cp_shard_counts - ) - dp_scatter(residual, hidden_states, forward_batch, cp_shard_counts) - if hidden_states.shape[0] != 0: - hidden_states = layernorm(hidden_states) - return hidden_states, residual - - -def _sum_group(group: SumGroup) -> GroupCoordinator: - parallel = get_parallel() - if group is SumGroup.ATTN_TP: - return parallel.attn_tp_group - if group is SumGroup.TP: - return parallel.tp_group - return post_experts_reduction_group() - - -def token_axis_sizes(*, cp_active: bool = False) -> Dict[TokenAxis, int]: - """The token axes' sizes for a batch: attention CP shards tokens only on - a CP extend (``cp_active``); otherwise every CP rank holds them all.""" - parallel = get_parallel() - return { - TokenAxis.ATTN_DP: parallel.attn_dp_size, - TokenAxis.ATTN_CP: parallel.attn_cp_size if cp_active else 1, - TokenAxis.ATTN_TP_SCATTER: parallel.attn_tp_size, - } - - -def tbo_split_moves(layer_input_rows: Layout) -> Tuple[Callable, Callable]: - """The moves around the two-batch-overlap split, which cuts the attention's - rows: from the rows the first overlapped layer takes to the attention's, - and back again for each half.""" - attention = Layout.sharded_over( - TokenAxis.ATTN_DP, TokenAxis.ATTN_CP, axis_sizes=token_axis_sizes() - ) - pair = CommunicateSummableTensorPairFn - if layer_input_rows == attention: - return pair._trivial, pair._trivial - if layer_input_rows.sharded - attention.sharded == {TokenAxis.ATTN_TP_SCATTER}: - # Each rank's slice: write the residual in and gather over attention - # TP, then take the slice of each half. - return pair._gather, pair._scatter - raise NotImplementedError(f"{layer_input_rows=}") - - -def _gathers_over_attention_cp() -> bool: - """Whether a CP extend gathers the FFN input over the attention-CP group in - equal shards and takes the output back with a reduce-scatter there: DSA and - MLA CP. GQA prefill CP gathers over the MoE-CP group instead.""" - return is_dsa_enable_prefill_cp() or is_mla_cp_enabled() - - -def _batch_shards_over_cp(forward_batch: ForwardBatch) -> bool: - """Whether this batch's tokens are split across the CP ranks. Only a context - parallel extend is, so other batches, decode graph capture among them, - never read the CP predicates.""" - if not forward_batch.forward_mode.is_context_parallel_extend(): - return False - if _gathers_over_attention_cp(): - return dsa_use_prefill_cp(forward_batch) or is_mla_cp_active(forward_batch) - return moe_cp_gathered_rows(forward_batch) is not None - - -class CpMoves(msgspec.Struct, frozen=True): - """How a CP extend's rows reach an FFN that needs all of them and come back, - chosen once for the kind of prefill CP: ``gather`` gathers the FFN input - after each rank has completed its own block, ``take_back`` returns this - rank's block of a complete output, and ``reduce_scatter``, where there is - one, completes a sum left over the ranks of ``reduce_scatter_group()`` and - returns the block in the same collective.""" - - gather: Callable - take_back: Callable - reduce_scatter: Optional[Callable] = None - reduce_scatter_group: Optional[Callable[[], GroupCoordinator]] = None - - -def _cp_moves() -> CpMoves: - """DSA and MLA CP gather equal shards over the attention-CP group and can - complete a sum over it. GQA prefill CP gathers blocks padded to the longest - over the MoE-CP group and takes back only a complete output.""" - if _gathers_over_attention_cp(): - return CpMoves( - gather=_mlp_input_gather_attention_cp, - take_back=CommunicateSummableTensorPairFn._take_back_attention_cp_shard, - reduce_scatter=CommunicateSummableTensorPairFn._reduce_scatter_over_cp, - reduce_scatter_group=lambda: get_parallel().attn_cp_group, - ) - return CpMoves( - gather=_mlp_input_gather_moe_cp, - take_back=CommunicateSummableTensorPairFn._scatter_hidden_states_moe, - ) - - -def _same_ranks(a: GroupCoordinator, b: GroupCoordinator) -> bool: - return sorted(a.ranks) == sorted(b.ranks) - - -def _hand_qkv_hook_its_input( - hidden_states: torch.Tensor, - forward_batch: ForwardBatch, - qkv_latent_func: Optional[Callable], -) -> torch.Tensor: - """Give the attention's QKV hook the attention input.""" - if qkv_latent_func is not None: - get_attn_tp_context().set_attn_inputs( - AttentionInputs(hidden_states, forward_batch, qkv_latent_func) - ) - return hidden_states - - -def _hand_scattered_input_to_attention( - hidden_states: torch.Tensor, - forward_batch: ForwardBatch, - qkv_latent_func: Optional[Callable], -) -> torch.Tensor: - """Input-scattered attention takes each rank's slice, and its QKV hook - gathers the rows after the projection. DSA and attention without a hook - consume full hidden states, so those are gathered here, and the hook is - told they are.""" - ctx = get_attn_tp_context() - if ctx.is_dsa or qkv_latent_func is None: - hidden_states = _tp_all_gather_scattered_rows(hidden_states, forward_batch) - if qkv_latent_func is not None: - ctx.set_attn_inputs( - AttentionInputs( - hidden_states, - forward_batch, - qkv_latent_func, - is_pre_gathered=ctx.is_dsa, - ) - ) - return hidden_states - - -def _tp_all_gather_scattered_rows( - hidden_states: torch.Tensor, forward_batch: ForwardBatch -) -> torch.Tensor: - # Input-scattered attention keeps the same number of tokens on every TP rank. - total_tokens = forward_batch.input_ids.shape[0] - output = hidden_states.new_empty((total_tokens, hidden_states.shape[-1])) - get_parallel().tp_group.all_gather_into_tensor(output, hidden_states) - return output - - -class BoundarySteps(msgspec.Struct, frozen=True): - """The steps a batch runs at a layer's boundaries: into the attention, - from the attention output to the FFN input, and the FFN output on to the - rows the layer hands on.""" - - # The half into the attention: completes what the input owes, writes the - # previous output into the residual and reads the attention input - # (_attention_input_step); attention_input then moves it. - attention_prepare: Callable - attention_input: Callable - ffn_input: Callable - # The rows ffn_input hands the FFN. - ffn_input_rows: Layout - # What the FFN exit reads: the FFN output's group and what it may leave. - ffn_output: StageOutput - # The postprocess that moves the FFN output on; None when it goes back over - # attention DP, whose step the FFN exit and postprocess choose per batch. - ffn_output_move: Optional[Callable] - # Whether the next layer's input can take the FFN's sum. - ffn_sum_is_movable: bool - # Whether ffn_output_move also completes the sum the FFN leaves. - ffn_output_move_completes_sum: bool = False - # The fused kernels ffn_input tries first. - fused: Tuple["FusedMlpInput", ...] = () - # Hands the attention its input once attention_input has moved it: - # (hidden_states, forward_batch, qkv_latent_func) -> hidden_states. - attention_handoff: Callable = _hand_qkv_hook_its_input - - @property - def returns_over_dp(self) -> bool: - return self.ffn_output_move is None - - -def _attention_input_step( - hidden_states: Union[torch.Tensor, "UnreducedOutput"], - residual: Optional[torch.Tensor], - forward_batch: ForwardBatch, - norm: torch.nn.Module, - context: "CommunicateContext", - *, - quant_format: str, - post_residual_addition: Optional[torch.Tensor], - layer_input: Optional[Callable], - fusions: Tuple[Callable, ...], - enters_stack: bool, - residual_ops: ResidualOps, -) -> Tuple[torch.Tensor, torch.Tensor]: - """A boundary's half into an attention: complete what the previous layer - left (a value that owes a sum, or a producer's handoff one of ``fusions`` - consumes with the add and norm), what the input owes by construction - (``layer_input``), then write the previous output into the residual and read - the attention input with ``norm``. The layer stack's first layer - (``enters_stack``) starts its residual from its input.""" - enters = residual is None and enters_stack - owed = None if isinstance(hidden_states, torch.Tensor) else hidden_states - if owed is not None and residual is None: - raise RuntimeError(f"{type(owed).__name__} requires residual input") - if isinstance(owed, UnreducedOutput) and owed.reduce_and_redistribute is not None: - # No fused kernel runs under attention DP: the reduce-scatter back to - # this rank's tokens comes first. - hidden_states, owed = reduce_output(owed), None - if owed is not None: - for fused in fusions: - result = fused(owed, residual, forward_batch, post_residual_addition) - if result is not None: - return result - if isinstance(owed, HandoffOutput): - # No fused kernel took the handoff: its producer completes it. - hidden_states, owed = reduce_output(owed), None - else: - hidden_states = owed.partial - if layer_input is not None: - hidden_states, residual = layer_input(hidden_states, residual, context) - if enters: - hidden_states, residual = residual_ops.enter(hidden_states), None - if owed is not None and hidden_states.shape[0] != 0: - hidden_states = reduce_output(owed) - if residual is None: - # The previous layer already wrote its output into the residual. - return residual_ops.read_attention_input(hidden_states, norm, quant_format) - return residual_ops.update_and_read_attention_input( - hidden_states, residual, norm, quant_format, post_residual_addition - ) - - -def _another_stage(*args, **kwargs): - """The steps of a stage a single-stage layer does not have.""" - raise RuntimeError("this layer is one stage and does not have the other") - - -class InputRead(Enum): - """How a boundary's consumer reads its input from the residual: with the - attention input norm (prepare_attn) or with the FFN input norm and its - fused kernels (prepare_mlp).""" - - ATTENTION = auto() - FFN = auto() - - -class LayerStage(msgspec.Struct, frozen=True): - """A layer that is one stage of a sequence of stages, each an - attention-like mixer or an FFN: how it reads its input, its two boundaries - (into it, and out of it onto the rows every layer hands on, as - ``boundary_layout.stage_edges`` gives them), and whether the layer stack - starts at it.""" - - reads: InputRead - edges: Tuple[EdgeDecl, EdgeDecl] - enters_stack: bool = False - - -class Boundary(msgspec.Struct, frozen=True): - """The steps one layer runs at one boundary, chosen from both sides' - declarations. A layer runs the consumer's half of a boundary into one of - its stages, and the producer's half of the boundary after its last stage; - the neighbouring layer runs the other half of that one.""" - - edge: EdgeDecl - # The consumer's half: completing what the input owes, the add and the - # norm; into the FFN also the move onto the rows it needs. - prepare: Optional[Callable] = None - # Into an attention, the move onto its rows after prepare. - input_move: Optional[Callable] = None - # The fused kernels an FFN's prepare tries first. - fused: Tuple["FusedMlpInput", ...] = () - # The producer's half: the postprocess that moves the output onto the rows - # the layer hands on; None when it goes back over attention DP, whose step - # the FFN exit and postprocess choose per batch. - output_move: Optional[Callable] = None - # Whether output_move also completes the sum the producer leaves. - output_move_completes_sum: bool = False - - @property - def input_rows(self) -> Layout: - """The rows the consumer is handed: what it needs, still sharded over - the axes it gathers itself.""" - need = self.edge.need - return Layout( - need.layout.sharded - | (self.edge.produced.layout.sharded & need.gathers_itself) - ) - - -def make_boundary( - edge: EdgeDecl, - *, - reads: Optional[InputRead], - fusions: Tuple = (), - force_layernorm_before_gather: bool = False, - cp_moves: Optional[CpMoves] = None, - residual_ops: ResidualOps = ADD_AND_NORM, - enters_stack: bool = False, -) -> Boundary: - """The steps a layer runs at ``edge``, around the residual operations; - ``cp_moves`` for an edge that gathers over or returns across attention CP. - ``reads`` is how the consumer reads its input, or None when the consumer - runs in the next layer and ``edge.need`` is the rows this layer hands on: - then only the producer's half runs here. ``fusions`` are the fused kernels - the consumer tries first (FusedMlpInput into an FFN, the attention input's - entries into an attention); ``enters_stack`` for the edge into the layer - stack's first attention. The consumer's half reads only this edge's - declarations, never what the producer chose for a batch; a sum left for a - batch arrives with the value.""" - if reads is None: - if edge.need.layout != edge.residual_to: - raise NotImplementedError(f"{edge=}") - returns_over_dp, output_move, completes_sum = _select_ffn_output_move( - edge.produced, - residual=edge.residual, - to=edge.residual_to, - cp_moves=cp_moves, - residual_ops=residual_ops, - ) - return Boundary( - edge, - output_move=None if returns_over_dp else output_move, - output_move_completes_sum=completes_sum, - ) - if reads is InputRead.FFN: - input_step, fused = _select_ffn_input( - edge.produced, - residual=edge.residual, - residual_to=edge.residual_to, - need=edge.need, - force_layernorm_before_gather=force_layernorm_before_gather, - fusions=fusions, - residual_joins_sum=edge.residual_joins_sum, - cp_moves=cp_moves, - residual_ops=residual_ops, - ) - return Boundary(edge, prepare=input_step, fused=fused) - layer_input = None - if edge.produced.always_leaves: - # A reduce-scatter completes the TP sum onto each rank's slice, which the - # attention takes: its group is the TP group without attention DP or CP. - if ( - edge.produced.group is not SumGroup.TP - or edge.residual.sharded - or TokenAxis.ATTN_TP_SCATTER not in edge.need.gathers_itself - or edge.residual_to.sharded != {TokenAxis.ATTN_TP_SCATTER} - ): - raise NotImplementedError(f"{edge=}") - layer_input = tp_reduce_scatter - return Boundary( - edge, - prepare=partial( - _attention_input_step, - layer_input=layer_input, - fusions=fusions, - enters_stack=enters_stack, - residual_ops=residual_ops, - ), - input_move=_select_attention_input_move(edge.residual_to, edge.need), - ) - - -def _select_boundary_steps( - sides: DecoderLayerSides, - *, - fusions: Tuple["FusedMlpInput", ...] = (), - force_layernorm_before_gather: bool = False, - cp_moves: Optional[CpMoves] = None, - residual_ops: ResidualOps = ADD_AND_NORM, - attention_handoff: Callable = _hand_qkv_hook_its_input, - attention_fusions: Tuple[Callable, ...] = (), - enters_stack: bool = False, -) -> BoundarySteps: - """The steps of a decoder layer: its three boundaries, each chosen from the - declarations of its two sides, around the layer's residual operations. Both - edges that cross attention CP take the same ``cp_moves``; the attention's - input tries ``attention_fusions``, and the layer stack's first layer - (``enters_stack``) starts its residual there.""" - edges = decoder_layer_edges(sides) - out_of_ffn = make_boundary( - edges.out_of_ffn, reads=None, cp_moves=cp_moves, residual_ops=residual_ops - ) - into_ffn = make_boundary( - edges.into_ffn, - reads=InputRead.FFN, - fusions=fusions, - force_layernorm_before_gather=force_layernorm_before_gather, - cp_moves=cp_moves, - residual_ops=residual_ops, - ) - into_attention = make_boundary( - edges.into_attention, - reads=InputRead.ATTENTION, - fusions=attention_fusions, - residual_ops=residual_ops, - enters_stack=enters_stack, - ) - return BoundarySteps( - attention_prepare=into_attention.prepare, - attention_input=into_attention.input_move, - ffn_input=into_ffn.prepare, - ffn_input_rows=into_ffn.input_rows, - ffn_output=edges.out_of_ffn.produced, - ffn_output_move=out_of_ffn.output_move, - ffn_output_move_completes_sum=out_of_ffn.output_move_completes_sum, - ffn_sum_is_movable=edges.out_of_ffn.produced.group is not None, - fused=into_ffn.fused, - attention_handoff=attention_handoff, - ) - - -def _select_attention_input_move(rows: Layout, need: StageInput) -> Callable: - """How the rows a layer takes become its attention's input: as they are, - or gathered over attention TP from each rank's slice, unless the attention - gathers them itself.""" - gathered = rows.sharded - need.layout.sharded - need.gathers_itself - if not need.layout.sharded <= rows.sharded or gathered not in ( - frozenset(), - {TokenAxis.ATTN_TP_SCATTER}, - ): - raise NotImplementedError(f"{rows=} {need=}") - if gathered: - return CommunicateSimpleFn._scattered_to_tp_attn_full - return CommunicateSimpleFn._trivial - - -def _select_ffn_input( - produced: StageOutput, - *, - residual: Layout, - residual_to: Layout, - need: StageInput, - force_layernorm_before_gather: bool, - fusions: Tuple[FusedMlpInput, ...], - residual_joins_sum: bool = False, - cp_moves: Optional[CpMoves] = None, - residual_ops: ResidualOps = ADD_AND_NORM, -) -> Tuple[Callable, Tuple[FusedMlpInput, ...]]: - """The steps from the attention output to the FFN input, and the fused - kernels they try first: complete the attention-TP sum, move the residual to - the rows it has while the FFN runs, write the output into it and read the - FFN input, and bring the rows to what the FFN's group needs: a gather over - attention DP, or each rank's own slice. A kernel in ``fusions`` is tried - only when nothing is gathered or sliced, and only if it completes the sum - the attention output owes. A write-back that is not a plain add runs only - after the sum completes.""" - if produced.leaves_for_next_layer and produced.group not in ( - None, - SumGroup.ATTN_TP, - ): - # A producer that leaves its sum only for some batches (an FFN before - # this one) hands that sum on with the value: complete it, then the - # input is a complete output. - step, fused = _select_ffn_input( - StageOutput(produced.layout), - residual=residual, - residual_to=residual_to, - need=need, - force_layernorm_before_gather=force_layernorm_before_gather, - fusions=fusions, - residual_joins_sum=residual_joins_sum, - cp_moves=cp_moves, - residual_ops=residual_ops, - ) - return partial(_mlp_input_completing_owed, step=step), fused - # What the attention output owes decides the steps: the attention-TP sum, - # always left by the output projection, or nothing. - owes_attention_tp = produced.group is SumGroup.ATTN_TP - if owes_attention_tp != produced.always_leaves or produced.group not in ( - None, - SumGroup.ATTN_TP, - ): - raise NotImplementedError(f"{produced=}") - gathered = produced.layout.sharded - need.layout.sharded - need.gathers_itself - sliced = need.layout.sharded - produced.layout.sharded - if sliced: - # Each attention-TP rank takes its own slice: the reduce-scatter - # completes the attention-TP sum and slices in one collective; a - # complete value is only sliced. - if ( - sliced != {TokenAxis.ATTN_TP_SCATTER} - or gathered - or residual_to != need.layout - or residual not in (produced.layout, need.layout) - ): - raise NotImplementedError(f"{produced=} {residual=} {need=}") - return ( - partial( - _mlp_input_scatter if owes_attention_tp else _mlp_input_slice, - scatters_residual=residual != residual_to, - residual_ops=residual_ops, - ), - (), - ) - if residual_to.sharded - produced.layout.sharded == {TokenAxis.ATTN_TP_SCATTER}: - # The residual stays on each rank's slice while the FFN takes the - # attention's rows (MHC on an input-scattered batch). - if gathered or not owes_attention_tp or residual != residual_to: - raise NotImplementedError(f"{produced=} {residual=} {need=}") - return partial(_mlp_input_on_residual_shard, residual_ops=residual_ops), () - if gathered == {TokenAxis.ATTN_CP}: - # Each CP rank completes its own chunk, then the CP moves gather them. - if cp_moves is None: - raise NotImplementedError(f"{produced=} {need=}") - on_chunk, fused = _select_ffn_input( - produced, - residual=residual, - residual_to=residual_to, - need=StageInput(produced.layout), - force_layernorm_before_gather=force_layernorm_before_gather, - fusions=fusions, - residual_joins_sum=residual_joins_sum, - residual_ops=residual_ops, - ) - return partial(cp_moves.gather, gather=on_chunk), fused - if ( - residual_to != produced.layout - or gathered - not in ( - frozenset(), - {TokenAxis.ATTN_DP}, - {TokenAxis.ATTN_DP, TokenAxis.ATTN_CP}, - ) - or residual.sharded - residual_to.sharded - not in (frozenset(), {TokenAxis.ATTN_TP_SCATTER}) - ): - raise NotImplementedError(f"{produced=} {residual=} {need=}") - # A residual arriving on each rank's slice is gathered back first. - gathers_residual = residual != residual_to - if not gathered: - if not owes_attention_tp: - return partial(_mlp_input_norm, residual_ops=residual_ops), () - if gathers_residual and residual_joins_sum: - # Each rank adds its slice of the residual into its share of the - # sum, so the all-reduce also brings the residual back to every row. - if not residual_ops.adds_plainly: - raise NotImplementedError(f"{produced=} {residual=} {need=}") - return _mlp_input_residual_into_sum, () - fused = tuple(f for f in fusions if f.completes is produced.group) - return ( - partial( - _mlp_input_without_dp, - gathers_residual=gathers_residual, - fusions=tuple(f.run for f in fused), - residual_ops=residual_ops, - ), - fused, - ) - # The partial order adds the residual on attention-TP rank 0 before the DP - # gather's collective completes that sum, which only a plain residual add - # allows. - # Over attention DP and CP, the DP gather puts each CP rank's shard in its - # DP group's slot, so the one DP sum gathers both axes. - places_cp_shards = TokenAxis.ATTN_CP in gathered - if ( - owes_attention_tp - and not force_layernorm_before_gather - and residual_ops.adds_plainly - ): - return ( - partial( - _mlp_input_dp_partial, - gathers_residual=gathers_residual, - places_cp_shards=places_cp_shards, - ), - (), - ) - return ( - partial( - _mlp_input_dp_replicate, - gathers_residual=gathers_residual, - reduces_attention_tp=owes_attention_tp, - places_cp_shards=places_cp_shards, - residual_ops=residual_ops, - ), - (), - ) - - -def _select_ffn_output_move( - produced: StageOutput, - *, - residual: Layout, - to: Layout, - cp_moves: Optional[CpMoves] = None, - residual_ops: ResidualOps = ADD_AND_NORM, -) -> Tuple[bool, Optional[Callable], bool]: - """How the FFN output reaches the rows the layer hands on: whether it goes - back by undoing the attention-DP gather (the FFN exit and postprocess run - that step), or else the postprocess that moves it, None when there is none - to choose here; and whether that move also completes the sum the FFN - leaves.""" - pair = CommunicateSummableTensorPairFn - if produced.layout == residual: - if to == residual: - return False, pair._trivial, False - if to.sharded == residual.sharded - {TokenAxis.ATTN_TP_SCATTER}: - # Each rank's slice back to the attention's rows: write the output - # into the residual, then gather over attention TP. - return False, partial(pair._gather, residual_ops=residual_ops), False - raise NotImplementedError(f"{produced=} {residual=} {to=}") - returned = residual.sharded - produced.layout.sharded - if returned == {TokenAxis.ATTN_TP_SCATTER} and to in (residual, produced.layout): - # The residual stays on each rank's slice (MHC on an input-scattered - # batch): a reduce-scatter onto the slice completes the sum the FFN - # leaves; a complete output is only sliced. - sums = produced.leaves_for_reduce_scatter - return ( - False, - partial( - pair._onto_residual_shard, - sums=sums, - gathers_back=to != residual, - residual_ops=residual_ops, - ), - sums, - ) - if to != residual or not produced.layout.sharded <= residual.sharded: - raise NotImplementedError(f"{produced=} {residual=} {to=}") - if returned == {TokenAxis.ATTN_CP}: - if cp_moves is None: - raise NotImplementedError(f"{produced=} {residual=} {to=}") - if not produced.leaves_for_reduce_scatter: - # A complete output: this rank's block of it, nothing summed. - return False, cp_moves.take_back, False - # The FFN leaves its sum: only a take-back that sums over the same - # ranks completes it. - if cp_moves.reduce_scatter is None or not _same_ranks( - _sum_group(produced.group), cp_moves.reduce_scatter_group() - ): - raise NotImplementedError(f"{produced=} {residual=} {to=}") - return False, cp_moves.reduce_scatter, True - if returned == {TokenAxis.ATTN_DP, TokenAxis.ATTN_CP}: - # This rank's CP shard, from where the DP gather put it. - return False, CommunicateSummableTensorPairFn._take_back_cp_shard, False - if returned != {TokenAxis.ATTN_DP}: - raise NotImplementedError(f"{produced=} {residual=} {to=}") - return True, None, False - - -def _mlp_input_completing_owed( - hidden_states: Union[torch.Tensor, "UnreducedOutput"], - residual: torch.Tensor, - forward_batch: ForwardBatch, - layernorm: torch.nn.Module, - context: CommunicateContext, - *, - step: Callable, -): - """An FFN input whose producer may have left its sum: complete what the - value owes, then run ``step`` on the complete output.""" - return step( - reduce_output(hidden_states), residual, forward_batch, layernorm, context - ) - - -def _mlp_input_norm( - hidden_states: torch.Tensor, - residual: torch.Tensor, - forward_batch: ForwardBatch, - layernorm: torch.nn.Module, - context: CommunicateContext, - *, - residual_ops: ResidualOps = ADD_AND_NORM, -): - return residual_ops.update_and_read_ffn_input(hidden_states, residual, layernorm) - - -def _mlp_input_scatter( - hidden_states: torch.Tensor, - residual: torch.Tensor, - forward_batch: ForwardBatch, - layernorm: torch.nn.Module, - context: CommunicateContext, - *, - scatters_residual: bool, - residual_ops: ResidualOps = ADD_AND_NORM, -): - hidden_states = _reduce_and_redistribute_output_to_attn_tp_shards( - hidden_states, context - ) - if scatters_residual: - residual = residual_ops.residual_to_attn_tp_shard(residual, context) - return residual_ops.update_and_read_ffn_input(hidden_states, residual, layernorm) - - -def _mlp_input_slice( - hidden_states: torch.Tensor, - residual: Optional[torch.Tensor], - forward_batch: ForwardBatch, - layernorm: torch.nn.Module, - context: CommunicateContext, - *, - scatters_residual: bool, - residual_ops: ResidualOps = ADD_AND_NORM, -): - hidden_states = _redistribute_to_attn_tp_shards(hidden_states, context).clone() - if scatters_residual and residual is not None: - residual = residual_ops.residual_to_attn_tp_shard(residual, context) - return residual_ops.update_and_read_ffn_input(hidden_states, residual, layernorm) - - -def _mlp_input_on_residual_shard( - hidden_states: torch.Tensor, - residual: torch.Tensor, - forward_batch: ForwardBatch, - layernorm: torch.nn.Module, - context: CommunicateContext, - *, - residual_ops: ResidualOps, -): - """The residual stays on each rank's slice while the FFN takes the full - rows: reduce-scatter the attention output onto the slice, which completes - its sum, write it into the residual and read the FFN input there, then - gather the input back into the attention output's rows.""" - if hidden_states.shape[0] == 0: - return hidden_states, hidden_states - shard = hidden_states.tensor_split(context.tp_size)[context.tp_rank] - get_parallel().tp_group.reduce_scatter_tensor(shard, hidden_states) - shard, residual = residual_ops.update_and_read_ffn_input(shard, residual, layernorm) - attn_tp_all_gather_into_tensor(hidden_states, shard) - return hidden_states, residual - - -def _mlp_input_residual_into_sum( - hidden_states: torch.Tensor, - residual: torch.Tensor, - forward_batch: ForwardBatch, - layernorm: torch.nn.Module, - context: CommunicateContext, -): - return _tp_all_reduce_with_scattered_residual( - hidden_states, residual, layernorm, context - ) - - -def _tp_all_reduce_with_scattered_residual( - hidden_states: torch.Tensor, - residual: torch.Tensor, - layernorm: torch.nn.Module, - context: CommunicateContext, -): - if hidden_states.shape[0] == 0: - return hidden_states, hidden_states - - scattered_states = hidden_states.tensor_split(context.tp_size)[context.tp_rank] - scattered_states += residual - residual = tensor_model_parallel_all_reduce(hidden_states) - hidden_states = layernorm(residual) - return hidden_states, residual - - -def _mlp_input_gather_attention_cp( - hidden_states: torch.Tensor, - residual: torch.Tensor, - forward_batch: ForwardBatch, - layernorm: torch.nn.Module, - context: CommunicateContext, - *, - gather: Callable, -): - """DSA and MLA CP: complete this rank's shard, then gather the shards, of - equal length, over the attention-CP group. The residual stays on the - shard.""" - hidden_states, residual = gather( - hidden_states, residual, forward_batch, layernorm, context - ) - return dsa_cp_gather_hidden_states(hidden_states), residual - - -def _mlp_input_gather_moe_cp( - hidden_states: torch.Tensor, - residual: torch.Tensor, - forward_batch: ForwardBatch, - layernorm: torch.nn.Module, - context: CommunicateContext, - *, - gather: Callable, -): - """Gather for the FFN, then over the MoE-CP group so each rank holds all - tokens of its MoE group (moe_dp_size < attn_cp_size). The residual stays at - TP_ATTN_FULL.""" - # Early return on empty tensor is safe for MOE_CP because: - # - During CP extend: zigzag split guarantees all CP ranks have non-zero tokens, - # so no rank hits this path while others proceed to the allgather. - # - During decode: moe_cp allgather is skipped (guarded by is_context_parallel_extend). - # - CUDA graph warmup: not applicable when --cuda-graph-backend-prefill=disabled is used. - if hidden_states.shape[0] == 0: - return hidden_states, residual - - hidden_states, residual = gather( - hidden_states, residual, forward_batch, layernorm, context - ) - - rows = moe_cp_gathered_rows(forward_batch) - if rows is not None and hidden_states.shape[0] > 0: - hidden_states = _redistribute_input_to_moe_cp( - hidden_states, rows, get_moe_cp_size() - ) - - return hidden_states, residual - - -def _dp_scatter_group() -> GroupCoordinator: - parallel = get_parallel() - if parallel.tp_size == parallel.attn_dp_size: - return parallel.tp_group - return parallel.attn_tp_group - - -def _reduce_and_redistribute_output_varlen( - local_hidden_states: torch.Tensor, - hidden_states: torch.Tensor, - forward_batch: ForwardBatch, -) -> None: - get_parallel().tp_group.reduce_scatterv( - hidden_states, - output=local_hidden_states, - sizes=get_dp_global_num_tokens(), - ) - - -def _reduce_and_redistribute_output_max_len( - local_hidden_states: torch.Tensor, - hidden_states: torch.Tensor, - forward_batch: ForwardBatch, -) -> None: - dp_reduce_scatter_tensor(local_hidden_states, hidden_states) - - -def _redistribute_output( - local_hidden_states: torch.Tensor, - hidden_states: torch.Tensor, - forward_batch: ForwardBatch, -) -> None: - dp_scatter(local_hidden_states, hidden_states, forward_batch) - - -def _redistribute_output_from_moe_cp( - hidden_states: torch.Tensor, rows: List[int] -) -> torch.Tensor: - moe_cp_rank = get_moe_cp_rank() - # The allgather was padded to max_tokens_per_rank (equal chunks). - # Extract this rank's actual (non-padded) tokens from its chunk. - max_tokens_per_rank = max(rows) - actual_local_tokens = rows[moe_cp_rank] - return hidden_states.narrow( - 0, moe_cp_rank * max_tokens_per_rank, actual_local_tokens - ).contiguous() - - -def _reduce_and_redistribute_output_step( - forward_batch: ForwardBatch, - *, - leaves_for_reduce_scatter: bool, - leaves_for_reduce_scatterv: bool, -) -> Optional[Callable[[torch.Tensor, torch.Tensor, ForwardBatch], None]]: - """The reduce-scatter that brings an FFN output gathered over attention - DP back to this rank's tokens when the FFN leaves its sum to it (see - StageOutput); None when the FFN reduces the output and only a scatter - remains.""" - if should_use_dp_reduce_scatterv() and leaves_for_reduce_scatterv: - return _reduce_and_redistribute_output_varlen - if ( - leaves_for_reduce_scatter - and forward_batch.dp_padding_mode.is_max_len() - and can_use_dp_reduce_scatter() - ): - return _reduce_and_redistribute_output_max_len - return None - - -def _to_local_tokens( - step: Callable[[torch.Tensor, torch.Tensor, ForwardBatch], None], - forward_batch: ForwardBatch, - hidden_states: torch.Tensor, -) -> torch.Tensor: - local_hidden_states = get_local_dp_buffer(_dp_scatter_group()) - step(local_hidden_states, hidden_states, forward_batch) - return local_hidden_states - - -def _all_reduce_then_to_local_tokens( - group: GroupCoordinator, forward_batch: ForwardBatch, hidden_states: torch.Tensor -) -> torch.Tensor: - return _to_local_tokens( - _redistribute_output, forward_batch, group.all_reduce(hidden_states) - ) - - -class CommunicateSummableTensorPairFn: - """It is allowed to make (hidden_states, residual) := (hidden_states + residual, None) if needed.""" - - @staticmethod - def _trivial( - hidden_states: torch.Tensor, - residual: torch.Tensor, - forward_batch: ForwardBatch, - context: CommunicateContext, - **kwargs, - ): - return hidden_states, residual - - @staticmethod - def _scatter_hidden_states( - hidden_states: torch.Tensor, - residual: torch.Tensor, - forward_batch: ForwardBatch, - context: CommunicateContext, - allow_reduce_scatter: bool = False, - is_layer_sparse: bool = False, - ): - step = ( - _reduce_and_redistribute_output_step( - forward_batch, - leaves_for_reduce_scatter=allow_reduce_scatter, - # A MoE block leaves its sum to reduce_scatterv whenever it - # applies (should_skip_post_experts_all_reduce). - leaves_for_reduce_scatterv=allow_reduce_scatter or is_layer_sparse, - ) - or _redistribute_output - ) - return _to_local_tokens(step, forward_batch, hidden_states), residual - - @staticmethod - def _gather( - hidden_states: torch.Tensor, - residual: torch.Tensor, - forward_batch: ForwardBatch, - context: CommunicateContext, - residual_ops: ResidualOps = ADD_AND_NORM, - **kwargs, - ): - hidden_states = residual_ops.update_residual(hidden_states, residual) - return _redistribute_from_attn_tp_shards(hidden_states), None - - @staticmethod - def _onto_residual_shard( - hidden_states: torch.Tensor, - residual: torch.Tensor, - forward_batch: ForwardBatch, - context: CommunicateContext, - *, - sums: bool, - gathers_back: bool, - residual_ops: ResidualOps, - **kwargs, - ): - """Bring the FFN output onto the slice of the rows the residual is on: - a reduce-scatter that also completes its sum when ``sums``, else this - rank's slice of the complete output. With ``gathers_back`` it is - written into the residual there and the full rows are gathered back.""" - if sums: - hidden_states, _ = tp_reduce_scatter(hidden_states, None, context) - else: - hidden_states = hidden_states.tensor_split(context.tp_size)[context.tp_rank] - if not gathers_back: - return hidden_states, residual - local_states = residual_ops.update_residual(hidden_states, residual) - hidden_states = local_states.new_empty( - local_states.shape[0] * context.tp_size, *local_states.shape[1:] - ) - get_parallel().tp_group.all_gather_into_tensor(hidden_states, local_states) - return hidden_states, None - - @staticmethod - def _scatter( - hidden_states: torch.Tensor, - residual: torch.Tensor, - forward_batch: ForwardBatch, - context: CommunicateContext, - ): - assert residual is None, "not yet handled residual!=None" - return _redistribute_to_attn_tp_shards(hidden_states, context), None - - @staticmethod - def _take_back_attention_cp_shard( - hidden_states: torch.Tensor, - residual: torch.Tensor, - forward_batch: ForwardBatch, - context: CommunicateContext, - **kwargs, - ): - """DSA and MLA CP: this rank's shard of a complete output gathered in - equal shards over the attention-CP group.""" - parallel = get_parallel() - shard = hidden_states.tensor_split(parallel.attn_cp_size)[parallel.attn_cp_rank] - return shard, residual - - @staticmethod - def _reduce_scatter_over_cp( - hidden_states: torch.Tensor, - residual: torch.Tensor, - forward_batch: ForwardBatch, - context: CommunicateContext, - **kwargs, - ): - """DSA and MLA CP: sum the FFN output over the attention-CP group and - keep this rank's shard.""" - return dsa_cp_reduce_scatter_hidden_states(hidden_states), residual - - @staticmethod - def _take_back_cp_shard( - hidden_states: torch.Tensor, - residual: torch.Tensor, - forward_batch: ForwardBatch, - context: CommunicateContext, - **kwargs, - ): - """This rank's CP shard of the rows the DP gather put in its DP group's - slot, at the shard's padded length with the padding zeroed.""" - held = forward_batch.attn_cp_metadata.per_rank_actual_token - local_hidden_states = get_local_dp_buffer(_dp_scatter_group())[ - : held[get_parallel().attn_cp_rank] - ] - dp_scatter( - local_hidden_states, - hidden_states, - forward_batch, - _cp_shard_token_rows(forward_batch), - ) - return local_hidden_states, residual - - @staticmethod - def _scatter_hidden_states_moe( - hidden_states: torch.Tensor, - residual: torch.Tensor, - forward_batch: ForwardBatch, - context: CommunicateContext, - **kwargs, - ): - """Scatter MoE output back to TP_ATTN_FULL after MOE_FULL computation. - - After moe_tensor_model_parallel_all_reduce (which runs unconditionally since - mlp_reduce_scatter=False for this path), all ranks in the moe_cp group hold the - full MoE result for all cp_per_moe token chunks. We simply slice out this rank's - CP-local portion. - - If DP>1, further scatter back to the local DP slice. - """ - # Only scatter back during prefill; decode was never allgathered so no-op. - # Safe w.r.t. empty tensors: same reasoning as _mlp_input_gather_moe_cp - # — CP extend always has non-zero tokens per rank, and decode skips this path. - rows = moe_cp_gathered_rows(forward_batch) - if rows is not None: - hidden_states = _redistribute_output_from_moe_cp(hidden_states, rows) - - if context.attn_dp_size > 1: - hidden_states = _to_local_tokens( - _redistribute_output, forward_batch, hidden_states - ) - - return hidden_states, residual diff --git a/python/sglang/srt/layers/communicator/__init__.py b/python/sglang/srt/layers/communicator/__init__.py new file mode 100644 index 0000000000..4ff31e3fcc --- /dev/null +++ b/python/sglang/srt/layers/communicator/__init__.py @@ -0,0 +1,124 @@ +# Copyright 2023-2024 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Communication between a decoder layer's stages.""" + +from sglang.srt.layers.communicator.adapters.attention import ( + AttentionInputs, + get_attn_tp_context, +) +from sglang.srt.layers.communicator.boundary import ( + BoundarySteps, + DecoderLayerSides, + EdgeDecl, + FusedMlpInput, + InputRead, + LayerStage, + StageDecl, + StageInput, + StageOutput, + decoder_layer_edges, + decoder_layer_sides, + input_scattered_layer_sides, + make_boundary, + sequence_parallel_layer_sides, + stage_edges, + tbo_split_moves, +) +from sglang.srt.layers.communicator.layer import ( + FfnCompletion, + FfnExit, + FfnExitFusion, + LayerCommunicator, + LayerScatterModes, + MHCLayerCommunicator, + MixerExit, +) +from sglang.srt.layers.communicator.layout import ( + CommunicateContext, + Layout, + ScatterMode, + SumGroup, + TokenAxis, + enable_moe_dense_fully_dp, + moe_cp_gathers_sparse_moe_input, + sparse_moe_gathers_over_moe_cp, + token_axis_sizes, +) +from sglang.srt.layers.communicator.ops import ( + CommunicateSimpleFn, + CommunicateSummableTensorPairFn, + layer_input_buffer, + move_rows, + tp_reduce_scatter, +) +from sglang.srt.layers.communicator.output import ( + HandoffOutput, + UnreducedOutput, + reduce_output, +) +from sglang.srt.layers.communicator.residual.add_norm import ( + ADD_AND_NORM, + FUSE_ALLREDUCE_MAX_BATCH_SIZE, +) +from sglang.srt.layers.communicator.residual.mhc import ( + MHCState, +) + +__all__ = [ + "ADD_AND_NORM", + "AttentionInputs", + "BoundarySteps", + "CommunicateContext", + "CommunicateSimpleFn", + "CommunicateSummableTensorPairFn", + "DecoderLayerSides", + "EdgeDecl", + "FUSE_ALLREDUCE_MAX_BATCH_SIZE", + "FfnCompletion", + "FfnExit", + "FfnExitFusion", + "FusedMlpInput", + "HandoffOutput", + "InputRead", + "LayerCommunicator", + "LayerScatterModes", + "LayerStage", + "Layout", + "MHCLayerCommunicator", + "MHCState", + "MixerExit", + "ScatterMode", + "StageDecl", + "StageInput", + "StageOutput", + "SumGroup", + "TokenAxis", + "UnreducedOutput", + "decoder_layer_edges", + "decoder_layer_sides", + "enable_moe_dense_fully_dp", + "get_attn_tp_context", + "input_scattered_layer_sides", + "layer_input_buffer", + "make_boundary", + "moe_cp_gathers_sparse_moe_input", + "move_rows", + "reduce_output", + "sequence_parallel_layer_sides", + "sparse_moe_gathers_over_moe_cp", + "stage_edges", + "tbo_split_moves", + "token_axis_sizes", + "tp_reduce_scatter", +] diff --git a/python/sglang/srt/layers/communicator/adapters/__init__.py b/python/sglang/srt/layers/communicator/adapters/__init__.py new file mode 100644 index 0000000000..61d2eafbfd --- /dev/null +++ b/python/sglang/srt/layers/communicator/adapters/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2023-2024 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""What the communicator takes from the modules it serves.""" diff --git a/python/sglang/srt/layers/communicator/adapters/attention.py b/python/sglang/srt/layers/communicator/adapters/attention.py new file mode 100644 index 0000000000..5050a597f6 --- /dev/null +++ b/python/sglang/srt/layers/communicator/adapters/attention.py @@ -0,0 +1,188 @@ +# Copyright 2023-2024 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""The attention's inputs: the attention-TP context, its cached inputs, and +the attention-TP gather and slice.""" + +import logging +from contextlib import contextmanager +from typing import Callable + +import torch + +from sglang.srt.layers.communicator.layout import ( + CommunicateContext, + enable_moe_dense_fully_dp, +) +from sglang.srt.layers.dp_attention import ( + attn_tp_all_gather_into_tensor, + get_local_dp_buffer, + is_dp_attention_enabled, +) +from sglang.srt.layers.moe import get_moe_a2a_backend +from sglang.srt.model_executor.cuda_graph_config import ( + Backend, + Phase, + check_cuda_graph_backend, +) +from sglang.srt.model_executor.forward_batch_info import ForwardBatch +from sglang.srt.runtime_context import get_forward, get_parallel, get_spec +from sglang.srt.utils import is_cuda, is_npu + +_is_cuda = is_cuda() +_is_npu = is_npu() + + +class AttentionInputs: + def __init__( + self, + hidden_states: torch.Tensor, + forward_batch: ForwardBatch, + qkv_latent_func: Callable, + *, + is_pre_gathered: bool = False, + ): + self.hidden_states_local = hidden_states + self.forward_batch = forward_batch + self.qkv_latent_func = qkv_latent_func + self.hidden_states_ = None + self.qkv_latent_ = None + # When True, hidden_states_local is already attn_tp-gathered upstream + # (e.g. by MHC's prepare_attn for DSA). fetch_* must NOT gather again. + self.is_pre_gathered = is_pre_gathered + + def tp_all_gather_hidden_states(self, hidden_states, forward_batch): + total_tokens = forward_batch.input_ids.shape[0] + output = hidden_states.new_empty((total_tokens, hidden_states.shape[-1])) + get_parallel().tp_group.all_gather_into_tensor(output, hidden_states) + return output + + def fetch_qkv_latent(self): + if self.qkv_latent_ is not None: + return self.qkv_latent_ + assert self.qkv_latent_func is not None + self.qkv_latent_ = self.qkv_latent_func( + self.hidden_states_local, self.forward_batch + ) + if get_attn_tp_context().input_scattered and not self.is_pre_gathered: + self.qkv_latent_ = self.tp_all_gather_hidden_states( + self.qkv_latent_, self.forward_batch + ) + return self.qkv_latent_ + + def fetch_hidden_states(self): + if self.hidden_states_ is not None: + return self.hidden_states_ + self.hidden_states_ = self.hidden_states_local + if get_attn_tp_context().input_scattered and not self.is_pre_gathered: + self.hidden_states_ = self.tp_all_gather_hidden_states( + self.hidden_states_, self.forward_batch + ) + return self.hidden_states_ + + +class AttnTpContext: + def __init__(self): + self.allow_input_scattered = False + self.is_dsa = False + + def init_context(self, q_lora_rank, is_dsa, is_mhc=False): + # Only MHC pre-gathers hidden states before DSA attention, so non-MHC DSA + # cannot use scattered inputs. + self.is_dsa = is_dsa + self.allow_input_scattered = ( + get_parallel().enable_attn_tp_input_scattered + and (_is_cuda or _is_npu) + and q_lora_rank is not None + and (is_mhc or not is_dsa) + and get_parallel().tp_size > 1 + and not is_dp_attention_enabled() + and get_moe_a2a_backend().is_none() + and not enable_moe_dense_fully_dp() + and not check_cuda_graph_backend(Phase.PREFILL, Backend.TC_PIECEWISE) + and get_spec().speculative_algorithm != "EAGLE3" + ) + if get_parallel().enable_attn_tp_input_scattered: + if not self.allow_input_scattered: + logging.info( + "attn_tp_input_scattered is not enabled while other conditions are not met" + ) + else: + logging.info("attn_tp_input_scattered is enabled") + + def use_input_scattered(self, forward_batch: ForwardBatch): + return ( + self.allow_input_scattered + and forward_batch.forward_mode.is_extend() + and not forward_batch.forward_mode.is_target_verify() + and forward_batch.input_ids is not None + and not forward_batch.can_run_tbo + ) + + @property + def input_scattered(self): + return get_forward().attn_input_scattered + + def set_attn_inputs(self, attn_inputs: AttentionInputs): + get_forward().set("attn_inputs", attn_inputs) + + def set_hidden_states_local(self, hidden_states: torch.Tensor) -> None: + attn_inputs = get_forward().attn_inputs + if attn_inputs is not None: + attn_inputs.hidden_states_local = hidden_states + + def fetch_qkv_latent(self): + attn_inputs = get_forward().attn_inputs + assert attn_inputs is not None + return attn_inputs.fetch_qkv_latent() + + def fetch_hidden_states(self): + attn_inputs = get_forward().attn_inputs + assert attn_inputs is not None + return attn_inputs.fetch_hidden_states() + + def clear_attn_inputs(self) -> None: + get_forward().set("attn_inputs", None) + + @contextmanager + def maybe_input_scattered(self, forward_batch: ForwardBatch): + flag = self.use_input_scattered(forward_batch) + forward = get_forward() + # scoped() also restores when the forward raises — the old in-place + # swap leaked the flag on exceptions. + with forward.scoped(attn_input_scattered=flag): + try: + yield + finally: + forward.set("attn_inputs", None) + + +ATTN_TP_CONTEXT = AttnTpContext() + + +def get_attn_tp_context(): + return ATTN_TP_CONTEXT + + +def _redistribute_from_attn_tp_shards(tensor: torch.Tensor) -> torch.Tensor: + gathered = get_local_dp_buffer( + get_parallel().attn_tp_group, hidden_size=tensor.shape[-1] + ) + attn_tp_all_gather_into_tensor(gathered, tensor) + return gathered + + +def _redistribute_to_attn_tp_shards( + tensor: torch.Tensor, context: CommunicateContext +) -> torch.Tensor: + return tensor.tensor_split(context.attn_tp_size)[context.attn_tp_rank] diff --git a/python/sglang/srt/layers/communicator/boundary.py b/python/sglang/srt/layers/communicator/boundary.py new file mode 100644 index 0000000000..285abefa94 --- /dev/null +++ b/python/sglang/srt/layers/communicator/boundary.py @@ -0,0 +1,969 @@ +# Copyright 2023-2024 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""What each stage produces and needs, and the boundary steps chosen from those +declarations.""" + +from enum import Enum, auto +from functools import partial +from typing import Callable, FrozenSet, Mapping, Optional, Tuple, Union + +import msgspec +import torch + +from sglang.srt.distributed import GroupCoordinator +from sglang.srt.layers.communicator.layout import ( + CommunicateContext, + Layout, + SumGroup, + TokenAxis, + _gathers_over_attention_cp, + _same_ranks, + _sum_group, + token_axis_sizes, +) +from sglang.srt.layers.communicator.ops import ( + CommunicateSimpleFn, + CommunicateSummableTensorPairFn, + _hand_qkv_hook_its_input, + _mlp_input_completing_owed, + _mlp_input_dp_partial, + _mlp_input_dp_replicate, + _mlp_input_gather_attention_cp, + _mlp_input_gather_moe_cp, + _mlp_input_norm, + _mlp_input_on_residual_shard, + _mlp_input_residual_into_sum, + _mlp_input_scatter, + _mlp_input_slice, + _mlp_input_without_dp, + tp_reduce_scatter, +) +from sglang.srt.layers.communicator.output import ( + HandoffOutput, + UnreducedOutput, + reduce_output, +) +from sglang.srt.layers.communicator.residual import ResidualOps +from sglang.srt.layers.communicator.residual.add_norm import ADD_AND_NORM +from sglang.srt.model_executor.forward_batch_info import ForwardBatch +from sglang.srt.runtime_context import get_parallel + + +class StageInput(msgspec.Struct, frozen=True): + """The rows a stage's consumer needs: sharded over the token axes its + compute group does not span.""" + + layout: Layout + # Token axes the consumer gathers over itself when its input arrives + # sharded over them. + gathers_itself: FrozenSet[TokenAxis] = frozenset() + + +class StageOutput(msgspec.Struct, frozen=True): + """What a stage's producer hands the boundary after it, fixed at + construction: its rows, the group its output is summed over, and when it + leaves that sum to the boundary instead of completing it. + + What one output actually owes is carried with the value: a plain + tensor handed across layers is complete, an ``UnreducedOutput`` names what + is left. A raw compute output entering a boundary is read with this + declaration and the boundary's decision for the batch.""" + + layout: Layout + # None when there is nothing to sum. + group: Optional[SumGroup] = None + # The producer never reduces its output, e.g. a row-parallel projection + # built with reduce_results=False. + always_leaves: bool = False + # Otherwise it leaves the sum only when the boundary publishes the flag for + # it: fuse_mlp_allreduce, or mlp_reduce_scatter. + leaves_for_next_layer: bool = False + leaves_for_reduce_scatter: bool = False + # Whether it leaves the sum to the attention-DP reduce_scatterv whenever that + # combine applies, as a MoE block does without any flag. + leaves_for_reduce_scatterv: bool = False + + +class DecoderLayerSides(msgspec.Struct, frozen=True): + """What a decoder layer declares about its attention, its FFN and the rows + between them; ``decoder_layer_edges`` turns it into the layer's + boundaries.""" + + # The rows the layer's input and residual arrive on. + input_rows: Layout + attention: StageInput + attention_output: StageOutput + ffn: StageInput + ffn_output: StageOutput + # The rows the residual is on while the FFN runs. + ffn_residual_rows: Layout + # The rows the layer hands on. + output_rows: Layout + # The group the layer's input arrives as a partial sum over, if any. + input_owes: Optional[SumGroup] = None + # Whether the residual is added into one rank's share of the attention + # output's sum before that sum completes, instead of after it. + residual_joins_attention_sum: bool = False + + +class StageDecl(msgspec.Struct, frozen=True): + """A computing stage's two sides: the rows its input must be on, and what + its output is.""" + + input: StageInput + output: StageOutput + + +class EdgeDecl(msgspec.Struct, frozen=True): + """One boundary between two stages, as the layer that runs one side of it + sees it: what arrives from the producer, what the consumer needs, and the + rows the residual is on before and after the boundary.""" + + produced: StageOutput + need: StageInput + residual: Layout + residual_to: Layout + # Whether the residual is added into one rank's share of the produced sum + # before that sum completes, instead of after it. + residual_joins_sum: bool = False + + +class DecoderLayerEdges(msgspec.Struct, frozen=True): + """A decoder layer's three boundaries: from the rows the previous layer + handed on into the attention, from the attention into the FFN, and from + the FFN onto the rows the layer hands on. The next layer runs the rest of + the last one.""" + + into_attention: EdgeDecl + into_ffn: EdgeDecl + out_of_ffn: EdgeDecl + + +def decoder_layer_edges(sides: DecoderLayerSides) -> DecoderLayerEdges: + """The boundaries of a decoder layer, each with the declarations of its two + sides. What arrives at the layer is complete unless it owes a sum by + construction (``input_owes``); a sum the previous layer leaves for a batch + comes with the value.""" + owes = sides.input_owes is not None + arrived = StageOutput(sides.input_rows, group=sides.input_owes, always_leaves=owes) + # Completing what the input owes leaves it and the residual on each + # attention-TP rank's slice. + attention_rows = ( + Layout(sides.input_rows.sharded | {TokenAxis.ATTN_TP_SCATTER}) + if owes + else sides.input_rows + ) + return DecoderLayerEdges( + into_attention=EdgeDecl( + produced=arrived, + need=sides.attention, + residual=sides.input_rows, + residual_to=attention_rows, + ), + into_ffn=EdgeDecl( + produced=sides.attention_output, + need=sides.ffn, + residual=attention_rows, + residual_to=sides.ffn_residual_rows, + residual_joins_sum=sides.residual_joins_attention_sum, + ), + out_of_ffn=EdgeDecl( + produced=sides.ffn_output, + need=StageInput(sides.output_rows), + residual=sides.ffn_residual_rows, + residual_to=sides.output_rows, + ), + ) + + +def stage_edges( + *, previous: Optional[StageOutput], stage: StageDecl, rows: Layout +) -> Tuple[EdgeDecl, EdgeDecl]: + """The two boundaries of a layer that is one stage of a sequence of stages: + into it from the previous stage's output (None at the start of the layer + stack) and out of it onto ``rows``, the rows every layer hands on and the + residual is on between stages. The previous output arrives on those rows: a + stage whose output is elsewhere (an FFN on the TP group) moves it back there + first, and what it may leave of its sum comes with the value; otherwise the + value is complete. The residual follows the input onto a finer slice and + stays where it is when the input is gathered.""" + arrived = ( + StageOutput( + rows, + group=previous.group, + always_leaves=previous.always_leaves, + leaves_for_next_layer=previous.leaves_for_next_layer, + ) + if previous is not None + and (previous.always_leaves or previous.leaves_for_next_layer) + else StageOutput(rows) + ) + during = stage.input.layout if rows.sharded <= stage.input.layout.sharded else rows + return ( + EdgeDecl(produced=arrived, need=stage.input, residual=rows, residual_to=during), + EdgeDecl( + produced=stage.output, + need=StageInput(rows), + residual=during, + residual_to=rows, + ), + ) + + +def sequence_parallel_layer_sides( + *, axis_sizes: Mapping[TokenAxis, int] +) -> DecoderLayerSides: + """A decoder layer while a LayerNorm SP region is active. Its linears + all-gather their input and reduce-scatter their output themselves, so the + layer's rows, its residual and both outputs stay on each TP rank's slice.""" + attention = Layout.sharded_over( + TokenAxis.ATTN_DP, TokenAxis.ATTN_CP, axis_sizes=axis_sizes + ) + local = Layout.sharded_over( + TokenAxis.ATTN_DP, + TokenAxis.ATTN_CP, + TokenAxis.ATTN_TP_SCATTER, + axis_sizes=axis_sizes, + ) + gathers = frozenset({TokenAxis.ATTN_TP_SCATTER}) + return DecoderLayerSides( + input_rows=local, + # qkv and gate_up gather the slices into their GEMM. + attention=StageInput(attention, gathers_itself=gathers), + # o_proj and down reduce-scatter out of theirs, whether fused or not. + attention_output=StageOutput(local), + ffn=StageInput( + Layout.sharded_over(axis_sizes=axis_sizes), gathers_itself=gathers + ), + ffn_output=StageOutput(local), + ffn_residual_rows=local, + output_rows=local, + ) + + +def scattered_residual_layer_sides( + *, + axis_sizes: Mapping[TokenAxis, int], + ffn_group: SumGroup, + is_first_layer: bool, + is_last_layer: bool, + leaves_for_reduce_scatter: bool, +) -> DecoderLayerSides: + """A decoder layer on an input-scattered batch whose residual stays on + each attention-TP rank's slice (MHC). The attention and the FFN compute on + the full rows. The attention takes the slice and gathers it itself (its + QKV hook, or the handoff before an attention that needs the rows); the + boundary brings the FFN its rows and moves both outputs back onto the + slice: the attention output's TP sum by a reduce-scatter, the FFN's too + when it leaves it (``leaves_for_reduce_scatter``). The first layer's input + is the embedding's partial sum on the full rows; the last layer hands on + the full rows again.""" + attention = Layout.sharded_over( + TokenAxis.ATTN_DP, TokenAxis.ATTN_CP, axis_sizes=axis_sizes + ) + local = Layout.sharded_over( + TokenAxis.ATTN_DP, + TokenAxis.ATTN_CP, + TokenAxis.ATTN_TP_SCATTER, + axis_sizes=axis_sizes, + ) + return DecoderLayerSides( + input_rows=attention if is_first_layer else local, + attention=StageInput( + attention, gathers_itself=frozenset({TokenAxis.ATTN_TP_SCATTER}) + ), + attention_output=StageOutput( + attention, group=SumGroup.ATTN_TP, always_leaves=True + ), + ffn=StageInput(Layout.sharded_over(axis_sizes=axis_sizes)), + ffn_output=StageOutput( + attention, + group=ffn_group, + leaves_for_reduce_scatter=leaves_for_reduce_scatter, + ), + ffn_residual_rows=local, + output_rows=attention if is_last_layer else local, + input_owes=SumGroup.TP if is_first_layer else None, + ) + + +def input_scattered_layer_sides( + *, + axis_sizes: Mapping[TokenAxis, int], + ffn_group: SumGroup, + hands_on_partial: bool, +) -> DecoderLayerSides: + """A decoder layer on a batch whose attention input is scattered over + attention TP. Each layer's input is a partial sum over TP on the full rows: + the embedding and every FFN before it leave that sum, and a reduce-scatter + completes it onto each rank's slice, which the attention gathers itself. + The residual comes back to the full rows inside the attention output's sum.""" + attention = Layout.sharded_over( + TokenAxis.ATTN_DP, TokenAxis.ATTN_CP, axis_sizes=axis_sizes + ) + return DecoderLayerSides( + input_rows=attention, + attention=StageInput( + attention, gathers_itself=frozenset({TokenAxis.ATTN_TP_SCATTER}) + ), + attention_output=StageOutput( + attention, group=SumGroup.ATTN_TP, always_leaves=True + ), + ffn=StageInput(Layout.sharded_over(axis_sizes=axis_sizes)), + # The next layer's input completes the sum; the last layer's FFN does. + ffn_output=StageOutput( + attention, group=ffn_group, leaves_for_reduce_scatter=hands_on_partial + ), + ffn_residual_rows=attention, + output_rows=attention, + input_owes=SumGroup.TP, + residual_joins_attention_sum=True, + ) + + +def decoder_layer_sides( + *, + axis_sizes: Mapping[TokenAxis, int], + ffn_on_local_rows: bool, + previous_on_local_rows: bool, + is_last_layer: bool, + attention_gathers_local_rows: bool, + ffn_group: SumGroup, + leaves_for_next_layer: bool, + leaves_for_reduce_scatter: bool, + leaves_for_reduce_scatterv: bool, + hands_on_attention_rows: bool = False, + ffn_shards_over_cp: bool = False, +) -> DecoderLayerSides: + """An attention followed by an FFN, derived from the groups each computes + over. The FFN runs either on the TP group (a dense MLP, or a MoE not + dispatched per DP shard) or on this rank's local rows (a MoE dispatched per + DP shard, which completes its own combine, or a dense MLP on every rank). + A MoE whose data-parallel groups are the CP ranks (``ffn_shards_over_cp``) + computes each CP shard on its own ranks: its rows stay sharded over CP. + A layer on local rows hands on the attention's rows when it is the last, or + with ``hands_on_attention_rows``.""" + # Attention computes over the attention-TP ranks of one DP (and CP) shard. + attention = Layout.sharded_over( + TokenAxis.ATTN_DP, TokenAxis.ATTN_CP, axis_sizes=axis_sizes + ) + # Each attention-TP rank's own slice of those rows. + local = Layout.sharded_over( + TokenAxis.ATTN_DP, + TokenAxis.ATTN_CP, + TokenAxis.ATTN_TP_SCATTER, + axis_sizes=axis_sizes, + ) + # The TP group spans every token axis, so an FFN on it needs every row. + ffn = ( + local + if ffn_on_local_rows + else Layout.sharded_over( + *((TokenAxis.ATTN_CP,) if ffn_shards_over_cp else ()), + axis_sizes=axis_sizes, + ) + ) + attention_tp = axis_sizes[TokenAxis.ATTN_TP_SCATTER] > 1 + return DecoderLayerSides( + input_rows=local if previous_on_local_rows else attention, + attention=StageInput( + attention, + gathers_itself=( + frozenset({TokenAxis.ATTN_TP_SCATTER}) + if attention_gathers_local_rows + else frozenset() + ), + ), + # The output projection leaves the attention-TP sum to prepare_mlp. + attention_output=StageOutput( + attention, + group=SumGroup.ATTN_TP if attention_tp else None, + always_leaves=attention_tp, + ), + ffn=StageInput(ffn), + ffn_output=( + StageOutput(ffn) + if ffn_on_local_rows + else StageOutput( + ffn, + group=ffn_group, + leaves_for_next_layer=leaves_for_next_layer, + leaves_for_reduce_scatter=leaves_for_reduce_scatter, + leaves_for_reduce_scatterv=leaves_for_reduce_scatterv, + ) + ), + ffn_residual_rows=local if ffn_on_local_rows else attention, + output_rows=( + local + if ffn_on_local_rows and not (is_last_layer or hands_on_attention_rows) + else attention + ), + ) + + +class FusedMlpInput(msgspec.Struct, frozen=True): + """A kernel that completes the sum the attention output owes together with + the residual add and the post-attention norm, in that order. + + ``run(hidden_states, residual, forward_batch)`` returns the FFN's input and + the residual, or None when it does not take the batch; it returns None only + before touching its inputs or starting a collective.""" + + # The group whose sum it completes. + completes: SumGroup + run: Callable[..., Optional[Tuple[torch.Tensor, torch.Tensor]]] + # It may hand back a new residual and leave the one it took unchanged. + may_return_new_residual: bool + + +def tbo_split_moves(layer_input_rows: Layout) -> Tuple[Callable, Callable]: + """The moves around the two-batch-overlap split, which cuts the attention's + rows: from the rows the first overlapped layer takes to the attention's, + and back again for each half.""" + attention = Layout.sharded_over( + TokenAxis.ATTN_DP, TokenAxis.ATTN_CP, axis_sizes=token_axis_sizes() + ) + pair = CommunicateSummableTensorPairFn + if layer_input_rows == attention: + return pair._trivial, pair._trivial + if layer_input_rows.sharded - attention.sharded == {TokenAxis.ATTN_TP_SCATTER}: + # Each rank's slice: write the residual in and gather over attention + # TP, then take the slice of each half. + return pair._gather, pair._scatter + raise NotImplementedError(f"{layer_input_rows=}") + + +class CpMoves(msgspec.Struct, frozen=True): + """How a CP extend's rows reach an FFN that needs all of them and come back, + chosen once for the kind of prefill CP: ``gather`` gathers the FFN input + after each rank has completed its own block, ``take_back`` returns this + rank's block of a complete output, and ``reduce_scatter``, where there is + one, completes a sum left over the ranks of ``reduce_scatter_group()`` and + returns the block in the same collective.""" + + gather: Callable + take_back: Callable + reduce_scatter: Optional[Callable] = None + reduce_scatter_group: Optional[Callable[[], GroupCoordinator]] = None + + +def _cp_moves() -> CpMoves: + """DSA and MLA CP gather equal shards over the attention-CP group and can + complete a sum over it. GQA prefill CP gathers blocks padded to the longest + over the MoE-CP group and takes back only a complete output.""" + if _gathers_over_attention_cp(): + return CpMoves( + gather=_mlp_input_gather_attention_cp, + take_back=CommunicateSummableTensorPairFn._take_back_attention_cp_shard, + reduce_scatter=CommunicateSummableTensorPairFn._reduce_scatter_over_cp, + reduce_scatter_group=lambda: get_parallel().attn_cp_group, + ) + return CpMoves( + gather=_mlp_input_gather_moe_cp, + take_back=CommunicateSummableTensorPairFn._scatter_hidden_states_moe, + ) + + +class BoundarySteps(msgspec.Struct, frozen=True): + """The steps a batch runs at a layer's boundaries: into the attention, + from the attention output to the FFN input, and the FFN output on to the + rows the layer hands on.""" + + # The half into the attention: completes what the input owes, writes the + # previous output into the residual and reads the attention input + # (_attention_input_step); attention_input then moves it. + attention_prepare: Callable + attention_input: Callable + ffn_input: Callable + # The rows ffn_input hands the FFN. + ffn_input_rows: Layout + # What the FFN exit reads: the FFN output's group and what it may leave. + ffn_output: StageOutput + # The postprocess that moves the FFN output on; None when it goes back over + # attention DP, whose step the FFN exit and postprocess choose per batch. + ffn_output_move: Optional[Callable] + # Whether the next layer's input can take the FFN's sum. + ffn_sum_is_movable: bool + # Whether ffn_output_move also completes the sum the FFN leaves. + ffn_output_move_completes_sum: bool = False + # The fused kernels ffn_input tries first. + fused: Tuple["FusedMlpInput", ...] = () + # Hands the attention its input once attention_input has moved it: + # (hidden_states, forward_batch, qkv_latent_func) -> hidden_states. + attention_handoff: Callable = _hand_qkv_hook_its_input + + @property + def returns_over_dp(self) -> bool: + return self.ffn_output_move is None + + +def _attention_input_step( + hidden_states: Union[torch.Tensor, "UnreducedOutput"], + residual: Optional[torch.Tensor], + forward_batch: ForwardBatch, + norm: torch.nn.Module, + context: "CommunicateContext", + *, + quant_format: str, + post_residual_addition: Optional[torch.Tensor], + layer_input: Optional[Callable], + fusions: Tuple[Callable, ...], + enters_stack: bool, + residual_ops: ResidualOps, +) -> Tuple[torch.Tensor, torch.Tensor]: + """A boundary's half into an attention: complete what the previous layer + left (a value that owes a sum, or a producer's handoff one of ``fusions`` + consumes with the add and norm), what the input owes by construction + (``layer_input``), then write the previous output into the residual and read + the attention input with ``norm``. The layer stack's first layer + (``enters_stack``) starts its residual from its input.""" + enters = residual is None and enters_stack + owed = None if isinstance(hidden_states, torch.Tensor) else hidden_states + if owed is not None and residual is None: + raise RuntimeError(f"{type(owed).__name__} requires residual input") + if isinstance(owed, UnreducedOutput) and owed.reduce_and_redistribute is not None: + # No fused kernel runs under attention DP: the reduce-scatter back to + # this rank's tokens comes first. + hidden_states, owed = reduce_output(owed), None + if owed is not None: + for fused in fusions: + result = fused(owed, residual, forward_batch, post_residual_addition) + if result is not None: + return result + if isinstance(owed, HandoffOutput): + # No fused kernel took the handoff: its producer completes it. + hidden_states, owed = reduce_output(owed), None + else: + hidden_states = owed.partial + if layer_input is not None: + hidden_states, residual = layer_input(hidden_states, residual, context) + if enters: + hidden_states, residual = residual_ops.enter(hidden_states), None + if owed is not None and hidden_states.shape[0] != 0: + hidden_states = reduce_output(owed) + if residual is None: + # The previous layer already wrote its output into the residual. + return residual_ops.read_attention_input(hidden_states, norm, quant_format) + return residual_ops.update_and_read_attention_input( + hidden_states, residual, norm, quant_format, post_residual_addition + ) + + +def _another_stage(*args, **kwargs): + """The steps of a stage a single-stage layer does not have.""" + raise RuntimeError("this layer is one stage and does not have the other") + + +class InputRead(Enum): + """How a boundary's consumer reads its input from the residual: with the + attention input norm (prepare_attn) or with the FFN input norm and its + fused kernels (prepare_mlp).""" + + ATTENTION = auto() + FFN = auto() + + +class LayerStage(msgspec.Struct, frozen=True): + """A layer that is one stage of a sequence of stages, each an + attention-like mixer or an FFN: how it reads its input, its two boundaries + (into it, and out of it onto the rows every layer hands on, as + ``stage_edges`` gives them), and whether the layer stack + starts at it.""" + + reads: InputRead + edges: Tuple[EdgeDecl, EdgeDecl] + enters_stack: bool = False + + +class Boundary(msgspec.Struct, frozen=True): + """The steps one layer runs at one boundary, chosen from both sides' + declarations. A layer runs the consumer's half of a boundary into one of + its stages, and the producer's half of the boundary after its last stage; + the neighbouring layer runs the other half of that one.""" + + edge: EdgeDecl + # The consumer's half: completing what the input owes, the add and the + # norm; into the FFN also the move onto the rows it needs. + prepare: Optional[Callable] = None + # Into an attention, the move onto its rows after prepare. + input_move: Optional[Callable] = None + # The fused kernels an FFN's prepare tries first. + fused: Tuple["FusedMlpInput", ...] = () + # The producer's half: the postprocess that moves the output onto the rows + # the layer hands on; None when it goes back over attention DP, whose step + # the FFN exit and postprocess choose per batch. + output_move: Optional[Callable] = None + # Whether output_move also completes the sum the producer leaves. + output_move_completes_sum: bool = False + + @property + def input_rows(self) -> Layout: + """The rows the consumer is handed: what it needs, still sharded over + the axes it gathers itself.""" + need = self.edge.need + return Layout( + need.layout.sharded + | (self.edge.produced.layout.sharded & need.gathers_itself) + ) + + +def make_boundary( + edge: EdgeDecl, + *, + reads: Optional[InputRead], + fusions: Tuple = (), + force_layernorm_before_gather: bool = False, + cp_moves: Optional[CpMoves] = None, + residual_ops: ResidualOps = ADD_AND_NORM, + enters_stack: bool = False, +) -> Boundary: + """The steps a layer runs at ``edge``, around the residual operations; + ``cp_moves`` for an edge that gathers over or returns across attention CP. + ``reads`` is how the consumer reads its input, or None when the consumer + runs in the next layer and ``edge.need`` is the rows this layer hands on: + then only the producer's half runs here. ``fusions`` are the fused kernels + the consumer tries first (FusedMlpInput into an FFN, the attention input's + entries into an attention); ``enters_stack`` for the edge into the layer + stack's first attention. The consumer's half reads only this edge's + declarations, never what the producer chose for a batch; a sum left for a + batch arrives with the value.""" + if reads is None: + if edge.need.layout != edge.residual_to: + raise NotImplementedError(f"{edge=}") + returns_over_dp, output_move, completes_sum = _select_ffn_output_move( + edge.produced, + residual=edge.residual, + to=edge.residual_to, + cp_moves=cp_moves, + residual_ops=residual_ops, + ) + return Boundary( + edge, + output_move=None if returns_over_dp else output_move, + output_move_completes_sum=completes_sum, + ) + if reads is InputRead.FFN: + input_step, fused = _select_ffn_input( + edge.produced, + residual=edge.residual, + residual_to=edge.residual_to, + need=edge.need, + force_layernorm_before_gather=force_layernorm_before_gather, + fusions=fusions, + residual_joins_sum=edge.residual_joins_sum, + cp_moves=cp_moves, + residual_ops=residual_ops, + ) + return Boundary(edge, prepare=input_step, fused=fused) + layer_input = None + if edge.produced.always_leaves: + # A reduce-scatter completes the TP sum onto each rank's slice, which the + # attention takes: its group is the TP group without attention DP or CP. + if ( + edge.produced.group is not SumGroup.TP + or edge.residual.sharded + or TokenAxis.ATTN_TP_SCATTER not in edge.need.gathers_itself + or edge.residual_to.sharded != {TokenAxis.ATTN_TP_SCATTER} + ): + raise NotImplementedError(f"{edge=}") + layer_input = tp_reduce_scatter + return Boundary( + edge, + prepare=partial( + _attention_input_step, + layer_input=layer_input, + fusions=fusions, + enters_stack=enters_stack, + residual_ops=residual_ops, + ), + input_move=_select_attention_input_move(edge.residual_to, edge.need), + ) + + +def _select_boundary_steps( + sides: DecoderLayerSides, + *, + fusions: Tuple["FusedMlpInput", ...] = (), + force_layernorm_before_gather: bool = False, + cp_moves: Optional[CpMoves] = None, + residual_ops: ResidualOps = ADD_AND_NORM, + attention_handoff: Callable = _hand_qkv_hook_its_input, + attention_fusions: Tuple[Callable, ...] = (), + enters_stack: bool = False, +) -> BoundarySteps: + """The steps of a decoder layer: its three boundaries, each chosen from the + declarations of its two sides, around the layer's residual operations. Both + edges that cross attention CP take the same ``cp_moves``; the attention's + input tries ``attention_fusions``, and the layer stack's first layer + (``enters_stack``) starts its residual there.""" + edges = decoder_layer_edges(sides) + out_of_ffn = make_boundary( + edges.out_of_ffn, reads=None, cp_moves=cp_moves, residual_ops=residual_ops + ) + into_ffn = make_boundary( + edges.into_ffn, + reads=InputRead.FFN, + fusions=fusions, + force_layernorm_before_gather=force_layernorm_before_gather, + cp_moves=cp_moves, + residual_ops=residual_ops, + ) + into_attention = make_boundary( + edges.into_attention, + reads=InputRead.ATTENTION, + fusions=attention_fusions, + residual_ops=residual_ops, + enters_stack=enters_stack, + ) + return BoundarySteps( + attention_prepare=into_attention.prepare, + attention_input=into_attention.input_move, + ffn_input=into_ffn.prepare, + ffn_input_rows=into_ffn.input_rows, + ffn_output=edges.out_of_ffn.produced, + ffn_output_move=out_of_ffn.output_move, + ffn_output_move_completes_sum=out_of_ffn.output_move_completes_sum, + ffn_sum_is_movable=edges.out_of_ffn.produced.group is not None, + fused=into_ffn.fused, + attention_handoff=attention_handoff, + ) + + +def _select_attention_input_move(rows: Layout, need: StageInput) -> Callable: + """How the rows a layer takes become its attention's input: as they are, + or gathered over attention TP from each rank's slice, unless the attention + gathers them itself.""" + gathered = rows.sharded - need.layout.sharded - need.gathers_itself + if not need.layout.sharded <= rows.sharded or gathered not in ( + frozenset(), + {TokenAxis.ATTN_TP_SCATTER}, + ): + raise NotImplementedError(f"{rows=} {need=}") + if gathered: + return CommunicateSimpleFn._scattered_to_tp_attn_full + return CommunicateSimpleFn._trivial + + +def _select_ffn_input( + produced: StageOutput, + *, + residual: Layout, + residual_to: Layout, + need: StageInput, + force_layernorm_before_gather: bool, + fusions: Tuple[FusedMlpInput, ...], + residual_joins_sum: bool = False, + cp_moves: Optional[CpMoves] = None, + residual_ops: ResidualOps = ADD_AND_NORM, +) -> Tuple[Callable, Tuple[FusedMlpInput, ...]]: + """The steps from the attention output to the FFN input, and the fused + kernels they try first: complete the attention-TP sum, move the residual to + the rows it has while the FFN runs, write the output into it and read the + FFN input, and bring the rows to what the FFN's group needs: a gather over + attention DP, or each rank's own slice. A kernel in ``fusions`` is tried + only when nothing is gathered or sliced, and only if it completes the sum + the attention output owes. A write-back that is not a plain add runs only + after the sum completes.""" + if produced.leaves_for_next_layer and produced.group not in ( + None, + SumGroup.ATTN_TP, + ): + # A producer that leaves its sum only for some batches (an FFN before + # this one) hands that sum on with the value: complete it, then the + # input is a complete output. + step, fused = _select_ffn_input( + StageOutput(produced.layout), + residual=residual, + residual_to=residual_to, + need=need, + force_layernorm_before_gather=force_layernorm_before_gather, + fusions=fusions, + residual_joins_sum=residual_joins_sum, + cp_moves=cp_moves, + residual_ops=residual_ops, + ) + return partial(_mlp_input_completing_owed, step=step), fused + # What the attention output owes decides the steps: the attention-TP sum, + # always left by the output projection, or nothing. + owes_attention_tp = produced.group is SumGroup.ATTN_TP + if owes_attention_tp != produced.always_leaves or produced.group not in ( + None, + SumGroup.ATTN_TP, + ): + raise NotImplementedError(f"{produced=}") + gathered = produced.layout.sharded - need.layout.sharded - need.gathers_itself + sliced = need.layout.sharded - produced.layout.sharded + if sliced: + # Each attention-TP rank takes its own slice: the reduce-scatter + # completes the attention-TP sum and slices in one collective; a + # complete value is only sliced. + if ( + sliced != {TokenAxis.ATTN_TP_SCATTER} + or gathered + or residual_to != need.layout + or residual not in (produced.layout, need.layout) + ): + raise NotImplementedError(f"{produced=} {residual=} {need=}") + return ( + partial( + _mlp_input_scatter if owes_attention_tp else _mlp_input_slice, + scatters_residual=residual != residual_to, + residual_ops=residual_ops, + ), + (), + ) + if residual_to.sharded - produced.layout.sharded == {TokenAxis.ATTN_TP_SCATTER}: + # The residual stays on each rank's slice while the FFN takes the + # attention's rows (MHC on an input-scattered batch). + if gathered or not owes_attention_tp or residual != residual_to: + raise NotImplementedError(f"{produced=} {residual=} {need=}") + return partial(_mlp_input_on_residual_shard, residual_ops=residual_ops), () + if gathered == {TokenAxis.ATTN_CP}: + # Each CP rank completes its own chunk, then the CP moves gather them. + if cp_moves is None: + raise NotImplementedError(f"{produced=} {need=}") + on_chunk, fused = _select_ffn_input( + produced, + residual=residual, + residual_to=residual_to, + need=StageInput(produced.layout), + force_layernorm_before_gather=force_layernorm_before_gather, + fusions=fusions, + residual_joins_sum=residual_joins_sum, + residual_ops=residual_ops, + ) + return partial(cp_moves.gather, gather=on_chunk), fused + if ( + residual_to != produced.layout + or gathered + not in ( + frozenset(), + {TokenAxis.ATTN_DP}, + {TokenAxis.ATTN_DP, TokenAxis.ATTN_CP}, + ) + or residual.sharded - residual_to.sharded + not in (frozenset(), {TokenAxis.ATTN_TP_SCATTER}) + ): + raise NotImplementedError(f"{produced=} {residual=} {need=}") + # A residual arriving on each rank's slice is gathered back first. + gathers_residual = residual != residual_to + if not gathered: + if not owes_attention_tp: + return partial(_mlp_input_norm, residual_ops=residual_ops), () + if gathers_residual and residual_joins_sum: + # Each rank adds its slice of the residual into its share of the + # sum, so the all-reduce also brings the residual back to every row. + if not residual_ops.adds_plainly: + raise NotImplementedError(f"{produced=} {residual=} {need=}") + return _mlp_input_residual_into_sum, () + fused = tuple(f for f in fusions if f.completes is produced.group) + return ( + partial( + _mlp_input_without_dp, + gathers_residual=gathers_residual, + fusions=tuple(f.run for f in fused), + residual_ops=residual_ops, + ), + fused, + ) + # The partial order adds the residual on attention-TP rank 0 before the DP + # gather's collective completes that sum, which only a plain residual add + # allows. + # Over attention DP and CP, the DP gather puts each CP rank's shard in its + # DP group's slot, so the one DP sum gathers both axes. + places_cp_shards = TokenAxis.ATTN_CP in gathered + if ( + owes_attention_tp + and not force_layernorm_before_gather + and residual_ops.adds_plainly + ): + return ( + partial( + _mlp_input_dp_partial, + gathers_residual=gathers_residual, + places_cp_shards=places_cp_shards, + ), + (), + ) + return ( + partial( + _mlp_input_dp_replicate, + gathers_residual=gathers_residual, + reduces_attention_tp=owes_attention_tp, + places_cp_shards=places_cp_shards, + residual_ops=residual_ops, + ), + (), + ) + + +def _select_ffn_output_move( + produced: StageOutput, + *, + residual: Layout, + to: Layout, + cp_moves: Optional[CpMoves] = None, + residual_ops: ResidualOps = ADD_AND_NORM, +) -> Tuple[bool, Optional[Callable], bool]: + """How the FFN output reaches the rows the layer hands on: whether it goes + back by undoing the attention-DP gather (the FFN exit and postprocess run + that step), or else the postprocess that moves it, None when there is none + to choose here; and whether that move also completes the sum the FFN + leaves.""" + pair = CommunicateSummableTensorPairFn + if produced.layout == residual: + if to == residual: + return False, pair._trivial, False + if to.sharded == residual.sharded - {TokenAxis.ATTN_TP_SCATTER}: + # Each rank's slice back to the attention's rows: write the output + # into the residual, then gather over attention TP. + return False, partial(pair._gather, residual_ops=residual_ops), False + raise NotImplementedError(f"{produced=} {residual=} {to=}") + returned = residual.sharded - produced.layout.sharded + if returned == {TokenAxis.ATTN_TP_SCATTER} and to in (residual, produced.layout): + # The residual stays on each rank's slice (MHC on an input-scattered + # batch): a reduce-scatter onto the slice completes the sum the FFN + # leaves; a complete output is only sliced. + sums = produced.leaves_for_reduce_scatter + return ( + False, + partial( + pair._onto_residual_shard, + sums=sums, + gathers_back=to != residual, + residual_ops=residual_ops, + ), + sums, + ) + if to != residual or not produced.layout.sharded <= residual.sharded: + raise NotImplementedError(f"{produced=} {residual=} {to=}") + if returned == {TokenAxis.ATTN_CP}: + if cp_moves is None: + raise NotImplementedError(f"{produced=} {residual=} {to=}") + if not produced.leaves_for_reduce_scatter: + # A complete output: this rank's block of it, nothing summed. + return False, cp_moves.take_back, False + # The FFN leaves its sum: only a take-back that sums over the same + # ranks completes it. + if cp_moves.reduce_scatter is None or not _same_ranks( + _sum_group(produced.group), cp_moves.reduce_scatter_group() + ): + raise NotImplementedError(f"{produced=} {residual=} {to=}") + return False, cp_moves.reduce_scatter, True + if returned == {TokenAxis.ATTN_DP, TokenAxis.ATTN_CP}: + # This rank's CP shard, from where the DP gather put it. + return False, CommunicateSummableTensorPairFn._take_back_cp_shard, False + if returned != {TokenAxis.ATTN_DP}: + raise NotImplementedError(f"{produced=} {residual=} {to=}") + return True, None, False diff --git a/python/sglang/srt/layers/communicator/layer.py b/python/sglang/srt/layers/communicator/layer.py new file mode 100644 index 0000000000..bb19c50b77 --- /dev/null +++ b/python/sglang/srt/layers/communicator/layer.py @@ -0,0 +1,1348 @@ +# Copyright 2023-2024 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""The communicator a model layer uses.""" + +from dataclasses import dataclass +from enum import Enum, auto +from functools import partial +from typing import Callable, Optional, Tuple, Union + +import msgspec +import torch + +from sglang.srt.distributed import GroupCoordinator +from sglang.srt.environ import envs +from sglang.srt.layers import layernorm_sp +from sglang.srt.layers.attention.dsa.utils import is_dsa_enable_prefill_cp +from sglang.srt.layers.aux_hidden_states import AuxHiddenStateAccumulator +from sglang.srt.layers.communicator.adapters.attention import get_attn_tp_context +from sglang.srt.layers.communicator.boundary import ( + BoundarySteps, + DecoderLayerSides, + FusedMlpInput, + InputRead, + LayerStage, + _another_stage, + _cp_moves, + _select_boundary_steps, + decoder_layer_sides, + input_scattered_layer_sides, + make_boundary, + scattered_residual_layer_sides, + sequence_parallel_layer_sides, +) +from sglang.srt.layers.communicator.layout import ( + CommunicateContext, + Layout, + ScatterMode, + SumGroup, + TokenAxis, + _batch_shards_over_cp, + _batch_size, + _ffn_has_tokens, + _gathers_over_attention_cp, + _generic_prefill_cp_shards_tokens, + _sum_group, + enable_dwdp, + enable_moe_dense_fully_dp, + sparse_moe_gathers_over_moe_cp, + token_axis_sizes, +) +from sglang.srt.layers.communicator.ops import ( + _all_reduce_then_to_local_tokens, + _hand_qkv_hook_its_input, + _hand_scattered_input_to_attention, + _redistribute_output, + _reduce_and_redistribute_output_step, + _to_local_tokens, + move_rows, +) +from sglang.srt.layers.communicator.output import ( + HandoffOutput, + UnreducedOutput, + reduce_output, +) +from sglang.srt.layers.communicator.residual import ResidualOps +from sglang.srt.layers.communicator.residual.add_norm import ( + ADD_AND_NORM, + aiter_all_reduce_fusion_enabled_for, + apply_aiter_all_reduce_fusion, + apply_flashinfer_allreduce_fusion, +) +from sglang.srt.layers.communicator.residual.mhc import MHCState +from sglang.srt.layers.cp.utils import is_mla_cp_enabled +from sglang.srt.layers.dp_attention import ( + is_dp_attention_enabled, + is_enable_moe_cp_allgather, +) +from sglang.srt.layers.moe import ( + can_merge_post_experts_all_reduce, + get_moe_a2a_backend, + is_moe_input_scattered_across_dp_ranks, + post_experts_reduction_group, + post_experts_sum_is_one_all_reduce, + should_use_flashinfer_cutlass_moe_fp4_allgather, +) +from sglang.srt.model_executor.forward_batch_info import ForwardBatch +from sglang.srt.runtime_context import ( + LoRABatchLayout, + get_exec, + get_forward, + get_lora, + get_parallel, + get_spec, +) +from sglang.srt.speculative.spec_info import SpeculativeAlgorithm +from sglang.srt.utils import get_bool_env_var, is_hip + +_use_aiter = get_bool_env_var("SGLANG_USE_AITER") and is_hip() +_use_ag_after_qlora = envs.SGLANG_USE_AG_AFTER_QLORA.get() + + +@dataclass +class _LayerModeComputationContext: + num_layers: int + layer_id: int + is_layer_sparse: bool + is_previous_layer_sparse: Optional[bool] + is_next_layer_sparse: Optional[bool] + + def previous_layer(self): + assert self.is_previous_layer_sparse is not None + return _LayerModeComputationContext( + num_layers=self.num_layers, + layer_id=self.layer_id - 1, + is_layer_sparse=self.is_previous_layer_sparse, + is_previous_layer_sparse=None, + is_next_layer_sparse=self.is_layer_sparse, + ) + + +def sparse_mlp_scatter_mode() -> ScatterMode: + """SCATTERED hands a sparse MLP this rank's own token shard; FULL and + MOE_FULL hand it a buffer gathered over the attn-TP or MoE-CP group.""" + if ( + # Token dispatch/combine will be handled outside of LayerCommunicator for these modes. + not get_moe_a2a_backend().is_none() + or should_use_flashinfer_cutlass_moe_fp4_allgather() + or enable_dwdp() + ): + return ScatterMode.SCATTERED + # DSA CP and MLA CP both don't support MOE_FULL yet; fall back to FULL. + if is_enable_moe_cp_allgather() and not ( + is_dsa_enable_prefill_cp() or is_mla_cp_enabled() + ): + return ScatterMode.MOE_FULL + return ScatterMode.FULL + + +@dataclass +class LayerScatterModes: + layer_input_mode: ScatterMode + attn_mode: ScatterMode + # Can be further split into e.g. mlp_input_mode and mlp_output_mode if needed + mlp_mode: ScatterMode + middle_residual_mode: ScatterMode + layer_output_mode: ScatterMode + is_layer_sparse: bool = False + # The model's first layer: its input is the embedding, not a layer output. + is_first_layer: bool = False + # The model's last layer: its output goes to the final norm, not a next layer. + is_last_layer: bool = False + # Whether the layer before this one has a sparse MLP; None when the modes + # were given directly, not planned from the layer sequence. + is_previous_layer_sparse: Optional[bool] = None + # Whether the layer after this one has a sparse MLP; None likewise. + is_next_layer_sparse: Optional[bool] = None + + @classmethod + def init_new(cls, **kwargs): + context = _LayerModeComputationContext(**kwargs) + return cls( + layer_input_mode=cls._compute_layer_input_mode(context), + attn_mode=ScatterMode.TP_ATTN_FULL, + mlp_mode=cls._compute_mlp_mode(context), + middle_residual_mode=cls._compute_middle_residual_mode(context), + layer_output_mode=cls._compute_layer_output_mode(context), + is_layer_sparse=context.is_layer_sparse, + is_first_layer=context.layer_id == 0, + is_last_layer=context.layer_id == context.num_layers - 1, + is_previous_layer_sparse=context.is_previous_layer_sparse, + is_next_layer_sparse=context.is_next_layer_sparse, + ) + + @classmethod + def _compute_layer_input_mode(cls, context: _LayerModeComputationContext): + if context.layer_id == 0: + return ScatterMode.model_input_output() + return cls._compute_layer_output_mode(context.previous_layer()) + + @classmethod + def _compute_mlp_mode(cls, context: _LayerModeComputationContext): + if context.is_layer_sparse: + return sparse_mlp_scatter_mode() + else: + if enable_moe_dense_fully_dp(): + return ScatterMode.SCATTERED + # A TP-sharded dense MLP reduces over the whole TP group, which spans + # every CP rank; a CP-sharded prefill must gather tokens across CP + # first or the all-reduce sums different tokens' partial outputs. + # MLA/DSA CP models gather over the attention-CP group instead. + if _generic_prefill_cp_shards_tokens() and not ( + is_dsa_enable_prefill_cp() or is_mla_cp_enabled() + ): + return ScatterMode.MOE_FULL + return ScatterMode.FULL + + @classmethod + def _should_gather_for_tbo(cls, context: _LayerModeComputationContext): + return ( + not context.is_layer_sparse + and context.is_next_layer_sparse + and enable_moe_dense_fully_dp() + and get_exec().overlap.enable_two_batch_overlap + ) + + @classmethod + def _compute_middle_residual_mode(cls, context: _LayerModeComputationContext): + mlp_mode = cls._compute_mlp_mode(context) + if mlp_mode == ScatterMode.SCATTERED: + return ScatterMode.SCATTERED + if mlp_mode in (ScatterMode.FULL, ScatterMode.MOE_FULL): + return ScatterMode.TP_ATTN_FULL + raise NotImplementedError + + @classmethod + def _compute_layer_output_mode(cls, context: _LayerModeComputationContext): + mlp_mode = cls._compute_mlp_mode(context) + if context.layer_id == context.num_layers - 1: + return ScatterMode.model_input_output() + if mlp_mode == ScatterMode.SCATTERED: + if cls._should_gather_for_tbo(context): + return ScatterMode.TP_ATTN_FULL + return ScatterMode.SCATTERED + if mlp_mode in (ScatterMode.FULL, ScatterMode.MOE_FULL): + return ScatterMode.TP_ATTN_FULL + raise NotImplementedError + + +def _unfused_completion_matches_the_ffn(forward_batch: ForwardBatch) -> bool: + """Whether the next layer's all-reduce is the one the FFN would have run + itself, as the MoE declares it (post_experts_sum_is_one_all_reduce), on a + batch the FFN runs.""" + return _ffn_has_tokens(forward_batch) and post_experts_sum_is_one_all_reduce() + + +class LayerCommunicator: + # Communicators built without __init__ (e.g. test doubles) publish no LoRA + # layout and try no fused kernel at the FFN exit. + _publish_lora_layout: bool = False + _ffn_exit_fusions: Tuple[Callable, ...] = () + # A plain residual unless the layer is built with its own. + _residual_ops: ResidualOps = ADD_AND_NORM + + def __init__( + self, + layer_scatter_modes: LayerScatterModes, + input_layernorm: torch.nn.Module, + post_attention_layernorm: torch.nn.Module, + # Reduce scatter requires skipping all-reduce in model code after MoE/MLP, so only enable for models which have that implemented. Remove flag once done for all models that use LayerCommunicator. + allow_reduce_scatter: bool = False, + qkv_latent_func: Optional[Callable] = None, + force_layernorm_before_dp_gather: bool = False, + enable_fused_ar_quant: bool = False, + fused_ar_quant_keep_bf16: bool = False, + # False for a layer whose FFN always completes its own reduction. + allow_deferred_ffn_reduction: bool = True, + # How the layer writes its residual and reads its stages' inputs. + residual_ops: ResidualOps = ADD_AND_NORM, + # A layer that is one stage of a sequence of stages, instead of an + # attention followed by an FFN. + stage: Optional["LayerStage"] = None, + ): + self.layer_scatter_modes = layer_scatter_modes + self.input_layernorm = input_layernorm + self.post_attention_layernorm = post_attention_layernorm + self.allow_reduce_scatter = allow_reduce_scatter + self.is_last_layer = layer_scatter_modes.is_last_layer + self.qkv_latent_func = qkv_latent_func + self.force_layernorm_before_dp_gather = force_layernorm_before_dp_gather + self.enable_fused_ar_quant = enable_fused_ar_quant + self.fused_ar_quant_keep_bf16 = fused_ar_quant_keep_bf16 + self.allow_deferred_ffn_reduction = allow_deferred_ffn_reduction + self._residual_ops = residual_ops + + self._context = CommunicateContext.init_new() + self._context.force_layernorm_before_dp_gather = ( + force_layernorm_before_dp_gather + ) + # The fused kernels every batch's attention input tries first. + self._attn_input_fusions = self._select_attn_input_fusions() + # The fused kernels of the next layer's input the FFN exit tries first. + self._ffn_exit_fusions = self._select_ffn_exit_fusions() + self._speculative_algo = SpeculativeAlgorithm.from_string( + get_spec().speculative_algorithm + ) + # LoRA kernels need the per-layer token layout only under DP attention. + self._publish_lora_layout = get_parallel().enable_dp_attention and bool( + get_lora().enable_lora + ) + if stage is not None: + self._init_stage(stage) + return + # Its two boundaries, for a layer that is one stage. + self.stage_edges = None + # The steps the layer's ordinary batches run. + sides = self._declared_sides() + self._declared = sides + self._steps = self._steps_from_declarations( + sides, + fusions=self._select_mlp_input_fusions(), + force_layernorm_before_gather=force_layernorm_before_dp_gather, + ) + # The steps a batch that shards its tokens over attention CP runs; None + # when no batch does. + self._cp_steps = ( + self._steps_from_declarations( + self._declared_sides(cp_active=True), + fusions=self._select_mlp_input_fusions(), + force_layernorm_before_gather=force_layernorm_before_dp_gather, + cp_moves=_cp_moves(), + ) + if _generic_prefill_cp_shards_tokens() + else None + ) + # The steps a batch with input-scattered attention runs; None where it + # cannot run. + self._input_scattered_steps = ( + self._steps_for_input_scattered(sides) + if self._input_can_be_scattered() + else None + ) + + # Under LayerNorm SP, the steps the layer runs while the region is + # active; None without SP. The two are exclusive: SP runs a model without + # q_lora, which input-scattered attention needs. + self._sp_steps = ( + _select_boundary_steps( + sequence_parallel_layer_sides(axis_sizes=token_axis_sizes()), + residual_ops=residual_ops, + attention_fusions=self._attn_input_fusions, + enters_stack=self.layer_scatter_modes.is_first_layer, + ) + if layernorm_sp.layernorm_sp_enabled() + else None + ) + + def _init_stage(self, stage: "LayerStage") -> None: + """A layer that is one stage: every batch runs the two boundaries its + declarations give.""" + if get_parallel().attn_cp_size > 1 or layernorm_sp.layernorm_sp_enabled(): + raise NotImplementedError( + "a layer that is one stage with attention CP or LayerNorm SP" + ) + self._declared = None + self._cp_steps = self._input_scattered_steps = self._sp_steps = None + self.stage_edges = stage.edges + into_edge, out_edge = stage.edges + reads_ffn = stage.reads is InputRead.FFN + into = make_boundary( + into_edge, + reads=stage.reads, + fusions=( + self._select_mlp_input_fusions() + if reads_ffn + else self._attn_input_fusions + ), + force_layernorm_before_gather=self.force_layernorm_before_dp_gather, + residual_ops=self._residual_ops, + enters_stack=stage.enters_stack, + ) + out = make_boundary(out_edge, reads=None, residual_ops=self._residual_ops) + self._steps = BoundarySteps( + attention_prepare=_another_stage if reads_ffn else into.prepare, + attention_input=_another_stage if reads_ffn else into.input_move, + ffn_input=into.prepare if reads_ffn else _another_stage, + ffn_input_rows=into.input_rows, + ffn_output=out_edge.produced, + ffn_output_move=out.output_move, + ffn_output_move_completes_sum=out.output_move_completes_sum, + ffn_sum_is_movable=out_edge.produced.group is not None, + fused=into.fused, + ) + + @property + def input_rows(self) -> Layout: + """The rows the layer's input and residual arrive on in a batch that + runs its ordinary steps.""" + if self._declared is not None: + return self._declared.input_rows + return self.stage_edges[0].residual + + @property + def input_on_attention_tp_slices(self) -> bool: + """Whether the layer's input arrives on each attention-TP rank's slice + of its rows (after a layer on local rows), in a batch that runs its + ordinary steps.""" + return TokenAxis.ATTN_TP_SCATTER in self.input_rows.sharded + + def _input_can_be_scattered(self) -> bool: + """Whether a batch may run this layer with input-scattered attention: + configured, on TP without attention DP, a prefill CP, an a2a backend or + a dense MLP on every rank. The rest of what ``AttnTpContext.init_context`` requires is only + known once the model is built.""" + parallel = get_parallel() + return ( + parallel.enable_attn_tp_input_scattered + and parallel.tp_size > 1 + and parallel.attn_dp_size == 1 + and not _generic_prefill_cp_shards_tokens() + and get_moe_a2a_backend().is_none() + and not enable_moe_dense_fully_dp() + ) + + def _declared_sides(self, *, cp_active: bool = False) -> DecoderLayerSides: + """The declarations this layer's boundaries are chosen from: an + attention and an FFN, in a layer whose previous layer is known. + ``cp_active`` gives a batch that shards its tokens over CP; the others + hold every token on each CP rank. An active LayerNorm SP region and + input-scattered attention have their own. Raises NotImplementedError + for the combinations the steps do not cover.""" + modes = self.layer_scatter_modes + parallel = get_parallel() + if not (modes.is_first_layer or modes.is_previous_layer_sparse is not None): + raise NotImplementedError( + "a layer built without the facts of the layer before it" + ) + # A MoE dispatched per DP shard computes on this rank's local rows and + # hands its layer's output on there; so does a dense MLP on every rank. + moe_on_local_rows = is_moe_input_scattered_across_dp_ranks() + dense_on_local_rows = enable_moe_dense_fully_dp() + + # Some batch shards its tokens over attention CP. + cp_shards = _generic_prefill_cp_shards_tokens() + if parallel.attn_cp_size > 1 and modes.is_layer_sparse: + self._refuse_uncovered_cp_moe(moe_on_local_rows, cp_shards) + # A MoE whose data-parallel groups are the CP ranks computes each CP + # shard of a GQA prefill CP on its own ranks. + ffn_on_cp_shards = ( + modes.is_layer_sparse + and parallel.attn_cp_size > 1 + and parallel.moe_dp_size == parallel.attn_cp_size + and not _gathers_over_attention_cp() + ) + + def on_local_rows(sparse: bool) -> bool: + return moe_on_local_rows if sparse else dense_on_local_rows + + def gathers_for_tbo(sparse: bool, next_sparse: Optional[bool]) -> bool: + # Under two-batch overlap a dense layer on local rows hands the + # sparse layer after it, where the split happens, the attention's + # rows. + return ( + dense_on_local_rows + and get_exec().overlap.enable_two_batch_overlap + and not sparse + and bool(next_sparse) + ) + + if cp_shards and _cp_moves().reduce_scatter is not None: + # A CP extend's FFN may leave its sum to the reduce-scatter that + # takes each rank's shard back (DSA and MLA CP). + may_leave = not cp_active + may_leave_to_reduce_scatter = True + elif cp_shards or parallel.attn_dp_size > 1: + # Otherwise under CP an FFN over every CP shard completes its own sum; + # one on its own CP shard hands on its rows as without CP. + may_leave = may_leave_to_reduce_scatter = ( + parallel.attn_cp_size == 1 or ffn_on_cp_shards + ) + else: + # No batch shards its tokens and there is no attention DP: as + # without CP. + may_leave = may_leave_to_reduce_scatter = True + return decoder_layer_sides( + axis_sizes=token_axis_sizes(cp_active=cp_active), + ffn_on_local_rows=on_local_rows(modes.is_layer_sparse), + ffn_shards_over_cp=ffn_on_cp_shards, + previous_on_local_rows=( + not modes.is_first_layer + and on_local_rows(modes.is_previous_layer_sparse) + and not gathers_for_tbo( + modes.is_previous_layer_sparse, modes.is_layer_sparse + ) + ), + is_last_layer=modes.is_last_layer, + hands_on_attention_rows=gathers_for_tbo( + modes.is_layer_sparse, modes.is_next_layer_sparse + ), + attention_gathers_local_rows=_use_ag_after_qlora, + ffn_group=SumGroup.MOE_OUTPUT if modes.is_layer_sparse else SumGroup.TP, + leaves_for_next_layer=self.allow_deferred_ffn_reduction and may_leave, + leaves_for_reduce_scatter=self.allow_reduce_scatter + and may_leave_to_reduce_scatter, + # A MoE block leaves its sum to reduce_scatterv whenever that combine + # applies (should_skip_post_experts_all_reduce). + leaves_for_reduce_scatterv=( + self.allow_reduce_scatter or modes.is_layer_sparse + ) + and may_leave, + ) + + @staticmethod + def _refuse_uncovered_cp_moe(moe_on_local_rows: bool, cp_shards: bool) -> None: + """A MoE layer under attention CP whose tokens the steps cannot bring + to it: under a prefill CP, one dispatched per DP shard under attention DP + and GQA CP, and one on the TP group whose data-parallel groups are the CP + ranks under DSA or MLA CP; and under attention DP, one on the TP group + whose data-parallel groups are the CP ranks.""" + parallel = get_parallel() + gqa = not _gathers_over_attention_cp() + if moe_on_local_rows: + if cp_shards and gqa and parallel.attn_dp_size > 1: + raise NotImplementedError( + "a MoE dispatched per DP shard under attention DP and GQA " + "prefill CP" + ) + elif parallel.moe_dp_size == parallel.attn_cp_size: + if cp_shards and not gqa: + raise NotImplementedError( + "a MoE on the TP group with moe_dp_size == attn_cp_size under " + "DSA or MLA prefill CP" + ) + if parallel.attn_dp_size > 1: + raise NotImplementedError( + "a MoE on the TP group with moe_dp_size == attn_cp_size under " + "attention DP and attention CP" + ) + + def _steps_for_input_scattered(self, sides: DecoderLayerSides) -> "BoundarySteps": + """The steps a batch with input-scattered attention runs at this layer, + for the layer's ordinary declarations ``sides``. A plain residual comes + back to the full rows inside the attention output's sum; one whose + write-back is not a plain add stays on each rank's slice.""" + if self._residual_ops.adds_plainly: + scattered = input_scattered_layer_sides( + axis_sizes=token_axis_sizes(), + ffn_group=sides.ffn_output.group, + hands_on_partial=self.allow_reduce_scatter and not self.is_last_layer, + ) + handoff = _hand_qkv_hook_its_input + else: + scattered = scattered_residual_layer_sides( + axis_sizes=token_axis_sizes(), + ffn_group=sides.ffn_output.group, + is_first_layer=self.layer_scatter_modes.is_first_layer, + is_last_layer=self.is_last_layer, + leaves_for_reduce_scatter=self.allow_reduce_scatter, + ) + # DSA and hook-less attention take the slice gathered. + handoff = _hand_scattered_input_to_attention + return _select_boundary_steps( + scattered, + residual_ops=self._residual_ops, + attention_handoff=handoff, + attention_fusions=self._attn_input_fusions, + enters_stack=self.layer_scatter_modes.is_first_layer, + ) + + def _steps_from_declarations( + self, sides: DecoderLayerSides, **kwargs + ) -> "BoundarySteps": + """The steps this layer runs for a set of declarations.""" + return _select_boundary_steps( + sides, + residual_ops=self._residual_ops, + attention_fusions=self._attn_input_fusions, + enters_stack=self.layer_scatter_modes.is_first_layer, + **kwargs, + ) + + def prepare_attn_and_capture_last_layer_outputs( + self, + hidden_states: torch.Tensor, + residual: torch.Tensor, + forward_batch: ForwardBatch, + captured_last_layer_outputs: Optional[AuxHiddenStateAccumulator] = None, + post_residual_addition: Optional[torch.Tensor] = None, + quant_format: str = "", + ): + hidden_states, residual = self.prepare_attn( + hidden_states, + residual, + forward_batch, + quant_format=quant_format, + post_residual_addition=post_residual_addition, + ) + if captured_last_layer_outputs is not None: + gathered_last_layer_output = self._batch_steps( + forward_batch + ).attention_input( + hidden_states=residual, + forward_batch=forward_batch, + context=self._context, + ) + if ( + gathered_last_layer_output is residual + # An accumulator that copies on append already holds a snapshot. + and not getattr(captured_last_layer_outputs, "copies_on_append", False) + and not self._post_attn_residual_is_read_only(residual, forward_batch) + ): + gathered_last_layer_output = residual.clone() + captured_last_layer_outputs.append(gathered_last_layer_output) + return hidden_states, residual + + def _post_attn_residual_is_read_only( + self, residual: torch.Tensor, forward_batch: ForwardBatch + ) -> bool: + """True if ``prepare_mlp``'s post-attention RMSNorm leaves ``residual`` + untouched, so Eagle3 aux capture can keep its reference and skip the clone. + + Of the base fused entry's kernels only flashinfer's writes a fresh + ``residual_out`` (see ``flashinfer_allreduce_residual_rmsnorm``); the aiter + kernel and every plain norm fold into ``residual`` in place. It runs when + ``prepare_mlp`` selected a fused entry that may return a new residual and + the batch is not input-scattered. + """ + return ( + any( + f.may_return_new_residual + for f in self._batch_steps(forward_batch).fused + ) + and not get_attn_tp_context().input_scattered + and apply_flashinfer_allreduce_fusion(residual.shape[0]) + ) + + def publish_attn_lora_layout(self) -> None: + """Attention consumes the DP-local token batch.""" + if self._publish_lora_layout: + get_forward().set("lora_batch_layout", LoRABatchLayout.DP_LOCAL) + + def publish_mlp_lora_layout(self, steps: "BoundarySteps") -> None: + """The MLP consumes the TP-global batch only when the batch's FFN input + is gathered over attention DP.""" + if self._publish_lora_layout: + get_forward().set( + "lora_batch_layout", + ( + LoRABatchLayout.DP_LOCAL + if TokenAxis.ATTN_DP in steps.ffn_input_rows.sharded + else LoRABatchLayout.TP_GLOBAL + ), + ) + + def prepare_attn( + self, + hidden_states: Union[torch.Tensor, UnreducedOutput], + residual: torch.Tensor, + forward_batch: ForwardBatch, + quant_format: str = "", + post_residual_addition: Optional[torch.Tensor] = None, + ): + self.publish_attn_lora_layout() + # The SP region opens at the first layer, re-evaluated per forward so a + # crash mid-loop cannot leak into the next one. It sets what the batch's + # steps are chosen from, and the first layer's input owes nothing. + if self._sp_steps is not None and self.layer_scatter_modes.is_first_layer: + get_forward().set( + "sp_active", layernorm_sp.runs_sp(forward_batch.forward_mode) + ) + if get_forward().sp_active: + hidden_states = layernorm_sp.sp_entry_scatter(hidden_states) + hidden_states, residual = self._batch_steps(forward_batch).attention_prepare( + hidden_states, + residual, + forward_batch, + self.input_layernorm, + self._context, + quant_format=quant_format, + post_residual_addition=post_residual_addition, + ) + return self._finish_prepare_attn( + hidden_states=hidden_states, + residual=residual, + forward_batch=forward_batch, + ) + + def _in_sp_region(self) -> bool: + return self._sp_steps is not None and get_forward().sp_active + + def _batch_steps(self, forward_batch: ForwardBatch) -> "BoundarySteps": + """The steps this batch runs: an active LayerNorm SP region's, + input-scattered attention's, a CP extend's, or the layer's ordinary + ones.""" + if self._in_sp_region(): + return self._sp_steps + if ( + self._input_scattered_steps is not None + and get_attn_tp_context().input_scattered + ): + return self._input_scattered_steps + if self._cp_steps is not None and _batch_shards_over_cp(forward_batch): + return self._cp_steps + return self._steps + + def _finish_prepare_attn(self, hidden_states, residual, forward_batch): + """Tail every prepare_attn path must run, or ``attn_inputs`` is unset.""" + steps = self._batch_steps(forward_batch) + hidden_states = steps.attention_input( + hidden_states=hidden_states, + forward_batch=forward_batch, + context=self._context, + ) + hidden_states = steps.attention_handoff( + hidden_states, forward_batch, self.qkv_latent_func + ) + return hidden_states, residual + + def _select_attn_input_fusions(self) -> Tuple[Callable, ...]: + """The fused kernels that complete what the previous layer left together + with the residual update and the input norm, in the order they are tried. + Each takes (owed, residual, forward_batch, post_residual_addition) and + returns None when it does not take the batch. They add the residual + plainly.""" + if not ( + self._residual_ops.adds_plainly + and hasattr(self.input_layernorm, "forward_with_allreduce_fusion") + ): + return () + self._attn_input_fuses_quant = ( + self.enable_fused_ar_quant + and _use_aiter + and hasattr( + self.input_layernorm, "forward_with_allreduce_fusion_quant_per_group" + ) + ) + return (self._reduce_output_and_update_and_read_residual,) + + def _reduce_output_and_update_and_read_residual( + self, + owed: UnreducedOutput, + residual: torch.Tensor, + forward_batch: ForwardBatch, + post_residual_addition: Optional[torch.Tensor], + ) -> Optional[Tuple[torch.Tensor, torch.Tensor]]: + """Complete the sum the previous layer left, add it to the residual and + apply the input norm in one aiter or flashinfer kernel; None when the + kernel does not take this batch. The result is not quantized for + ``quant_format``.""" + if ( + not isinstance(owed, UnreducedOutput) + # The kernel does not add it. + or post_residual_addition is not None + # The kernel reduces over the MoE output's group. + or owed.group is not post_experts_reduction_group() + ): + return None + hidden_states = owed.partial + if not ( + apply_aiter_all_reduce_fusion(hidden_states, forward_batch) + or apply_flashinfer_allreduce_fusion(hidden_states.shape[0]) + ): + return None + if self._attn_input_fuses_quant: + # Falls back to AR+RMSNorm + separate quant internally when the + # fully-fused kernel cannot service the shape. + quant_result = ( + self.input_layernorm.forward_with_allreduce_fusion_quant_per_group( + hidden_states, + residual, + use_attn_tp_group=False, + keep_bf16=self.fused_ar_quant_keep_bf16, + ) + ) + if quant_result is not None: + return quant_result + return self.input_layernorm.forward_with_allreduce_fusion( + hidden_states, residual, use_attn_tp_group=False + ) + + def _select_mlp_input_fusions(self) -> Tuple["FusedMlpInput", ...]: + """The fused kernels that can take the attention -> FFN steps, in the + order they are tried. They add the residual plainly.""" + if not ( + self._residual_ops.adds_plainly + and hasattr(self.post_attention_layernorm, "forward_with_allreduce_fusion") + ): + return () + return ( + FusedMlpInput( + completes=SumGroup.ATTN_TP, + run=self._mlp_input_reduce_output_and_update_and_read_residual, + may_return_new_residual=True, + ), + ) + + def _mlp_input_reduce_output_and_update_and_read_residual( + self, + hidden_states: torch.Tensor, + residual: torch.Tensor, + forward_batch: ForwardBatch, + ) -> Optional[Tuple[torch.Tensor, torch.Tensor]]: + """The attention-TP all-reduce, residual add and norm in one aiter or + flashinfer kernel; None when neither takes the batch.""" + if not ( + apply_aiter_all_reduce_fusion(hidden_states, forward_batch) + or apply_flashinfer_allreduce_fusion(hidden_states.shape[0]) + ): + return None + return self.post_attention_layernorm.forward_with_allreduce_fusion( + hidden_states, residual, use_attn_tp_group=True + ) + + def prepare_mlp( + self, + hidden_states: Union[torch.Tensor, UnreducedOutput], + residual: torch.Tensor, + forward_batch: ForwardBatch, + cache=None, + ): + steps = self._batch_steps(forward_batch) + self.publish_mlp_lora_layout(steps) + if cache is not None: + self._context.cache = cache + + return steps.ffn_input( + hidden_states, + residual, + forward_batch, + self.post_attention_layernorm, + self._context, + ) + + def maybe_prefetch_next_full_attention_kv( + self, + forward_batch: ForwardBatch, + next_full_attention_layer_id: Optional[int], + ) -> None: + return + + def postprocess_layer( + self, + hidden_states: torch.Tensor, + residual: torch.Tensor, + forward_batch: ForwardBatch, + ): + return self._complete_ffn_output_now( + hidden_states, + residual, + forward_batch=forward_batch, + dp_step=self._postprocess_dp_step(forward_batch), + ) + + def _local_token_move_can_go_to_next_layer( + self, forward_batch: ForwardBatch + ) -> bool: + """Whether the next layer's input can run this layer's move of its FFN + output back to this rank's tokens: the base postprocess scatter, when + the next layer's input also writes the output into the residual.""" + return ( + self._batch_steps(forward_batch).returns_over_dp + and not self._residual_ops.updates_residual_after_ffn + ) + + def _postprocess_dp_step( + self, forward_batch: ForwardBatch + ) -> Optional[Callable[[torch.Tensor, torch.Tensor, ForwardBatch], None]]: + """The reduce-scatter that brings this layer's FFN output back to this + rank's tokens under attention DP; None when the base postprocess would + only scatter, or does not move tokens.""" + steps = self._batch_steps(forward_batch) + if not steps.returns_over_dp: + return None + return _reduce_and_redistribute_output_step( + forward_batch, + leaves_for_reduce_scatter=steps.ffn_output.leaves_for_reduce_scatter, + leaves_for_reduce_scatterv=steps.ffn_output.leaves_for_reduce_scatterv, + ) + + def _ffn_leaves_sum_to_reduce_scatter( + self, forward_batch: ForwardBatch, dp_step: Optional[Callable] + ) -> bool: + """Whether the FFN leaves its sum out because a reduce-scatter completes + it: the attention-DP one ``dp_step`` names, or the CP / input-scattered + one.""" + steps = self._batch_steps(forward_batch) + if not steps.ffn_output.leaves_for_reduce_scatter: + return False + if dp_step is not None or steps.ffn_output_move_completes_sum: + return True + return get_attn_tp_context().input_scattered and not self.is_last_layer + + def ffn_reduction_group(self, forward_batch: ForwardBatch) -> GroupCoordinator: + """The group this layer's FFN output owes its sum over: the MoE output's + group on a sparse layer, the TP group a dense MLP reduces over.""" + return _sum_group(self._batch_steps(forward_batch).ffn_output.group) + + def _select_ffn_completion(self, forward_batch: ForwardBatch) -> "FfnCompletion": + """Decide once, before the FFN runs, what it skips and what completes its + output: the next layer's input, or this layer's postprocess step.""" + dp_step = self._postprocess_dp_step(forward_batch) + mlp_reduce_scatter = self._ffn_leaves_sum_to_reduce_scatter( + forward_batch, dp_step + ) + complete_now = partial( + self._complete_ffn_output_now, + forward_batch=forward_batch, + dp_step=dp_step, + ) + steps = self._batch_steps(forward_batch) + if not steps.ffn_output.leaves_for_next_layer: + return FfnCompletion( + defer_moe_finalize=False, + fuse_mlp_allreduce=False, + mlp_reduce_scatter=mlp_reduce_scatter, + complete=complete_now, + ) + fusion = next( + filter(None, (fused(forward_batch) for fused in self._ffn_exit_fusions)), + None, + ) + defer_moe_finalize = fusion is FfnExitFusion.DEFER_MOE_FINALIZE + # A fused kernel that takes the sum, a handoff included, skips the + # post-experts all-reduce. + fuse_mlp_allreduce = fusion is not None or self._ffn_sum_moves_to_next_layer( + forward_batch, mlp_reduce_scatter=mlp_reduce_scatter, dp_step=dp_step + ) + if fuse_mlp_allreduce: + group = self.ffn_reduction_group(forward_batch) + if steps.returns_over_dp: + # Under attention DP the next layer also brings the sum back to + # this rank's tokens. + wrap = partial( + UnreducedOutput, + reduce_and_redistribute=partial( + _all_reduce_then_to_local_tokens, group, forward_batch + ), + ) + else: + wrap = partial(UnreducedOutput, group=group) + complete = partial(_leave_to_next_layer, wrap) + elif ( + dp_step is not None + and not self.is_last_layer + and self._local_token_move_can_go_to_next_layer(forward_batch) + ): + complete = partial( + _leave_to_next_layer, + partial( + UnreducedOutput, + reduce_and_redistribute=partial( + _to_local_tokens, dp_step, forward_batch + ), + ), + ) + else: + complete = complete_now + return FfnCompletion( + defer_moe_finalize=defer_moe_finalize, + fuse_mlp_allreduce=fuse_mlp_allreduce, + mlp_reduce_scatter=mlp_reduce_scatter, + complete=complete, + ) + + def _complete_ffn_output_now( + self, + hidden_states: torch.Tensor, + residual: torch.Tensor, + *, + forward_batch: ForwardBatch, + dp_step: Optional[Callable], + ) -> Tuple[torch.Tensor, torch.Tensor]: + """This layer's postprocess, run with the attention-DP step already + chosen: the move back to where the next layer reads the FFN output, then + the write-back into the residual for a layer that does it itself.""" + steps = self._batch_steps(forward_batch) + if steps.returns_over_dp: + hidden_states = _to_local_tokens( + dp_step or _redistribute_output, forward_batch, hidden_states + ) + else: + hidden_states, residual = steps.ffn_output_move( + hidden_states=hidden_states, + residual=residual, + forward_batch=forward_batch, + context=self._context, + allow_reduce_scatter=self.allow_reduce_scatter, + is_layer_sparse=self.layer_scatter_modes.is_layer_sparse, + ) + if residual is not None and self._residual_ops.updates_residual_after_ffn: + hidden_states = self._residual_ops.update_residual(hidden_states, residual) + residual = None + return hidden_states, residual + + def mixer_exit(self, forward_batch: ForwardBatch) -> "MixerExit": + """Decide once whether this stage's mixer (an attention-like stage) + skips its output all-reduce. Use the result as a context manager around + the mixer, then call ``finish``.""" + return MixerExit(self, forward_batch) + + def ffn_exit(self, forward_batch: ForwardBatch) -> "FfnExit": + """Decide once how this layer's FFN output reduction completes. Use the + result as a context manager around the FFN call, then call ``finish``.""" + return FfnExit(self, forward_batch) + + def _branch_rows( + self, forward_batch: ForwardBatch + ) -> Tuple[Layout, Layout, Layout]: + """The rows of this layer's FFN input, of its residual while the FFN + runs, and of what the layer hands on, for a batch that runs its ordinary + steps.""" + if self._batch_steps(forward_batch) is not self._steps: + raise NotImplementedError( + "a branch on a batch with attention CP, LayerNorm SP or " + "input-scattered attention" + ) + if self._declared is None: + raise NotImplementedError("a branch on a layer that is one stage") + sides = self._declared + return sides.ffn.layout, sides.ffn_residual_rows, sides.output_rows + + def branch_input( + self, + source: "LayerCommunicator", + hidden_states: torch.Tensor, + residual: torch.Tensor, + forward_batch: ForwardBatch, + ) -> Tuple[torch.Tensor, torch.Tensor]: + """The FFN input and residual that ``source``'s boundary read for its + own FFN, for this layer's FFN, which branches from the same input: moved + to the rows this FFN needs and its residual's rows.""" + rows, residual_rows, _ = source._branch_rows(forward_batch) + to, residual_to, _ = self._branch_rows(forward_batch) + return ( + move_rows(hidden_states, rows, to, forward_batch), + move_rows(residual, residual_rows, residual_to, forward_batch), + ) + + def branch_output( + self, hidden_states: torch.Tensor, forward_batch: ForwardBatch + ) -> torch.Tensor: + """This layer's complete FFN output as a branch's contribution, which + adds to the layer's output without writing the residual: moved to the + rows the layer hands on.""" + rows, _, to = self._branch_rows(forward_batch) + return move_rows(hidden_states, rows, to, forward_batch) + + def merge_branch( + self, + contribution: torch.Tensor, + hidden_states: torch.Tensor, + residual: torch.Tensor, + source: "LayerCommunicator", + forward_batch: ForwardBatch, + ) -> Tuple[torch.Tensor, torch.Tensor]: + """A contribution from ``branch_output`` summed with what ``source``'s + layer hands on, ``hidden_states`` and ``residual``, moved to the rows + this layer hands on.""" + _, _, rows = source._branch_rows(forward_batch) + _, _, to = self._branch_rows(forward_batch) + hidden_states = move_rows(hidden_states, rows, to, forward_batch) + residual = move_rows(residual, rows, to, forward_batch) + return contribution + hidden_states, residual + + def finish_layer_stack( + self, + hidden_states: Union[torch.Tensor, UnreducedOutput, HandoffOutput], + residual: Optional[torch.Tensor], + forward_batch: ForwardBatch, + *, + final_norm_takes_handoff: bool = False, + ) -> Tuple[Union[torch.Tensor, HandoffOutput], Optional[torch.Tensor]]: + """Complete what this layer left for a next layer. Call it on the last + layer of this rank before its output reaches the final norm, the next + pipeline rank, or any other consumer outside the layers. A final norm + that does a producer's handoff together with its own work + (``final_norm_takes_handoff``) receives it as it is.""" + if final_norm_takes_handoff and isinstance(hidden_states, HandoffOutput): + return hidden_states, residual + return reduce_output(hidden_states), residual + + def should_use_reduce_scatter(self, forward_batch: ForwardBatch): + """Whether the FFN leaves its sum to a reduce-scatter, for layers that run + their FFN outside ffn_exit.""" + return self._ffn_leaves_sum_to_reduce_scatter( + forward_batch, self._postprocess_dp_step(forward_batch) + ) + + def _select_ffn_exit_fusions( + self, + ) -> Tuple[Callable[[ForwardBatch], Optional["FfnExitFusion"]], ...]: + """The fused kernels of the next layer's input that may take this layer's + FFN sum, in the order they are tried. Each returns what the exit does + when its kernel takes this batch, or None.""" + return (self._next_input_norm_takes_ffn_sum,) + + def _next_input_norm_takes_ffn_sum( + self, forward_batch: ForwardBatch + ) -> Optional["FfnExitFusion"]: + """The aiter / flashinfer AR + add + norm of the next layer's input.""" + if self.should_fuse_mlp_allreduce_with_next_layer(forward_batch): + return FfnExitFusion.NEXT_INPUT + return None + + def _ffn_sum_can_move_to_next_layer(self, forward_batch: ForwardBatch) -> bool: + # Under the MoE-CP all-gather the fusion path would skip postprocess_layer + # and its MoE-CP scatter, leaving hidden_states longer than the residual. + if ( + is_enable_moe_cp_allgather() + or not self._batch_steps(forward_batch).ffn_sum_is_movable + ): + return False + + # The fused residual+LN reduces over a single group. Hybrid EP+TP spans + # two disjoint groups; post_experts_all_reduce() merges them into one + # _TP reduction when moe_dp_size == 1, which the fused kernel can absorb. + # When merging is blocked, no single group covers both, so fusion stays off. + parallel = get_parallel() + if ( + parallel.moe_ep_size > 1 + and parallel.moe_tp_size > 1 + and not can_merge_post_experts_all_reduce() + ): + return False + + if ( + is_dp_attention_enabled() + and self._speculative_algo is not None + and self._speculative_algo.is_eagle() + ): + return False + + return not get_attn_tp_context().input_scattered + + # NOTE: This function will cause torch recompilation + def should_fuse_mlp_allreduce_with_next_layer( + self, forward_batch: ForwardBatch + ) -> bool: + if not self._ffn_sum_can_move_to_next_layer(forward_batch): + return False + batch_size = _batch_size(forward_batch) + return ( + ( + apply_flashinfer_allreduce_fusion(batch_size) + or ( + _use_aiter + and batch_size > 0 + and get_parallel().tp_size != 6 + and not is_dp_attention_enabled() + and get_moe_a2a_backend().is_none() + and aiter_all_reduce_fusion_enabled_for(forward_batch.forward_mode) + ) + ) + and (not self.is_last_layer) + and (self._context.tp_size > 1) + ) + + def _ffn_sum_moves_to_next_layer( + self, + forward_batch: ForwardBatch, + *, + mlp_reduce_scatter: bool, + dp_step: Optional[Callable], + ) -> bool: + """Whether the FFN leaves its output's all-reduce to the next layer's + input when no fused kernel takes it: when the next layer would run the + same all-reduce the FFN itself would have.""" + return ( + self._context.tp_size > 1 + and not self.is_last_layer + and self._ffn_sum_can_move_to_next_layer(forward_batch) + and _unfused_completion_matches_the_ffn(forward_batch) + and not mlp_reduce_scatter + # Under attention DP the next layer must also run postprocess's + # scatter back to this rank's tokens, and nothing more. + and ( + not is_dp_attention_enabled() + or ( + self._local_token_move_can_go_to_next_layer(forward_batch) + and dp_step is None + ) + ) + ) + + +class FfnExitFusion(Enum): + """What an FFN exit does when a fused kernel of the next layer's input takes + its sum.""" + + # The MoE hands its unfinalized output on; the next input finalizes it. + DEFER_MOE_FINALIZE = auto() + # The FFN leaves its all-reduce to the next input norm. + NEXT_INPUT = auto() + + +class FfnCompletion(msgspec.Struct, frozen=True): + """One FFN's reduction decision. The flags are published while the FFN runs; + ``complete(hidden_states, residual)`` then either wraps the output for the + next layer's input to complete, or runs this layer's postprocess step.""" + + defer_moe_finalize: bool + fuse_mlp_allreduce: bool + mlp_reduce_scatter: bool + complete: Callable[[torch.Tensor, torch.Tensor], Tuple] + + +def _leave_to_next_layer( + wrap: Callable[[torch.Tensor], UnreducedOutput], + hidden_states: torch.Tensor, + residual: torch.Tensor, +) -> Tuple[UnreducedOutput, torch.Tensor]: + return wrap(hidden_states), residual + + +class MixerExit: + """The scope that publishes a mixer's decision while it runs: inside the + ``with`` block ``fuse_mlp_allreduce`` on ``get_forward()`` tells its + row-parallel output projection to skip the all-reduce. It skips when the + stage's output always leaves its sum (to an FFN stage, which completes it in + its input), and when it may leave it and the fused kernel takes it into the + next input norm.""" + + __slots__ = ("skips_reduction", "_hands_on", "_scope") + + def __init__(self, communicator: LayerCommunicator, forward_batch: ForwardBatch): + produced = communicator._batch_steps(forward_batch).ffn_output + self._hands_on = ( + produced.leaves_for_next_layer + and communicator.should_fuse_mlp_allreduce_with_next_layer(forward_batch) + ) + self.skips_reduction = produced.always_leaves or self._hands_on + self._scope = get_forward().scoped(fuse_mlp_allreduce=self.skips_reduction) + + def __enter__(self) -> "MixerExit": + self._scope.__enter__() + return self + + def __exit__(self, *exc_info): + return self._scope.__exit__(*exc_info) + + def finish( + self, hidden_states: torch.Tensor + ) -> Union[torch.Tensor, UnreducedOutput]: + """The mixer's output: its partial sum to the FFN stage after it, as a + value that owes the sum to a mixer after it, or complete.""" + if self._hands_on: + return UnreducedOutput(hidden_states, group=get_parallel().tp_group) + return hidden_states + + +class FfnExit: + """The scope that publishes an FfnCompletion while the FFN runs: inside the + ``with`` block it is ``fuse_mlp_allreduce`` / ``mlp_reduce_scatter`` / + ``defer_moe_finalize`` on ``get_forward()``.""" + + __slots__ = ( + "communicator", + "forward_batch", + "defer_moe_finalize", + "fuse_mlp_allreduce", + "mlp_reduce_scatter", + "_complete", + "_scope", + ) + + def __init__(self, communicator: LayerCommunicator, forward_batch: ForwardBatch): + self.communicator = communicator + self.forward_batch = forward_batch + completion = communicator._select_ffn_completion(forward_batch) + self.defer_moe_finalize = completion.defer_moe_finalize + self.fuse_mlp_allreduce = completion.fuse_mlp_allreduce + self.mlp_reduce_scatter = completion.mlp_reduce_scatter + self._complete = completion.complete + self._scope = get_forward().scoped( + fuse_mlp_allreduce=self.fuse_mlp_allreduce, + mlp_reduce_scatter=self.mlp_reduce_scatter, + defer_moe_finalize=self.defer_moe_finalize, + ) + + def __enter__(self) -> "FfnExit": + self._scope.__enter__() + return self + + def __exit__(self, *exc_info): + return self._scope.__exit__(*exc_info) + + def finish( + self, hidden_states: torch.Tensor, residual: torch.Tensor + ) -> Tuple[Union[torch.Tensor, UnreducedOutput], torch.Tensor]: + """Leave the reduction to the next layer's input, or postprocess.""" + if not isinstance(hidden_states, torch.Tensor): + # A deferred MoE finalize handoff, consumed by the next prepare_attn. + assert self.defer_moe_finalize, "unrequested deferred MoE handoff" + return hidden_states, residual + return self._complete(hidden_states, residual) + + +class MHCLayerCommunicator(LayerCommunicator): + """A layer whose residual is hyper-connection streams: the shared boundary + steps, run with MHCState's residual operations.""" + + def __init__( + self, + layer_scatter_modes: LayerScatterModes, + input_layernorm: torch.nn.Module, + post_attention_layernorm: torch.nn.Module, + allow_reduce_scatter: bool = False, + qkv_latent_func: Optional[Callable] = None, + *, + hc_mult: int, + hc_attn_pre: Callable, + hc_ffn_pre: Callable, + hc_post: Callable, + hc_ffn_post_pre: Optional[Callable] = None, + ): + self.mhc = MHCState( + hc_mult=hc_mult, + hc_attn_pre=hc_attn_pre, + hc_ffn_pre=hc_ffn_pre, + hc_post=hc_post, + hc_ffn_post_pre=hc_ffn_post_pre, + is_last_layer=layer_scatter_modes.is_last_layer, + ) + if layer_scatter_modes.is_layer_sparse and sparse_moe_gathers_over_moe_cp(): + raise NotImplementedError( + "MHCLayerCommunicator does not support a MoE gathered over the " + "MoE-CP group (moe_dp_size < attention_context_parallel_size). " + "Increase moe_dp_size to match attention_context_parallel_size." + ) + # The postprocess writes the FFN output into the streams, so the FFN's + # sum never waits for the next layer. + super().__init__( + layer_scatter_modes, + input_layernorm, + post_attention_layernorm, + allow_reduce_scatter, + qkv_latent_func, + allow_deferred_ffn_reduction=False, + residual_ops=self.mhc, + ) + # MHC has not been run with input-scattered attention under attention CP. + if self._context.attn_cp_size > 1 and self._input_can_be_scattered(): + raise NotImplementedError( + "MHCLayerCommunicator with input-scattered attention under attention CP" + ) + + def _steps_from_declarations( + self, sides: DecoderLayerSides, **kwargs + ) -> BoundarySteps: + # MHC has not been run with an FFN input gathered over attention CP. + if ( + TokenAxis.ATTN_CP + in sides.attention_output.layout.sharded - sides.ffn.layout.sharded + ): + raise NotImplementedError( + f"MHCLayerCommunicator with a gather over attention CP: {sides=}" + ) + return super()._steps_from_declarations(sides, **kwargs) diff --git a/python/sglang/srt/layers/communicator/layout.py b/python/sglang/srt/layers/communicator/layout.py new file mode 100644 index 0000000000..e2f7b603df --- /dev/null +++ b/python/sglang/srt/layers/communicator/layout.py @@ -0,0 +1,251 @@ +# Copyright 2023-2024 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Token axes, layouts and process groups of what a layer hands across its +boundaries, and the configuration and batch facts they depend on.""" + +from dataclasses import dataclass +from enum import Enum, auto +from typing import Dict, FrozenSet, List, Mapping, Optional + +import msgspec + +from sglang.srt.distributed import GroupCoordinator +from sglang.srt.layers.attention.dsa.utils import ( + dsa_use_prefill_cp, + is_dsa_enable_prefill_cp, +) +from sglang.srt.layers.cp.utils import is_mla_cp_active, is_mla_cp_enabled +from sglang.srt.layers.dp_attention import ( + get_moe_cp_size, + is_dp_attention_enabled, + is_enable_moe_cp_allgather, +) +from sglang.srt.layers.moe import ( + is_moe_input_scattered_across_dp_ranks, + post_experts_reduction_group, +) +from sglang.srt.model_executor.forward_batch_info import ForwardBatch +from sglang.srt.runtime_context import get_parallel + + +class TokenAxis(Enum): + """A parallel dimension across which ranks hold different tokens.""" + + ATTN_DP = auto() + ATTN_CP = auto() + # Each attention-TP rank holds a slice of its group's tokens. + ATTN_TP_SCATTER = auto() + + +class Layout(msgspec.Struct, frozen=True): + """The token axes a rank's rows are sharded over. Single-rank axes are left + out, so two layouts are equal exactly when every rank holds the same tokens.""" + + sharded: FrozenSet[TokenAxis] + + @classmethod + def sharded_over( + cls, *axes: TokenAxis, axis_sizes: Mapping[TokenAxis, int] + ) -> "Layout": + return cls(frozenset(axis for axis in axes if axis_sizes[axis] > 1)) + + +class SumGroup(Enum): + """The group a stage's output is summed over, by name: the handle comes + from get_parallel() when the sum runs.""" + + ATTN_TP = auto() + TP = auto() + # The group one all-reduce of a MoE output runs over. + MOE_OUTPUT = auto() + + +class ScatterMode(Enum): + """ + Suppose we have TP=4, DP=2, enable-dp-attention, and the system handles seq a,b,c,d + Model input/output: [ab, ab, cd, cd] for four ranks respectively + SCATTERED: [a, b, c, d] + TP_ATTN_FULL: [ab, ab, cd, cd], i.e. all ranks inside a TP attn group have full data of the group + FULL: [abcd, abcd, abcd, abcd] + MOE_FULL: full within the MoE group (cp_per_moe CP chunks), used when moe_dp_size < attn_cp_size + """ + + SCATTERED = auto() + TP_ATTN_FULL = auto() + FULL = auto() + MOE_FULL = auto() + + @staticmethod + def model_input_output(): + """The scatter mode for model forward pass input and output data""" + if is_dsa_enable_prefill_cp() or is_mla_cp_enabled(): + return ScatterMode.SCATTERED + + return ScatterMode.TP_ATTN_FULL + + +def enable_moe_dense_fully_dp(): + return get_parallel().moe_dense_tp_size == 1 + + +def _generic_prefill_cp_shards_tokens() -> bool: + """Whether the strategy prefill CP path shards prefill tokens across CP ranks.""" + parallel = get_parallel() + return parallel.attn_cp_size > 1 and parallel.enable_prefill_cp + + +def enable_dwdp(): + return get_parallel().dwdp_size > 1 + + +def _batch_size(forward_batch: ForwardBatch) -> int: + return ( + forward_batch.input_ids.shape[0] if hasattr(forward_batch, "input_ids") else 0 + ) + + +def _ffn_has_tokens(forward_batch: ForwardBatch) -> bool: + if is_dp_attention_enabled(): + # The FFN runs on every DP rank's tokens, so every rank decides alike. + return (getattr(forward_batch, "global_dp_buffer_len", None) or 0) > 0 + return _batch_size(forward_batch) > 0 + + +@dataclass +class CommunicateContext: + process_group_sizes: Dict[ScatterMode, int] + attn_tp_rank: int + attn_tp_size: int + attn_dp_size: int + attn_cp_rank: int + attn_cp_size: int + tp_size: int + cache = None + tp_rank: int + force_layernorm_before_dp_gather: bool = False + + @classmethod + def init_new(cls): + attn_tp_rank = get_parallel().attn_tp_rank + attn_tp_size = get_parallel().attn_tp_size + attn_dp_size = get_parallel().attn_dp_size + attn_cp_size = get_parallel().attn_cp_size + attn_cp_rank = get_parallel().attn_cp_rank + tp_size = get_parallel().tp_size + tp_rank = get_parallel().tp_rank + moe_cp_size = get_moe_cp_size() + process_group_sizes = { + ScatterMode.SCATTERED: 1, + ScatterMode.TP_ATTN_FULL: attn_tp_size, + # TODO: support --moe-dense-tp-size > 1 + # With context parallel enabled, we should exclude + # the attn_cp_size from the total tp_size + ScatterMode.FULL: tp_size // attn_cp_size, + ScatterMode.MOE_FULL: tp_size // (attn_cp_size // moe_cp_size), + } + return cls( + process_group_sizes=process_group_sizes, + attn_tp_rank=attn_tp_rank, + attn_tp_size=attn_tp_size, + attn_dp_size=attn_dp_size, + attn_cp_rank=attn_cp_rank, + attn_cp_size=attn_cp_size, + tp_size=tp_size, + tp_rank=tp_rank, + ) + + +def moe_cp_gathered_rows(forward_batch: ForwardBatch) -> Optional[List[int]]: + """The real rows each MoE-CP rank contributes when this batch's FFN input is + gathered across the MoE-CP group, or None when it is not: only a context + parallel extend with CP metadata gathers, and only when the group has more + than one rank. The batch is read first, so a batch that is not a CP extend + never reads the group.""" + if ( + forward_batch.forward_mode.is_context_parallel_extend() + and forward_batch.attn_cp_metadata is not None + and get_moe_cp_size() > 1 + ): + return forward_batch.attn_cp_metadata.per_rank_actual_token + return None + + +def sparse_moe_gathers_over_moe_cp() -> bool: + """Whether a sparse MoE's input is gathered over the MoE-CP group on a CP + extend: a MoE on the TP group under GQA CP whose MoE-CP group is wider than + the MoE's data-parallel groups. DSA and MLA CP gather over attention CP + instead.""" + return ( + not is_moe_input_scattered_across_dp_ranks() + and is_enable_moe_cp_allgather() + and not _gathers_over_attention_cp() + ) + + +def moe_cp_gathers_sparse_moe_input(forward_batch: ForwardBatch) -> bool: + """Whether a sparse MoE's input is gathered over the MoE-CP group on this + batch: a CP extend, for a MoE that gathers there.""" + return ( + sparse_moe_gathers_over_moe_cp() + and moe_cp_gathered_rows(forward_batch) is not None + ) + + +def _cp_shard_token_rows(forward_batch: ForwardBatch) -> List[int]: + """Rows of each CP rank's shard that hold tokens; the shards are padded to + one length after them.""" + metadata = forward_batch.attn_cp_metadata + return metadata.per_rank_logical_token or metadata.per_rank_actual_token + + +def _sum_group(group: SumGroup) -> GroupCoordinator: + parallel = get_parallel() + if group is SumGroup.ATTN_TP: + return parallel.attn_tp_group + if group is SumGroup.TP: + return parallel.tp_group + return post_experts_reduction_group() + + +def token_axis_sizes(*, cp_active: bool = False) -> Dict[TokenAxis, int]: + """The token axes' sizes for a batch: attention CP shards tokens only on + a CP extend (``cp_active``); otherwise every CP rank holds them all.""" + parallel = get_parallel() + return { + TokenAxis.ATTN_DP: parallel.attn_dp_size, + TokenAxis.ATTN_CP: parallel.attn_cp_size if cp_active else 1, + TokenAxis.ATTN_TP_SCATTER: parallel.attn_tp_size, + } + + +def _gathers_over_attention_cp() -> bool: + """Whether a CP extend gathers the FFN input over the attention-CP group in + equal shards and takes the output back with a reduce-scatter there: DSA and + MLA CP. GQA prefill CP gathers over the MoE-CP group instead.""" + return is_dsa_enable_prefill_cp() or is_mla_cp_enabled() + + +def _batch_shards_over_cp(forward_batch: ForwardBatch) -> bool: + """Whether this batch's tokens are split across the CP ranks. Only a context + parallel extend is, so other batches, decode graph capture among them, + never read the CP predicates.""" + if not forward_batch.forward_mode.is_context_parallel_extend(): + return False + if _gathers_over_attention_cp(): + return dsa_use_prefill_cp(forward_batch) or is_mla_cp_active(forward_batch) + return moe_cp_gathered_rows(forward_batch) is not None + + +def _same_ranks(a: GroupCoordinator, b: GroupCoordinator) -> bool: + return sorted(a.ranks) == sorted(b.ranks) diff --git a/python/sglang/srt/layers/communicator/ops.py b/python/sglang/srt/layers/communicator/ops.py new file mode 100644 index 0000000000..f53e74c0be --- /dev/null +++ b/python/sglang/srt/layers/communicator/ops.py @@ -0,0 +1,794 @@ +# Copyright 2023-2024 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""The communication a boundary runs: the attention input, the FFN input and the +FFN output's way back, over attention TP, DP and CP.""" + +from typing import Callable, List, Optional, Tuple, Union + +import torch + +from sglang.srt.distributed import ( + GroupCoordinator, + attention_tensor_model_parallel_all_reduce, + attention_tensor_model_parallel_quant_all_reduce, + tensor_model_parallel_all_reduce, +) +from sglang.srt.distributed.device_communicators.pynccl_allocator import ( + use_symmetric_memory, +) +from sglang.srt.layers.communicator.adapters.attention import ( + AttentionInputs, + _redistribute_from_attn_tp_shards, + _redistribute_to_attn_tp_shards, + get_attn_tp_context, +) +from sglang.srt.layers.communicator.layout import ( + CommunicateContext, + Layout, + TokenAxis, + _cp_shard_token_rows, + moe_cp_gathered_rows, +) +from sglang.srt.layers.communicator.output import UnreducedOutput, reduce_output +from sglang.srt.layers.communicator.residual import ResidualOps +from sglang.srt.layers.communicator.residual.add_norm import ADD_AND_NORM +from sglang.srt.layers.communicator_dsa_cp import ( + dsa_cp_gather_hidden_states, + dsa_cp_reduce_scatter_hidden_states, +) +from sglang.srt.layers.dp_attention import ( + attn_tp_all_gather_into_tensor, + attn_tp_reduce_scatter_tensor, + can_use_dp_reduce_scatter, + dp_gather_partial, + dp_gather_replicate, + dp_reduce_scatter_tensor, + dp_scatter, + get_dp_global_num_tokens, + get_global_dp_buffer, + get_local_dp_buffer, + get_moe_cp_rank, + get_moe_cp_size, + is_allocation_symmetric, + moe_cp_all_gather_into_tensor, +) +from sglang.srt.layers.moe import should_use_dp_reduce_scatterv +from sglang.srt.model_executor.forward_batch_info import ForwardBatch +from sglang.srt.runtime_context import get_exec, get_parallel +from sglang.srt.utils import ( + is_npu, +) + +_is_npu = is_npu() + + +if _is_npu: + from sglang.srt.hardware_backend.npu.cmo import prepare_weight_cache + + +def tp_reduce_scatter( + hidden_states: torch.Tensor, + residual: Optional[torch.Tensor], + context: "CommunicateContext", +) -> Tuple[torch.Tensor, Optional[torch.Tensor]]: + """Module-level so MHC communicators can reuse it without a + ``LayerCommunicator`` instance.""" + if hidden_states.shape[0] == 0: + return hidden_states, hidden_states + assert hidden_states.shape[0] % context.tp_size == 0, ( + f"Expected total tokens {hidden_states.shape[0]} % tp_size {context.tp_size} to be 0" + ) + local_tokens = hidden_states.shape[0] // context.tp_size + output = hidden_states.new_empty(local_tokens, *hidden_states.shape[1:]) + get_parallel().tp_group.reduce_scatter_tensor(output, hidden_states) + if residual is not None: + residual = residual.tensor_split(context.tp_size)[context.tp_rank] + return output, residual + + +def layer_input_buffer( + hidden_states: Union[torch.Tensor, UnreducedOutput], +) -> torch.Tensor: + """The tensor holding a layer's input, for reusing its memory without reading it.""" + if isinstance(hidden_states, UnreducedOutput): + return hidden_states.partial + return hidden_states + + +class CommunicateSimpleFn: + @staticmethod + def _trivial( + hidden_states: torch.Tensor, + forward_batch: ForwardBatch, + context: CommunicateContext, + ) -> torch.Tensor: + return hidden_states + + @staticmethod + def _scattered_to_tp_attn_full( + hidden_states: Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]], + forward_batch: ForwardBatch, + context: CommunicateContext, + ) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]: + if isinstance(hidden_states, tuple): + gathered_hidden_states = [] + for local_hidden_states in hidden_states: + with use_symmetric_memory( + get_parallel().tp_group, + disabled=not is_allocation_symmetric(), + ): + output = torch.empty( + ( + local_hidden_states.shape[0] * context.attn_tp_size, + *local_hidden_states.shape[1:], + ), + dtype=local_hidden_states.dtype, + device=local_hidden_states.device, + ) + attn_tp_all_gather_into_tensor( + output, + local_hidden_states, + ) + gathered_hidden_states.append(output) + return tuple(gathered_hidden_states) + + return _redistribute_from_attn_tp_shards(hidden_states) + + +def _reduce_and_redistribute_output_to_attn_tp_shards( + hidden_states: torch.Tensor, context: CommunicateContext +) -> torch.Tensor: + local_hidden_states = hidden_states.tensor_split(context.attn_tp_size)[ + context.attn_tp_rank + ] + attn_tp_reduce_scatter_tensor(local_hidden_states, hidden_states) + return local_hidden_states + + +def _redistribute_input_to_moe_cp( + hidden_states: torch.Tensor, rows: List[int], moe_cp_size: int +) -> torch.Tensor: + # Zigzag split can produce unequal token counts across CP ranks + # (when seq_len % (cp_size * 2) != 0). NCCL allgather requires + # equal input sizes, so pad to the max per-rank token count. + max_tokens = max(rows) + pad_size = max_tokens - hidden_states.shape[0] + if pad_size > 0: + hidden_states = torch.nn.functional.pad(hidden_states, [0, 0, 0, pad_size]) + + output = torch.empty( + (max_tokens * moe_cp_size, hidden_states.shape[1]), + dtype=hidden_states.dtype, + device=hidden_states.device, + ) + moe_cp_all_gather_into_tensor(output, hidden_states) + return output + + +def _mlp_input_reduce_output( + hidden_states: torch.Tensor, forward_batch: ForwardBatch, may_quantize: bool = True +) -> torch.Tensor: + if ( + may_quantize + and not forward_batch.forward_mode.is_decode_or_idle() + and get_exec().comm.enable_quant_communications + ): + return attention_tensor_model_parallel_quant_all_reduce(hidden_states) + return attention_tensor_model_parallel_all_reduce(hidden_states) + + +def _redistribute_input_to_dp( + hidden_states: torch.Tensor, + forward_batch: ForwardBatch, + cp_shard_counts: Optional[List[int]] = None, +) -> torch.Tensor: + global_hidden_states = get_global_dp_buffer(get_parallel().tp_group) + dp_gather_replicate( + global_hidden_states, hidden_states, forward_batch, cp_shard_counts + ) + return global_hidden_states + + +def _reduce_and_redistribute_output_to_dp( + hidden_states: torch.Tensor, + forward_batch: ForwardBatch, + cp_shard_counts: Optional[List[int]] = None, +) -> torch.Tensor: + global_hidden_states = get_global_dp_buffer(get_parallel().tp_group) + dp_gather_partial( + global_hidden_states, hidden_states, forward_batch, cp_shard_counts + ) + return global_hidden_states + + +def move_rows( + hidden_states: torch.Tensor, + rows: Layout, + to: Layout, + forward_batch: ForwardBatch, +) -> torch.Tensor: + """A complete value from the rows it is on to ``to``: gathered over the + token axes ``to`` does not shard (attention TP, then attention DP), or cut to + this rank's share of those it does (attention DP, then attention TP).""" + gathered, cut = rows.sharded - to.sharded, to.sharded - rows.sharded + if (gathered and cut) or TokenAxis.ATTN_CP in gathered | cut: + raise NotImplementedError(f"{rows=} {to=}") + if TokenAxis.ATTN_TP_SCATTER in gathered: + hidden_states = _redistribute_from_attn_tp_shards(hidden_states) + if TokenAxis.ATTN_DP in gathered: + hidden_states = _redistribute_input_to_dp(hidden_states, forward_batch) + if TokenAxis.ATTN_DP in cut: + hidden_states = _to_local_tokens( + _redistribute_output, forward_batch, hidden_states + ) + if TokenAxis.ATTN_TP_SCATTER in cut: + parallel = get_parallel() + hidden_states = hidden_states.tensor_split(parallel.attn_tp_size)[ + parallel.attn_tp_rank + ] + return hidden_states + + +def _mlp_input_without_dp( + hidden_states: torch.Tensor, + residual: torch.Tensor, + forward_batch: ForwardBatch, + layernorm: torch.nn.Module, + context: CommunicateContext, + *, + gathers_residual: bool, + fusions: Tuple[Callable, ...], + residual_ops: ResidualOps = ADD_AND_NORM, +): + if gathers_residual: + residual = residual_ops.residual_from_attn_tp_shards(residual) + for fused in fusions: + result = fused(hidden_states, residual, forward_batch) + if result is not None: + return result + # MHC sums its streams in full precision. + hidden_states = _mlp_input_reduce_output( + hidden_states, forward_batch, may_quantize=residual_ops.adds_plainly + ) + if _is_npu and context.cache is not None: + _ = prepare_weight_cache(hidden_states, context.cache) + return residual_ops.update_and_read_ffn_input(hidden_states, residual, layernorm) + + +def _mlp_input_dp_replicate( + hidden_states: torch.Tensor, + residual: torch.Tensor, + forward_batch: ForwardBatch, + layernorm: torch.nn.Module, + context: CommunicateContext, + *, + gathers_residual: bool, + reduces_attention_tp: bool, + places_cp_shards: bool = False, + residual_ops: ResidualOps = ADD_AND_NORM, +): + """Attention DP: complete the attention-TP sum if it is owed, write the + output into the residual and read the FFN input locally, then gather. With + ``places_cp_shards`` each CP rank puts its shard of the DP group's tokens + beside the others' in the group's slot.""" + if gathers_residual: + residual = residual_ops.residual_from_attn_tp_shards(residual) + if hidden_states.shape[0] != 0: + if reduces_attention_tp: + hidden_states = attention_tensor_model_parallel_all_reduce(hidden_states) + with use_symmetric_memory( + get_parallel().tp_group, + disabled=not is_allocation_symmetric(), + ): + hidden_states, residual = residual_ops.update_and_read_ffn_input( + hidden_states, residual, layernorm + ) + else: + hidden_states, residual = residual_ops.update_and_read_ffn_input( + hidden_states, residual, layernorm + ) + cp_shard_counts = _cp_shard_token_rows(forward_batch) if places_cp_shards else None + return ( + _redistribute_input_to_dp(hidden_states, forward_batch, cp_shard_counts), + residual, + ) + + +def _mlp_input_dp_partial( + hidden_states: torch.Tensor, + residual: torch.Tensor, + forward_batch: ForwardBatch, + layernorm: torch.nn.Module, + context: CommunicateContext, + *, + gathers_residual: bool, + places_cp_shards: bool = False, +): + """Attention DP: one rank adds the residual, the gather sums it, then + normalize. With ``places_cp_shards`` each CP rank puts its shard of the DP + group's tokens beside the others' in the group's slot.""" + if gathers_residual: + residual = _redistribute_from_attn_tp_shards(residual) + if context.attn_tp_rank == 0: + hidden_states += residual + cp_shard_counts = _cp_shard_token_rows(forward_batch) if places_cp_shards else None + hidden_states = _reduce_and_redistribute_output_to_dp( + hidden_states, forward_batch, cp_shard_counts + ) + dp_scatter(residual, hidden_states, forward_batch, cp_shard_counts) + if hidden_states.shape[0] != 0: + hidden_states = layernorm(hidden_states) + return hidden_states, residual + + +def _hand_qkv_hook_its_input( + hidden_states: torch.Tensor, + forward_batch: ForwardBatch, + qkv_latent_func: Optional[Callable], +) -> torch.Tensor: + """Give the attention's QKV hook the attention input.""" + if qkv_latent_func is not None: + get_attn_tp_context().set_attn_inputs( + AttentionInputs(hidden_states, forward_batch, qkv_latent_func) + ) + return hidden_states + + +def _hand_scattered_input_to_attention( + hidden_states: torch.Tensor, + forward_batch: ForwardBatch, + qkv_latent_func: Optional[Callable], +) -> torch.Tensor: + """Input-scattered attention takes each rank's slice, and its QKV hook + gathers the rows after the projection. DSA and attention without a hook + consume full hidden states, so those are gathered here, and the hook is + told they are.""" + ctx = get_attn_tp_context() + if ctx.is_dsa or qkv_latent_func is None: + hidden_states = _tp_all_gather_scattered_rows(hidden_states, forward_batch) + if qkv_latent_func is not None: + ctx.set_attn_inputs( + AttentionInputs( + hidden_states, + forward_batch, + qkv_latent_func, + is_pre_gathered=ctx.is_dsa, + ) + ) + return hidden_states + + +def _tp_all_gather_scattered_rows( + hidden_states: torch.Tensor, forward_batch: ForwardBatch +) -> torch.Tensor: + # Input-scattered attention keeps the same number of tokens on every TP rank. + total_tokens = forward_batch.input_ids.shape[0] + output = hidden_states.new_empty((total_tokens, hidden_states.shape[-1])) + get_parallel().tp_group.all_gather_into_tensor(output, hidden_states) + return output + + +def _mlp_input_completing_owed( + hidden_states: Union[torch.Tensor, "UnreducedOutput"], + residual: torch.Tensor, + forward_batch: ForwardBatch, + layernorm: torch.nn.Module, + context: CommunicateContext, + *, + step: Callable, +): + """An FFN input whose producer may have left its sum: complete what the + value owes, then run ``step`` on the complete output.""" + return step( + reduce_output(hidden_states), residual, forward_batch, layernorm, context + ) + + +def _mlp_input_norm( + hidden_states: torch.Tensor, + residual: torch.Tensor, + forward_batch: ForwardBatch, + layernorm: torch.nn.Module, + context: CommunicateContext, + *, + residual_ops: ResidualOps = ADD_AND_NORM, +): + return residual_ops.update_and_read_ffn_input(hidden_states, residual, layernorm) + + +def _mlp_input_scatter( + hidden_states: torch.Tensor, + residual: torch.Tensor, + forward_batch: ForwardBatch, + layernorm: torch.nn.Module, + context: CommunicateContext, + *, + scatters_residual: bool, + residual_ops: ResidualOps = ADD_AND_NORM, +): + hidden_states = _reduce_and_redistribute_output_to_attn_tp_shards( + hidden_states, context + ) + if scatters_residual: + residual = residual_ops.residual_to_attn_tp_shard(residual, context) + return residual_ops.update_and_read_ffn_input(hidden_states, residual, layernorm) + + +def _mlp_input_slice( + hidden_states: torch.Tensor, + residual: Optional[torch.Tensor], + forward_batch: ForwardBatch, + layernorm: torch.nn.Module, + context: CommunicateContext, + *, + scatters_residual: bool, + residual_ops: ResidualOps = ADD_AND_NORM, +): + hidden_states = _redistribute_to_attn_tp_shards(hidden_states, context).clone() + if scatters_residual and residual is not None: + residual = residual_ops.residual_to_attn_tp_shard(residual, context) + return residual_ops.update_and_read_ffn_input(hidden_states, residual, layernorm) + + +def _mlp_input_on_residual_shard( + hidden_states: torch.Tensor, + residual: torch.Tensor, + forward_batch: ForwardBatch, + layernorm: torch.nn.Module, + context: CommunicateContext, + *, + residual_ops: ResidualOps, +): + """The residual stays on each rank's slice while the FFN takes the full + rows: reduce-scatter the attention output onto the slice, which completes + its sum, write it into the residual and read the FFN input there, then + gather the input back into the attention output's rows.""" + if hidden_states.shape[0] == 0: + return hidden_states, hidden_states + shard = hidden_states.tensor_split(context.tp_size)[context.tp_rank] + get_parallel().tp_group.reduce_scatter_tensor(shard, hidden_states) + shard, residual = residual_ops.update_and_read_ffn_input(shard, residual, layernorm) + attn_tp_all_gather_into_tensor(hidden_states, shard) + return hidden_states, residual + + +def _mlp_input_residual_into_sum( + hidden_states: torch.Tensor, + residual: torch.Tensor, + forward_batch: ForwardBatch, + layernorm: torch.nn.Module, + context: CommunicateContext, +): + return _tp_all_reduce_with_scattered_residual( + hidden_states, residual, layernorm, context + ) + + +def _tp_all_reduce_with_scattered_residual( + hidden_states: torch.Tensor, + residual: torch.Tensor, + layernorm: torch.nn.Module, + context: CommunicateContext, +): + if hidden_states.shape[0] == 0: + return hidden_states, hidden_states + + scattered_states = hidden_states.tensor_split(context.tp_size)[context.tp_rank] + scattered_states += residual + residual = tensor_model_parallel_all_reduce(hidden_states) + hidden_states = layernorm(residual) + return hidden_states, residual + + +def _mlp_input_gather_attention_cp( + hidden_states: torch.Tensor, + residual: torch.Tensor, + forward_batch: ForwardBatch, + layernorm: torch.nn.Module, + context: CommunicateContext, + *, + gather: Callable, +): + """DSA and MLA CP: complete this rank's shard, then gather the shards, of + equal length, over the attention-CP group. The residual stays on the + shard.""" + hidden_states, residual = gather( + hidden_states, residual, forward_batch, layernorm, context + ) + return dsa_cp_gather_hidden_states(hidden_states), residual + + +def _mlp_input_gather_moe_cp( + hidden_states: torch.Tensor, + residual: torch.Tensor, + forward_batch: ForwardBatch, + layernorm: torch.nn.Module, + context: CommunicateContext, + *, + gather: Callable, +): + """Gather for the FFN, then over the MoE-CP group so each rank holds all + tokens of its MoE group (moe_dp_size < attn_cp_size). The residual stays at + TP_ATTN_FULL.""" + # Early return on empty tensor is safe for MOE_CP because: + # - During CP extend: zigzag split guarantees all CP ranks have non-zero tokens, + # so no rank hits this path while others proceed to the allgather. + # - During decode: moe_cp allgather is skipped (guarded by is_context_parallel_extend). + # - CUDA graph warmup: not applicable when --cuda-graph-backend-prefill=disabled is used. + if hidden_states.shape[0] == 0: + return hidden_states, residual + + hidden_states, residual = gather( + hidden_states, residual, forward_batch, layernorm, context + ) + + rows = moe_cp_gathered_rows(forward_batch) + if rows is not None and hidden_states.shape[0] > 0: + hidden_states = _redistribute_input_to_moe_cp( + hidden_states, rows, get_moe_cp_size() + ) + + return hidden_states, residual + + +def _dp_scatter_group() -> GroupCoordinator: + parallel = get_parallel() + if parallel.tp_size == parallel.attn_dp_size: + return parallel.tp_group + return parallel.attn_tp_group + + +def _reduce_and_redistribute_output_varlen( + local_hidden_states: torch.Tensor, + hidden_states: torch.Tensor, + forward_batch: ForwardBatch, +) -> None: + get_parallel().tp_group.reduce_scatterv( + hidden_states, + output=local_hidden_states, + sizes=get_dp_global_num_tokens(), + ) + + +def _reduce_and_redistribute_output_max_len( + local_hidden_states: torch.Tensor, + hidden_states: torch.Tensor, + forward_batch: ForwardBatch, +) -> None: + dp_reduce_scatter_tensor(local_hidden_states, hidden_states) + + +def _redistribute_output( + local_hidden_states: torch.Tensor, + hidden_states: torch.Tensor, + forward_batch: ForwardBatch, +) -> None: + dp_scatter(local_hidden_states, hidden_states, forward_batch) + + +def _redistribute_output_from_moe_cp( + hidden_states: torch.Tensor, rows: List[int] +) -> torch.Tensor: + moe_cp_rank = get_moe_cp_rank() + # The allgather was padded to max_tokens_per_rank (equal chunks). + # Extract this rank's actual (non-padded) tokens from its chunk. + max_tokens_per_rank = max(rows) + actual_local_tokens = rows[moe_cp_rank] + return hidden_states.narrow( + 0, moe_cp_rank * max_tokens_per_rank, actual_local_tokens + ).contiguous() + + +def _reduce_and_redistribute_output_step( + forward_batch: ForwardBatch, + *, + leaves_for_reduce_scatter: bool, + leaves_for_reduce_scatterv: bool, +) -> Optional[Callable[[torch.Tensor, torch.Tensor, ForwardBatch], None]]: + """The reduce-scatter that brings an FFN output gathered over attention + DP back to this rank's tokens when the FFN leaves its sum to it (see + StageOutput); None when the FFN reduces the output and only a scatter + remains.""" + if should_use_dp_reduce_scatterv() and leaves_for_reduce_scatterv: + return _reduce_and_redistribute_output_varlen + if ( + leaves_for_reduce_scatter + and forward_batch.dp_padding_mode.is_max_len() + and can_use_dp_reduce_scatter() + ): + return _reduce_and_redistribute_output_max_len + return None + + +def _to_local_tokens( + step: Callable[[torch.Tensor, torch.Tensor, ForwardBatch], None], + forward_batch: ForwardBatch, + hidden_states: torch.Tensor, +) -> torch.Tensor: + local_hidden_states = get_local_dp_buffer(_dp_scatter_group()) + step(local_hidden_states, hidden_states, forward_batch) + return local_hidden_states + + +def _all_reduce_then_to_local_tokens( + group: GroupCoordinator, forward_batch: ForwardBatch, hidden_states: torch.Tensor +) -> torch.Tensor: + return _to_local_tokens( + _redistribute_output, forward_batch, group.all_reduce(hidden_states) + ) + + +class CommunicateSummableTensorPairFn: + """It is allowed to make (hidden_states, residual) := (hidden_states + residual, None) if needed.""" + + @staticmethod + def _trivial( + hidden_states: torch.Tensor, + residual: torch.Tensor, + forward_batch: ForwardBatch, + context: CommunicateContext, + **kwargs, + ): + return hidden_states, residual + + @staticmethod + def _scatter_hidden_states( + hidden_states: torch.Tensor, + residual: torch.Tensor, + forward_batch: ForwardBatch, + context: CommunicateContext, + allow_reduce_scatter: bool = False, + is_layer_sparse: bool = False, + ): + step = ( + _reduce_and_redistribute_output_step( + forward_batch, + leaves_for_reduce_scatter=allow_reduce_scatter, + # A MoE block leaves its sum to reduce_scatterv whenever it + # applies (should_skip_post_experts_all_reduce). + leaves_for_reduce_scatterv=allow_reduce_scatter or is_layer_sparse, + ) + or _redistribute_output + ) + return _to_local_tokens(step, forward_batch, hidden_states), residual + + @staticmethod + def _gather( + hidden_states: torch.Tensor, + residual: torch.Tensor, + forward_batch: ForwardBatch, + context: CommunicateContext, + residual_ops: ResidualOps = ADD_AND_NORM, + **kwargs, + ): + hidden_states = residual_ops.update_residual(hidden_states, residual) + return _redistribute_from_attn_tp_shards(hidden_states), None + + @staticmethod + def _onto_residual_shard( + hidden_states: torch.Tensor, + residual: torch.Tensor, + forward_batch: ForwardBatch, + context: CommunicateContext, + *, + sums: bool, + gathers_back: bool, + residual_ops: ResidualOps, + **kwargs, + ): + """Bring the FFN output onto the slice of the rows the residual is on: + a reduce-scatter that also completes its sum when ``sums``, else this + rank's slice of the complete output. With ``gathers_back`` it is + written into the residual there and the full rows are gathered back.""" + if sums: + hidden_states, _ = tp_reduce_scatter(hidden_states, None, context) + else: + hidden_states = hidden_states.tensor_split(context.tp_size)[context.tp_rank] + if not gathers_back: + return hidden_states, residual + local_states = residual_ops.update_residual(hidden_states, residual) + hidden_states = local_states.new_empty( + local_states.shape[0] * context.tp_size, *local_states.shape[1:] + ) + get_parallel().tp_group.all_gather_into_tensor(hidden_states, local_states) + return hidden_states, None + + @staticmethod + def _scatter( + hidden_states: torch.Tensor, + residual: torch.Tensor, + forward_batch: ForwardBatch, + context: CommunicateContext, + ): + assert residual is None, "not yet handled residual!=None" + return _redistribute_to_attn_tp_shards(hidden_states, context), None + + @staticmethod + def _take_back_attention_cp_shard( + hidden_states: torch.Tensor, + residual: torch.Tensor, + forward_batch: ForwardBatch, + context: CommunicateContext, + **kwargs, + ): + """DSA and MLA CP: this rank's shard of a complete output gathered in + equal shards over the attention-CP group.""" + parallel = get_parallel() + shard = hidden_states.tensor_split(parallel.attn_cp_size)[parallel.attn_cp_rank] + return shard, residual + + @staticmethod + def _reduce_scatter_over_cp( + hidden_states: torch.Tensor, + residual: torch.Tensor, + forward_batch: ForwardBatch, + context: CommunicateContext, + **kwargs, + ): + """DSA and MLA CP: sum the FFN output over the attention-CP group and + keep this rank's shard.""" + return dsa_cp_reduce_scatter_hidden_states(hidden_states), residual + + @staticmethod + def _take_back_cp_shard( + hidden_states: torch.Tensor, + residual: torch.Tensor, + forward_batch: ForwardBatch, + context: CommunicateContext, + **kwargs, + ): + """This rank's CP shard of the rows the DP gather put in its DP group's + slot, at the shard's padded length with the padding zeroed.""" + held = forward_batch.attn_cp_metadata.per_rank_actual_token + local_hidden_states = get_local_dp_buffer(_dp_scatter_group())[ + : held[get_parallel().attn_cp_rank] + ] + dp_scatter( + local_hidden_states, + hidden_states, + forward_batch, + _cp_shard_token_rows(forward_batch), + ) + return local_hidden_states, residual + + @staticmethod + def _scatter_hidden_states_moe( + hidden_states: torch.Tensor, + residual: torch.Tensor, + forward_batch: ForwardBatch, + context: CommunicateContext, + **kwargs, + ): + """Scatter MoE output back to TP_ATTN_FULL after MOE_FULL computation. + + After moe_tensor_model_parallel_all_reduce (which runs unconditionally since + mlp_reduce_scatter=False for this path), all ranks in the moe_cp group hold the + full MoE result for all cp_per_moe token chunks. We simply slice out this rank's + CP-local portion. + + If DP>1, further scatter back to the local DP slice. + """ + # Only scatter back during prefill; decode was never allgathered so no-op. + # Safe w.r.t. empty tensors: same reasoning as _mlp_input_gather_moe_cp + # — CP extend always has non-zero tokens per rank, and decode skips this path. + rows = moe_cp_gathered_rows(forward_batch) + if rows is not None: + hidden_states = _redistribute_output_from_moe_cp(hidden_states, rows) + + if context.attn_dp_size > 1: + hidden_states = _to_local_tokens( + _redistribute_output, forward_batch, hidden_states + ) + + return hidden_states, residual diff --git a/python/sglang/srt/layers/communicator/output.py b/python/sglang/srt/layers/communicator/output.py new file mode 100644 index 0000000000..c55b23f014 --- /dev/null +++ b/python/sglang/srt/layers/communicator/output.py @@ -0,0 +1,59 @@ +# Copyright 2023-2024 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Outputs that carry work their consumer still owes.""" + +from typing import Callable, Optional, Union + +import msgspec +import torch + +from sglang.srt.distributed import GroupCoordinator + + +class UnreducedOutput(msgspec.Struct, frozen=True): + """A layer output that still owes its sum, left for the next layer's input. + Hand it to the next layer, or pass it through reduce_output() before reading + it any other way. The producer says what is owed: one all-reduce over + ``group`` that keeps the layout, or ``reduce_and_redistribute``.""" + + partial: torch.Tensor + group: Optional[GroupCoordinator] = None + # Under attention DP: the reduction that also brings ``partial`` back to this + # rank's tokens (a reduce-scatter, or an all-reduce then a scatter). + reduce_and_redistribute: Optional[Callable[[torch.Tensor], torch.Tensor]] = None + + +class HandoffOutput(msgspec.Struct, frozen=True): + """A layer output that still owes work only its producer knows how to do (a + MoE's finalize and sum), left for the next layer's input. A fused kernel + there may do that work together with its own; anything else passes it + through reduce_output(), which calls ``complete()``.""" + + def complete(self) -> torch.Tensor: + """Do the owed work, unfused, and return the complete output.""" + raise NotImplementedError + + +def reduce_output( + hidden_states: Union[torch.Tensor, UnreducedOutput, HandoffOutput, None], +) -> Optional[torch.Tensor]: + """Run the work an UnreducedOutput or a HandoffOutput still owes; pass + anything else through.""" + if isinstance(hidden_states, UnreducedOutput): + if hidden_states.reduce_and_redistribute is not None: + return hidden_states.reduce_and_redistribute(hidden_states.partial) + return hidden_states.group.all_reduce(hidden_states.partial) + if isinstance(hidden_states, HandoffOutput): + return hidden_states.complete() + return hidden_states diff --git a/python/sglang/srt/layers/communicator/residual/__init__.py b/python/sglang/srt/layers/communicator/residual/__init__.py new file mode 100644 index 0000000000..e4dfdc0759 --- /dev/null +++ b/python/sglang/srt/layers/communicator/residual/__init__.py @@ -0,0 +1,60 @@ +# Copyright 2023-2024 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""How a layer writes its stages' outputs into the residual and reads the next +stage's input back.""" + +from typing import Protocol, Tuple + +import torch + + +class ResidualOps(Protocol): + """How a layer writes each stage's output into its residual and reads the + next stage's input from it. The boundary steps complete sums and move + tokens around these operations. AddAndNorm adds and normalizes; MHC's + hyper-connection streams implement the same operations (residual.mhc).""" + + # The write-back is a plain add, which one rank may run before the sum it + # adds into completes: the DP partial order, and a residual that joins the + # attention output's sum. + adds_plainly: bool + # The layer writes its FFN output into the residual itself instead of + # leaving that to the next layer's input. + updates_residual_after_ffn: bool + + def enter(self, hidden_states: torch.Tensor) -> torch.Tensor: + """The residual the layer stack starts from, given its input.""" + + def read_attention_input(self, residual, norm, quant_format: str) -> Tuple: + """The attention input and the residual, from a residual that already + holds the previous layer's output.""" + + def update_and_read_attention_input( + self, hidden_states, residual, norm, quant_format: str, post_residual_addition + ) -> Tuple: + """Write the previous layer's output into the residual, then read the + attention input from it.""" + + def update_and_read_ffn_input(self, hidden_states, residual, norm) -> Tuple: + """Write the attention output into the residual, then read the FFN + input from it.""" + + def update_residual(self, hidden_states, residual) -> torch.Tensor: + """The residual with the FFN output written into it.""" + + def residual_to_attn_tp_shard(self, residual, context) -> torch.Tensor: + """This attention-TP rank's slice of the residual.""" + + def residual_from_attn_tp_shards(self, residual) -> torch.Tensor: + """The residual gathered from every attention-TP rank's slice.""" diff --git a/python/sglang/srt/layers/communicator/residual/add_norm.py b/python/sglang/srt/layers/communicator/residual/add_norm.py new file mode 100644 index 0000000000..eb4c33f60d --- /dev/null +++ b/python/sglang/srt/layers/communicator/residual/add_norm.py @@ -0,0 +1,311 @@ +# Copyright 2023-2024 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""The plain residual: add and RMSNorm, and the fused all-reduce + add + norm +kernels.""" + +from typing import Optional + +import torch + +from sglang.srt.layers.communicator.adapters.attention import ( + _redistribute_from_attn_tp_shards, + _redistribute_to_attn_tp_shards, + get_attn_tp_context, +) +from sglang.srt.layers.dp_attention import is_dp_attention_enabled +from sglang.srt.layers.flashinfer_comm_fusion import ( + is_flashinfer_allreduce_unavailable, + uses_cutedsl_ar_fusion, +) +from sglang.srt.layers.quantization.fp8_utils import ( + _use_aiter_bpreshuffle_gfx95, + materialize_bpreshuffle_fp8_scale_tuple, +) +from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode +from sglang.srt.runtime_context import get_exec, get_parallel, get_platform +from sglang.srt.utils import ( + get_bool_env_var, + is_cuda, + is_flashinfer_available, + is_gfx95_supported, + is_gfx1250_supported, + is_hip, +) + +_is_cuda = is_cuda() +_is_flashinfer_available = is_flashinfer_available() +_is_sm90_supported = _is_cuda and get_platform().is_sm90 +_is_sm100_supported = _is_cuda and get_platform().is_sm100 +_use_aiter = get_bool_env_var("SGLANG_USE_AITER") and is_hip() +_is_gfx95_supported = is_gfx95_supported() +_is_gfx1250_supported = is_gfx1250_supported() + + +if _use_aiter: + from sglang.kernels.ops.quantization.fp8_kernel import fp8_dtype as _aiter_fp8_dtype + + if _is_gfx1250_supported: + from aiter.ops.triton.fused_fp8_quant import fused_rms_fp8_group_quant + else: + from aiter.ops.rmsnorm import add_rmsnorm_quant as _aiter_add_rmsnorm_quant + from aiter.ops.rmsnorm import rmsnorm_quant as _aiter_rmsnorm_quant + + if _is_gfx95_supported: + from aiter.ops.triton.fused_fp8_quant import fused_rms_fp8_group_quant + + from sglang.srt.layers.quantization.rocm_mxfp4_utils import ( + fused_rms_mxfp4_quant, + ) + + +def _fused_rmsnorm_fp8_per_token_quant( + hidden_states: torch.Tensor, + weight: torch.Tensor, + epsilon: float, + residual: Optional[torch.Tensor] = None, +): + """Fused (optional residual-add +) RMSNorm + FP8 per-token quantization. + + Only used with the aiter (ROCm) backend. + + Args: + residual: if provided, computes hidden_states + residual before RMSNorm + and returns updated residual_out as second element. + + Returns: + If residual is None: (out_fp8, scale) + If residual provided: ((out_fp8, scale), residual_out) + """ + if _is_gfx1250_supported: + # per-token quant == group quant with group_size == hidden size, giving + # an (M, 1) scale. + N = hidden_states.shape[-1] + (out_fp8, scale), _out1, _out2, residual_out = fused_rms_fp8_group_quant( + hidden_states, + weight, + epsilon, + group_size=N, + dtype_quant=_aiter_fp8_dtype, + res1=residual, + ) + if residual is not None: + return (out_fp8, scale), residual_out + return (out_fp8, scale) + + M, N = hidden_states.shape + out_fp8 = torch.empty((M, N), dtype=_aiter_fp8_dtype, device=hidden_states.device) + scale = torch.empty(M, dtype=torch.float32, device=hidden_states.device) + if residual is not None: + residual_out = torch.empty_like(hidden_states) + _aiter_add_rmsnorm_quant( + out_fp8, + hidden_states, + residual, + residual_out, + scale, + weight, + epsilon, + 0, # group_size=0 → per-token + ) + return (out_fp8, scale.unsqueeze(1)), residual_out + else: + _aiter_rmsnorm_quant( + out_fp8, + hidden_states, + scale, + weight, + epsilon, + 0, # group_size=0 → per-token + ) + return (out_fp8, scale.unsqueeze(1)) + + +# TODO: According to the discussion in https://github.com/flashinfer-ai/flashinfer/issues/1223#issuecomment-3047256465 +# We set the max token num to 128 for allreduce fusion with min-latency case(use_oneshot=True). +FUSE_ALLREDUCE_MAX_BATCH_SIZE = 2048 + + +def apply_flashinfer_allreduce_fusion(batch_size: int): + return ( + # NOTE: flashinfer 0.6.1 caused performance regression on sm100 for allreduce fusion + # Ref: https://github.com/sgl-project/sglang/issues/17237 + (_is_sm90_supported or _is_sm100_supported) + and _is_flashinfer_available + and not is_dp_attention_enabled() + and get_exec().comm.flashinfer_allreduce_fusion_backend is not None + # cutedsl runs its own fused path from the fusion communicator. + and not uses_cutedsl_ar_fusion() + and not is_flashinfer_allreduce_unavailable() + # Symbolic size checks stay last: under Dynamo tracing they guard on + # the dynamic token dim, so statically-off configs must short-circuit + # before reaching them. + and batch_size > 0 + and batch_size <= FUSE_ALLREDUCE_MAX_BATCH_SIZE + ) + + +def aiter_all_reduce_fusion_enabled_for(forward_mode: ForwardMode) -> bool: + comm = get_exec().comm + if not comm.enable_aiter_allreduce_fusion: + return False + if forward_mode.is_extend_or_draft_extend_or_mixed(): + return not comm.disable_aiter_allreduce_fusion_in_prefill + return not comm.disable_aiter_allreduce_fusion_in_decode + + +def apply_aiter_all_reduce_fusion( + input_tensor: torch.Tensor, forward_batch: ForwardBatch +): + n = input_tensor.shape[-1] + total_bytes = input_tensor.numel() * input_tensor.element_size() + # Aiter's should_custom_ar uses <= max_size/2 (64 MB); match that boundary. + return ( + _use_aiter + and total_bytes > 0 + and n <= 16384 + and total_bytes <= 8 * 1024 * 8192 + and get_parallel().tp_size != 6 + and not is_dp_attention_enabled() + and aiter_all_reduce_fusion_enabled_for(forward_batch.forward_mode) + ) + + +def _update_and_read_residual_plain( + norm, hidden_states, residual, post_residual_addition +): + if residual is None: + return norm(hidden_states), hidden_states + return norm(hidden_states, residual, post_residual_addition) + + +def _update_and_read_residual_aiter_mxfp4( + norm, hidden_states, residual, post_residual_addition +): + # post_residual_addition is not applied on this path. + output, *_, residual_out = fused_rms_mxfp4_quant( + hidden_states, + norm.weight, + norm.variance_epsilon, + None, + None, + None, + residual, + ) + return output, hidden_states if residual is None else residual_out + + +def _update_and_read_residual_aiter_fp8_group( + norm, hidden_states, residual, post_residual_addition +): + """aiter (ROCm gfx95) fused RMSNorm + FP8 group quant. Under DSA the + unquantized bf16 output rides along as a third element, so the DSA indexer + can skip dequantizing. post_residual_addition is not applied on this path.""" + needs_bf16 = get_attn_tp_context().is_dsa + output, unquantized, _, residual_out = fused_rms_fp8_group_quant( + hidden_states, + norm.weight, + norm.variance_epsilon, + inp2=None, + inp2_weight=None, + inp2_epsilon=None, + group_size=128, + dtype_quant=torch.float8_e4m3fn, + res1=residual, + output_unquantized_inp1=needs_bf16, + transpose_scale=False, + ) + if _use_aiter_bpreshuffle_gfx95: + output = materialize_bpreshuffle_fp8_scale_tuple(output) + if needs_bf16: + output = (output[0], output[1], unquantized) + return output, hidden_states if residual is None else residual_out + + +def _update_and_read_residual_aiter_fp8_per_token( + norm, hidden_states, residual, post_residual_addition +): + if residual is None: + output = _fused_rmsnorm_fp8_per_token_quant( + hidden_states, norm.weight.data, norm.variance_epsilon + ) + return output, hidden_states + if post_residual_addition is not None: + residual = residual + post_residual_addition + return _fused_rmsnorm_fp8_per_token_quant( + hidden_states, norm.weight.data, norm.variance_epsilon, residual=residual + ) + + +def _attn_input_update_and_read_residual(quant_format: str): + """Add the previous layer's output to the residual and read the attention + input from it: the input norm, fused with the quantization this format + wants. Without a residual (the first layer, or one already folded into + hidden_states) the input itself becomes the residual.""" + if _use_aiter and _is_gfx95_supported and "mxfp4" in quant_format: + return _update_and_read_residual_aiter_mxfp4 + if _use_aiter and _is_gfx95_supported and quant_format == "fp8": + return _update_and_read_residual_aiter_fp8_group + if _use_aiter and quant_format == "fp8_per_token": + return _update_and_read_residual_aiter_fp8_per_token + return _update_and_read_residual_plain + + +class AddAndNorm: + """A plain residual: a stage's output is added into it, and the next + stage's input is its norm. An empty batch skips the norms.""" + + adds_plainly = True + updates_residual_after_ffn = False + + def enter(self, hidden_states): + return hidden_states + + def read_attention_input(self, residual, norm, quant_format): + if residual.shape[0] == 0: + return residual, residual + return _attn_input_update_and_read_residual(quant_format)( + norm, residual, None, None + ) + + def update_and_read_attention_input( + self, hidden_states, residual, norm, quant_format, post_residual_addition + ): + if hidden_states.shape[0] == 0: + return hidden_states, hidden_states + return _attn_input_update_and_read_residual(quant_format)( + norm, hidden_states, residual, post_residual_addition + ) + + def update_and_read_ffn_input(self, hidden_states, residual, norm): + if residual is None: + # The layer stack starts at this FFN: its input is the residual. + if hidden_states.shape[0] == 0: + return hidden_states, hidden_states + return norm(hidden_states), hidden_states + if hidden_states.shape[0] == 0: + return hidden_states, residual + return norm(hidden_states, residual) + + def update_residual(self, hidden_states, residual): + hidden_states += residual + return hidden_states + + def residual_to_attn_tp_shard(self, residual, context): + return _redistribute_to_attn_tp_shards(residual, context) + + def residual_from_attn_tp_shards(self, residual): + return _redistribute_from_attn_tp_shards(residual) + + +ADD_AND_NORM = AddAndNorm() diff --git a/python/sglang/srt/layers/communicator_mhc.py b/python/sglang/srt/layers/communicator/residual/mhc.py similarity index 68% rename from python/sglang/srt/layers/communicator_mhc.py rename to python/sglang/srt/layers/communicator/residual/mhc.py index e276a6cfd9..aeb598531a 100644 --- a/python/sglang/srt/layers/communicator_mhc.py +++ b/python/sglang/srt/layers/communicator/residual/mhc.py @@ -11,19 +11,14 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== +"""Hyper-connection residual streams.""" + from dataclasses import dataclass from typing import Callable, Optional import torch from sglang.kernels.ops.layernorm.mhc import hc_contract, hc_expand -from sglang.srt.layers.boundary_layout import DecoderLayerSides, TokenAxis -from sglang.srt.layers.communicator import ( - BoundarySteps, - LayerCommunicator, - LayerScatterModes, - sparse_moe_gathers_over_moe_cp, -) @dataclass @@ -141,66 +136,3 @@ class MHCState: raise NotImplementedError( "Unsupported: h_res/h_post allgather not implemented." ) - - -class MHCLayerCommunicator(LayerCommunicator): - """A layer whose residual is hyper-connection streams: the shared boundary - steps, run with MHCState's residual operations.""" - - def __init__( - self, - layer_scatter_modes: LayerScatterModes, - input_layernorm: torch.nn.Module, - post_attention_layernorm: torch.nn.Module, - allow_reduce_scatter: bool = False, - qkv_latent_func: Optional[Callable] = None, - *, - hc_mult: int, - hc_attn_pre: Callable, - hc_ffn_pre: Callable, - hc_post: Callable, - hc_ffn_post_pre: Optional[Callable] = None, - ): - self.mhc = MHCState( - hc_mult=hc_mult, - hc_attn_pre=hc_attn_pre, - hc_ffn_pre=hc_ffn_pre, - hc_post=hc_post, - hc_ffn_post_pre=hc_ffn_post_pre, - is_last_layer=layer_scatter_modes.is_last_layer, - ) - if layer_scatter_modes.is_layer_sparse and sparse_moe_gathers_over_moe_cp(): - raise NotImplementedError( - "MHCLayerCommunicator does not support a MoE gathered over the " - "MoE-CP group (moe_dp_size < attention_context_parallel_size). " - "Increase moe_dp_size to match attention_context_parallel_size." - ) - # The postprocess writes the FFN output into the streams, so the FFN's - # sum never waits for the next layer. - super().__init__( - layer_scatter_modes, - input_layernorm, - post_attention_layernorm, - allow_reduce_scatter, - qkv_latent_func, - allow_deferred_ffn_reduction=False, - residual_ops=self.mhc, - ) - # MHC has not been run with input-scattered attention under attention CP. - if self._context.attn_cp_size > 1 and self._input_can_be_scattered(): - raise NotImplementedError( - "MHCLayerCommunicator with input-scattered attention under attention CP" - ) - - def _steps_from_declarations( - self, sides: DecoderLayerSides, **kwargs - ) -> BoundarySteps: - # MHC has not been run with an FFN input gathered over attention CP. - if ( - TokenAxis.ATTN_CP - in sides.attention_output.layout.sharded - sides.ffn.layout.sharded - ): - raise NotImplementedError( - f"MHCLayerCommunicator with a gather over attention CP: {sides=}" - ) - return super()._steps_from_declarations(sides, **kwargs) diff --git a/python/sglang/srt/layers/moe/cutedsl_ar_fusion.py b/python/sglang/srt/layers/moe/cutedsl_ar_fusion.py index d22b530b25..7eee467cb7 100644 --- a/python/sglang/srt/layers/moe/cutedsl_ar_fusion.py +++ b/python/sglang/srt/layers/moe/cutedsl_ar_fusion.py @@ -12,12 +12,13 @@ from typing import Callable, Optional, Sequence import torch -from sglang.srt.layers.boundary_layout import SumGroup, TokenAxis from sglang.srt.layers.communicator import ( FfnExitFusion, FusedMlpInput, HandoffOutput, LayerCommunicator, + SumGroup, + TokenAxis, UnreducedOutput, get_attn_tp_context, ) diff --git a/python/sglang/srt/models/glm5_next.py b/python/sglang/srt/models/glm5_next.py index 8631154f4a..ed71974c24 100644 --- a/python/sglang/srt/models/glm5_next.py +++ b/python/sglang/srt/models/glm5_next.py @@ -28,12 +28,12 @@ from sglang.srt.layers.attention.vision import VisionAttention from sglang.srt.layers.communicator import ( LayerCommunicator, LayerScatterModes, + MHCLayerCommunicator, enable_moe_dense_fully_dp, get_attn_tp_context, layer_input_buffer, reduce_output, ) -from sglang.srt.layers.communicator_mhc import MHCLayerCommunicator from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import ( ColumnParallelBatchedLinear, diff --git a/python/sglang/srt/models/nemotron_h_utils.py b/python/sglang/srt/models/nemotron_h_utils.py index 77f142ce2f..0532afc2c8 100644 --- a/python/sglang/srt/models/nemotron_h_utils.py +++ b/python/sglang/srt/models/nemotron_h_utils.py @@ -3,21 +3,19 @@ from typing import Optional from sglang.srt.configs.nemotron_h import ATTENTION, MAMBA, MOE -from sglang.srt.layers.boundary_layout import ( +from sglang.srt.layers.communicator import ( + InputRead, + LayerCommunicator, + LayerScatterModes, + LayerStage, Layout, + ScatterMode, StageDecl, StageInput, StageOutput, SumGroup, TokenAxis, stage_edges, -) -from sglang.srt.layers.communicator import ( - InputRead, - LayerCommunicator, - LayerScatterModes, - LayerStage, - ScatterMode, token_axis_sizes, ) from sglang.srt.layers.layernorm import RMSNorm diff --git a/python/sglang/test/communicator_patch.py b/python/sglang/test/communicator_patch.py new file mode 100644 index 0000000000..7dfb19b8fd --- /dev/null +++ b/python/sglang/test/communicator_patch.py @@ -0,0 +1,52 @@ +"""Patching a name the communicator package's modules look up. + +Each module of ``sglang.srt.layers.communicator`` binds the names it imports +in its own namespace, so a test double has to go into every module that reads +the name, not only into the package. +""" + +import contextlib +from unittest import mock + +from sglang.srt.layers import communicator +from sglang.srt.layers.communicator import boundary, layer, layout, ops, output +from sglang.srt.layers.communicator import residual as residual_ops +from sglang.srt.layers.communicator.adapters import attention +from sglang.srt.layers.communicator.residual import add_norm, mhc + +COMMUNICATOR_MODULES = ( + communicator, + layout, + output, + attention, + residual_ops, + add_norm, + mhc, + ops, + boundary, + layer, +) + + +@contextlib.contextmanager +def patch_communicator(name, new=mock.DEFAULT, **kwargs): + """``mock.patch.object(module, name, new, **kwargs)`` on every communicator + module that has ``name``, all with the same replacement; with + ``create=True`` and no module having it, on every module. Yields the + replacement.""" + targets = [m for m in COMMUNICATOR_MODULES if hasattr(m, name)] + if not targets: + if not kwargs.get("create"): + raise AttributeError(f"no communicator module has {name!r}") + targets = list(COMMUNICATOR_MODULES) + with contextlib.ExitStack() as stack: + replacement = stack.enter_context( + mock.patch.object(targets[0], name, new, **kwargs) + ) + for module in targets[1:]: + stack.enter_context( + mock.patch.object( + module, name, replacement, create=kwargs.get("create", False) + ) + ) + yield replacement diff --git a/test/registered/kernels/ops/layernorm/test_mhc_kernels.py b/test/registered/kernels/ops/layernorm/test_mhc_kernels.py index b2010cd6ce..5a7c936b5e 100644 --- a/test/registered/kernels/ops/layernorm/test_mhc_kernels.py +++ b/test/registered/kernels/ops/layernorm/test_mhc_kernels.py @@ -143,7 +143,7 @@ def test_mhc_fused_post_pre_matches_unfused( def _check_glm_boundary(x, residual, post, comb, fn, scale, base, *, use_norm): from sglang.srt.environ import envs - from sglang.srt.layers.communicator_mhc import MHCState + from sglang.srt.layers.communicator import MHCState from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.models.glm5_next import Glm5NextDecoderLayer diff --git a/test/registered/ops/test_aiter_allreduce_fusion_amd.py b/test/registered/ops/test_aiter_allreduce_fusion_amd.py index 9099409446..e5bb9121ed 100755 --- a/test/registered/ops/test_aiter_allreduce_fusion_amd.py +++ b/test/registered/ops/test_aiter_allreduce_fusion_amd.py @@ -7,14 +7,13 @@ import types import unittest from contextlib import ExitStack from pathlib import Path -from unittest import mock import torch -from sglang.srt.layers import communicator as comm from sglang.srt.layers.communicator import LayerCommunicator, ScatterMode from sglang.srt.model_executor.forward_batch_info import ForwardMode from sglang.test.ci.ci_register import register_amd_ci +from sglang.test.communicator_patch import patch_communicator from sglang.test.test_utils import CustomTestCase, publish_build_topology register_amd_ci(est_time=240, suite="stage-c-test-large-8-gpu-amd") @@ -400,27 +399,25 @@ class TestAiterAllreduceFusionGate(CustomTestCase): with ExitStack() as stack: stack.enter_context( - mock.patch.object(comm, "is_enable_moe_cp_allgather", lambda: False) + patch_communicator("is_enable_moe_cp_allgather", lambda: False) ) stack.enter_context( - mock.patch.object( - comm, + patch_communicator( "get_attn_tp_context", lambda: types.SimpleNamespace(input_scattered=False), ) ) # Force the NVIDIA/flashinfer term off so the aiter branch decides. stack.enter_context( - mock.patch.object( - comm, "apply_flashinfer_allreduce_fusion", lambda batch_size: False + patch_communicator( + "apply_flashinfer_allreduce_fusion", lambda batch_size: False ) ) - stack.enter_context(mock.patch.object(comm, "_use_aiter", use_aiter)) + stack.enter_context(patch_communicator("_use_aiter", use_aiter)) # moe_ep_size/moe_tp_size of 1 keep the hybrid EP+TP guard inactive # so the aiter branch is what decides. stack.enter_context( - mock.patch.object( - comm, + patch_communicator( "get_parallel", lambda: types.SimpleNamespace( tp_size=tp_world_size, moe_ep_size=1, moe_tp_size=1 @@ -440,7 +437,7 @@ class TestAiterAllreduceFusionGate(CustomTestCase): stack.enter_context(get_flags().dp.override(enabled=dp_attention)) stack.enter_context( - mock.patch.object(comm, "get_moe_a2a_backend", lambda: a2a_backend) + patch_communicator("get_moe_a2a_backend", lambda: a2a_backend) ) fake_self = _fake_self( diff --git a/test/registered/unit/batch_overlap/test_tbo_unreduced_input.py b/test/registered/unit/batch_overlap/test_tbo_unreduced_input.py index 9543a5e52f..de61383765 100644 --- a/test/registered/unit/batch_overlap/test_tbo_unreduced_input.py +++ b/test/registered/unit/batch_overlap/test_tbo_unreduced_input.py @@ -9,8 +9,7 @@ from unittest.mock import patch import torch from sglang.srt.batch_overlap import two_batch_overlap as tbo -from sglang.srt.layers.boundary_layout import Layout -from sglang.srt.layers.communicator import UnreducedOutput +from sglang.srt.layers.communicator import Layout, UnreducedOutput from sglang.srt.utils import empty_context from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.test_utils import CustomTestCase diff --git a/test/registered/unit/layers/moe/test_cutedsl_ar_fusion.py b/test/registered/unit/layers/moe/test_cutedsl_ar_fusion.py index cada5f5ba5..6be3fd8e8c 100644 --- a/test/registered/unit/layers/moe/test_cutedsl_ar_fusion.py +++ b/test/registered/unit/layers/moe/test_cutedsl_ar_fusion.py @@ -5,15 +5,17 @@ from unittest.mock import MagicMock, patch import pytest import torch -from sglang.srt.layers.boundary_layout import Layout, SumGroup, TokenAxis from sglang.srt.layers.communicator import ( ADD_AND_NORM, FfnExitFusion, LayerCommunicator, + Layout, + SumGroup, + TokenAxis, UnreducedOutput, - _attention_input_step, reduce_output, ) +from sglang.srt.layers.communicator.boundary import _attention_input_step from sglang.srt.layers.flashinfer_mnnvl_cutedsl import ( FlashInferMNNVLCuteDSLARFusion, _retargeted_config, @@ -31,6 +33,7 @@ from sglang.srt.model_executor.forward_batch_info import ForwardMode from sglang.srt.runtime_context import get_forward, publish, reset_context from sglang.srt.server_args import ServerArgs from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.communicator_patch import patch_communicator register_cpu_ci(est_time=9, suite="base-a-test-cpu") @@ -116,8 +119,8 @@ def test_last_layer_consumes_but_does_not_skip_the_pending_all_reduce(eligible): residual, ), ), - patch( - "sglang.srt.layers.communicator.reduce_output", + patch_communicator( + "reduce_output", lambda *a, **k: pytest.fail("fell through to the unfused path"), ), ): diff --git a/test/registered/unit/layers/test_boundary_edges.py b/test/registered/unit/layers/test_boundary_edges.py index f1cd675128..409b361c45 100644 --- a/test/registered/unit/layers/test_boundary_edges.py +++ b/test/registered/unit/layers/test_boundary_edges.py @@ -6,18 +6,23 @@ from unittest.mock import MagicMock import torch from sglang.srt.layers import communicator as comm -from sglang.srt.layers.boundary_layout import ( +from sglang.srt.layers.communicator import ( EdgeDecl, + InputRead, Layout, StageInput, StageOutput, SumGroup, TokenAxis, +) +from sglang.srt.layers.communicator import boundary as comm_boundary +from sglang.srt.layers.communicator import ( decoder_layer_edges, decoder_layer_sides, input_scattered_layer_sides, + make_boundary, ) -from sglang.srt.layers.communicator import InputRead, make_boundary +from sglang.srt.layers.communicator import ops as comm_ops from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.test_utils import CustomTestCase @@ -150,7 +155,7 @@ class TestTheConsumerHalfReadsOnlyItsOwnSide(CustomTestCase): ) self.assertEqual(len(halves), 1) ((func, keywords, move),) = halves - self.assertIs(func, comm._attention_input_step) + self.assertIs(func, comm_boundary._attention_input_step) self.assertIsNone(dict(keywords)["layer_input"]) self.assertIs(move, comm.CommunicateSimpleFn._trivial) @@ -188,7 +193,7 @@ class TestNonAlternatingEdges(CustomTestCase): residual_to=attention, ) step = make_boundary(complete, reads=InputRead.FFN).prepare - self.assertIs(step.func, comm._mlp_input_dp_replicate) + self.assertIs(step.func, comm_ops._mlp_input_dp_replicate) self.assertFalse(step.keywords["reduces_attention_tp"]) # A sum the value carries is completed first, then the same steps run. leaves = EdgeDecl( @@ -200,8 +205,8 @@ class TestNonAlternatingEdges(CustomTestCase): residual_to=attention, ) step = make_boundary(leaves, reads=InputRead.FFN).prepare - self.assertIs(step.func, comm._mlp_input_completing_owed) - self.assertIs(step.keywords["step"].func, comm._mlp_input_dp_replicate) + self.assertIs(step.func, comm_ops._mlp_input_completing_owed) + self.assertIs(step.keywords["step"].func, comm_ops._mlp_input_dp_replicate) # A producer that always leaves a sum other than the attention TP's is # not supported. always = EdgeDecl( diff --git a/test/registered/unit/layers/test_communicator_ffn_exit.py b/test/registered/unit/layers/test_communicator_ffn_exit.py index 04decb74df..fbea9e12ec 100644 --- a/test/registered/unit/layers/test_communicator_ffn_exit.py +++ b/test/registered/unit/layers/test_communicator_ffn_exit.py @@ -3,26 +3,29 @@ import types import unittest from functools import partial from pathlib import Path -from unittest.mock import MagicMock, patch +from unittest.mock import MagicMock import torch import sglang from sglang.srt.layers import communicator as comm -from sglang.srt.layers.boundary_layout import ( +from sglang.srt.layers.communicator import ( + LayerCommunicator, Layout, StageOutput, SumGroup, TokenAxis, - sequence_parallel_layer_sides, -) -from sglang.srt.layers.communicator import ( - LayerCommunicator, UnreducedOutput, +) +from sglang.srt.layers.communicator import boundary as comm_boundary +from sglang.srt.layers.communicator import ops as comm_ops +from sglang.srt.layers.communicator import ( reduce_output, + sequence_parallel_layer_sides, ) from sglang.srt.runtime_context import get_forward from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.communicator_patch import patch_communicator from sglang.test.test_utils import CustomTestCase register_cpu_ci(est_time=10, suite="base-a-test-cpu") @@ -30,7 +33,7 @@ register_cpu_ci(est_time=10, suite="base-a-test-cpu") def sp_region_steps(): """The steps a layer runs while a LayerNorm SP region is active.""" - return comm._select_boundary_steps( + return comm_boundary._select_boundary_steps( sequence_parallel_layer_sides( axis_sizes={ TokenAxis.ATTN_DP: 1, @@ -51,14 +54,14 @@ def ordinary_steps(ffn_output, *, returns_over_dp=False): reads and whether the output goes back over attention DP.""" return comm.BoundarySteps( attention_prepare=partial( - comm._attention_input_step, + comm_boundary._attention_input_step, layer_input=None, fusions=(), enters_stack=False, residual_ops=comm.ADD_AND_NORM, ), attention_input=comm.CommunicateSimpleFn._trivial, - ffn_input=comm._mlp_input_norm, + ffn_input=comm_ops._mlp_input_norm, ffn_input_rows=Layout(frozenset()), ffn_output=ffn_output, ffn_output_move=( @@ -157,7 +160,7 @@ class TestFfnExit(CustomTestCase): self.assertEqual(seen, (False, True)) self.assertIsInstance(hidden_states, UnreducedOutput) bound = hidden_states.reduce_and_redistribute - self.assertIs(bound.func, comm._to_local_tokens) + self.assertIs(bound.func, comm_ops._to_local_tokens) self.assertEqual(bound.args, (step, self.forward_batch)) self.assertIs(residual, self.residual) communicator._complete_ffn_output_now.assert_not_called() @@ -171,7 +174,7 @@ class TestFfnExit(CustomTestCase): _, (hidden_states, _) = self.run_exit(communicator) self.assertIsInstance(hidden_states, UnreducedOutput) bound = hidden_states.reduce_and_redistribute - self.assertIs(bound.func, comm._all_reduce_then_to_local_tokens) + self.assertIs(bound.func, comm_ops._all_reduce_then_to_local_tokens) self.assertEqual(bound.args, (group, self.forward_batch)) communicator._complete_ffn_output_now.assert_not_called() group.all_reduce.assert_not_called() @@ -371,8 +374,8 @@ class TestSelectFfnCompletion(CustomTestCase): return communicator def left(self, communicator, step, forward_batch=None): - with patch.object( - comm, "_reduce_and_redistribute_output_step", return_value=step + with patch_communicator( + "_reduce_and_redistribute_output_step", return_value=step ): completion = communicator._select_ffn_completion(forward_batch) hidden_states, _ = completion.complete(torch.ones(3, 4), None) @@ -381,17 +384,17 @@ class TestSelectFfnCompletion(CustomTestCase): def test_a_reduce_scatter_is_bound_for_the_next_layer(self): forward_batch = object() for step in ( - comm._reduce_and_redistribute_output_varlen, - comm._reduce_and_redistribute_output_max_len, + comm_ops._reduce_and_redistribute_output_varlen, + comm_ops._reduce_and_redistribute_output_max_len, ): with self.subTest(step=step.__name__): left = self.left(self.communicator(), step, forward_batch) bound = left.reduce_and_redistribute - self.assertIs(bound.func, comm._to_local_tokens) + self.assertIs(bound.func, comm_ops._to_local_tokens) self.assertEqual(bound.args, (step, forward_batch)) def test_postprocess_keeps_everything_else(self): - reduce_scatter = comm._reduce_and_redistribute_output_varlen + reduce_scatter = comm_ops._reduce_and_redistribute_output_varlen for name, communicator, step in ( ("scatter only", self.communicator(), None), ("last layer", self.communicator(is_last_layer=True), reduce_scatter), @@ -404,7 +407,7 @@ class TestSelectFfnCompletion(CustomTestCase): communicator = self.communicator(sp_region=True) with get_forward().scoped(sp_active=True): self.assertIsNone( - self.left(communicator, comm._reduce_and_redistribute_output_varlen) + self.left(communicator, comm_ops._reduce_and_redistribute_output_varlen) ) def test_a_deferred_sum_keeps_its_layout_or_scatters_back(self): @@ -415,7 +418,7 @@ class TestSelectFfnCompletion(CustomTestCase): moved = self.left(self.communicator(fuse=True), None, forward_batch) self.assertIsNone(moved.group) bound = moved.reduce_and_redistribute - self.assertIs(bound.func, comm._all_reduce_then_to_local_tokens) + self.assertIs(bound.func, comm_ops._all_reduce_then_to_local_tokens) self.assertEqual(bound.args, (self.group, forward_batch)) def test_the_all_reduce_runs_before_the_scatter(self): @@ -426,19 +429,17 @@ class TestSelectFfnCompletion(CustomTestCase): all_reduce=lambda x: calls.append(("all_reduce", x)) or x * 2 ) with ( - patch.object(comm, "_dp_scatter_group", return_value="group"), - patch.object( - comm, + patch_communicator("_dp_scatter_group", return_value="group"), + patch_communicator( "get_local_dp_buffer", side_effect=lambda group: calls.append(("buffer", group)) or local, ), - patch.object( - comm, + patch_communicator( "dp_scatter", side_effect=lambda out, full, fb: calls.append(("scatter", out)), ), ): - result = comm._all_reduce_then_to_local_tokens(group, object(), partial) + result = comm_ops._all_reduce_then_to_local_tokens(group, object(), partial) self.assertEqual([c[0] for c in calls], ["all_reduce", "buffer", "scatter"]) self.assertIs(calls[0][1], partial) self.assertIs(result, local) diff --git a/test/registered/unit/layers/test_declared_attention_cp.py b/test/registered/unit/layers/test_declared_attention_cp.py index 7327e41fd4..cb71fb34c2 100644 --- a/test/registered/unit/layers/test_declared_attention_cp.py +++ b/test/registered/unit/layers/test_declared_attention_cp.py @@ -19,6 +19,7 @@ from sglang.srt.layers import communicator as comm from sglang.srt.layers import communicator_dsa_cp as dsa_cp from sglang.srt.layers import layernorm_sp from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.communicator_patch import patch_communicator from sglang.test.test_utils import CustomTestCase register_cpu_ci(est_time=5, suite="base-a-test-cpu") @@ -154,7 +155,11 @@ class TestAttentionCpBoundary(CustomTestCase): collectives["reduce_scatter"], ), ]: - stack.enter_context(patch.object(*target, value)) + stack.enter_context( + patch_communicator(target[1], value) + if target[0] is comm + else patch.object(*target, value) + ) yield SimpleNamespace(parallel=parallel, flags=flags) def build(self, allow_reduce_scatter): diff --git a/test/registered/unit/layers/test_declared_decoder_boundary.py b/test/registered/unit/layers/test_declared_decoder_boundary.py index ec7f249729..37d518e83f 100644 --- a/test/registered/unit/layers/test_declared_decoder_boundary.py +++ b/test/registered/unit/layers/test_declared_decoder_boundary.py @@ -21,24 +21,32 @@ import msgspec import torch from sglang.srt.layers import communicator as comm -from sglang.srt.layers import communicator_mhc as mhc_module -from sglang.srt.layers.boundary_layout import ( - SumGroup, - TokenAxis, - decoder_layer_sides, - input_scattered_layer_sides, - sequence_parallel_layer_sides, -) +from sglang.srt.layers import layernorm_sp from sglang.srt.layers.communicator import ( LayerCommunicator, LayerScatterModes, + MHCLayerCommunicator, ScatterMode, + SumGroup, + TokenAxis, UnreducedOutput, ) -from sglang.srt.layers.communicator_mhc import MHCLayerCommunicator +from sglang.srt.layers.communicator import boundary as comm_boundary +from sglang.srt.layers.communicator import ( + decoder_layer_sides, + input_scattered_layer_sides, +) +from sglang.srt.layers.communicator import layer as comm_layer +from sglang.srt.layers.communicator import layout as comm_layout +from sglang.srt.layers.communicator import ops as comm_ops +from sglang.srt.layers.communicator import ( + sequence_parallel_layer_sides, +) +from sglang.srt.layers.communicator.residual import mhc as mhc_module from sglang.srt.layers.moe import utils as moe_utils from sglang.srt.runtime_context import LoRABatchLayout from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.communicator_patch import COMMUNICATOR_MODULES, patch_communicator from sglang.test.test_utils import CustomTestCase register_cpu_ci(est_time=15, suite="base-a-test-cpu") @@ -85,36 +93,33 @@ def planning(parallel, *, sp=False, a2a=False, dsa_cp=False): return get_parallel().attn_cp_size > get_parallel().moe_dp_size with ( - patch.object(comm, "get_parallel", get_parallel), - patch.object(comm, "is_dsa_enable_prefill_cp", lambda: dsa_cp), - patch.object(comm, "is_mla_cp_enabled", lambda: False), - patch.object( - comm, + patch_communicator("get_parallel", get_parallel), + patch_communicator("is_dsa_enable_prefill_cp", lambda: dsa_cp), + patch_communicator("is_mla_cp_enabled", lambda: False), + patch_communicator( "get_moe_cp_size", lambda: get_parallel().attn_cp_size if moe_cp_gathers() else 1, ), - patch.object(comm.layernorm_sp, "layernorm_sp_enabled", lambda: sp), - patch.object( - comm, "get_spec", lambda: SimpleNamespace(speculative_algorithm=None) + patch.object(layernorm_sp, "layernorm_sp_enabled", lambda: sp), + patch_communicator( + "get_spec", lambda: SimpleNamespace(speculative_algorithm=None) ), - patch.object( - comm, + patch_communicator( "get_moe_a2a_backend", lambda: SimpleNamespace(is_none=lambda: not a2a), ), - patch.object(comm, "is_moe_input_scattered_across_dp_ranks", lambda: a2a), - patch.object( - comm, "should_use_flashinfer_cutlass_moe_fp4_allgather", lambda: False + patch_communicator("is_moe_input_scattered_across_dp_ranks", lambda: a2a), + patch_communicator( + "should_use_flashinfer_cutlass_moe_fp4_allgather", lambda: False ), - patch.object(comm, "is_enable_moe_cp_allgather", moe_cp_gathers), - patch.object(comm, "get_lora", lambda: SimpleNamespace(enable_lora=False)), + patch_communicator("is_enable_moe_cp_allgather", moe_cp_gathers), + patch_communicator("get_lora", lambda: SimpleNamespace(enable_lora=False)), # A MoE whose EP and TP sums merge: its output's group is the TP group. - patch.object( - comm, "post_experts_reduction_group", lambda: get_parallel().tp_group + patch_communicator( + "post_experts_reduction_group", lambda: get_parallel().tp_group ), # Planning asks whether a dense layer gathers for two-batch overlap. - patch.object( - comm, + patch_communicator( "get_exec", lambda: SimpleNamespace( overlap=SimpleNamespace(enable_two_batch_overlap=False) @@ -233,10 +238,10 @@ class TestWhichLayersUseDeclarations(CustomTestCase): def declared(self, communicator): return getattr(communicator._steps.ffn_input, "func", None) in ( - comm._mlp_input_dp_partial, - comm._mlp_input_dp_replicate, - comm._mlp_input_without_dp, - comm._mlp_input_scatter, + comm_ops._mlp_input_dp_partial, + comm_ops._mlp_input_dp_replicate, + comm_ops._mlp_input_without_dp, + comm_ops._mlp_input_scatter, ) def test_a_dense_model(self): @@ -261,7 +266,7 @@ class TestWhichLayersUseDeclarations(CustomTestCase): post_attention_layernorm=norm(), ) self.assertIs( - communicator._steps.ffn_input.func, comm._mlp_input_without_dp + communicator._steps.ffn_input.func, comm_ops._mlp_input_without_dp ) entry = ( communicator._mlp_input_reduce_output_and_update_and_read_residual @@ -284,13 +289,13 @@ class TestWhichLayersUseDeclarations(CustomTestCase): single = build(layer_facts(1, 3), one_rank) with planning(one_rank): self.assertIsNotNone(single._declared_sides()) - self.assertIs(single._steps.ffn_input.func, comm._mlp_input_norm) + self.assertIs(single._steps.ffn_input.func, comm_ops._mlp_input_norm) def test_the_order_follows_the_attention_output(self): for attn_tp, force, order in ( - (2, False, comm._mlp_input_dp_partial), - (2, True, comm._mlp_input_dp_replicate), - (1, False, comm._mlp_input_dp_replicate), + (2, False, comm_ops._mlp_input_dp_partial), + (2, True, comm_ops._mlp_input_dp_replicate), + (1, False, comm_ops._mlp_input_dp_replicate), ): with self.subTest(attn_tp=attn_tp, force=force): communicator = build( @@ -332,7 +337,7 @@ class TestWhichLayersUseDeclarations(CustomTestCase): # dense layer; the next one takes it already sliced. self.assertEqual( [layers[i]._steps.ffn_input.func for i in (1, 2)], - [comm._mlp_input_scatter] * 2, + [comm_ops._mlp_input_scatter] * 2, ) self.assertEqual( [ @@ -347,7 +352,7 @@ class TestWhichLayersUseDeclarations(CustomTestCase): comm.CommunicateSimpleFn._scattered_to_tp_attn_full, ) self.assertIs( - layers[3]._steps.ffn_input.func, comm._mlp_input_dp_partial + layers[3]._steps.ffn_input.func, comm_ops._mlp_input_dp_partial ) self.assertTrue(layers[3]._steps.ffn_input.keywords["gathers_residual"]) @@ -361,7 +366,7 @@ class TestWhichLayersUseDeclarations(CustomTestCase): for a2a in (False, True): with ( self.subTest(a2a=a2a), - patch.object(comm, "get_exec", lambda: no_overlap), + patch_communicator("get_exec", lambda: no_overlap), ): layers = [ build( @@ -383,7 +388,9 @@ class TestWhichLayersUseDeclarations(CustomTestCase): ] self.assertEqual([self.declared(layer) for layer in layers], [True] * 4) for dense in layers[:2]: - self.assertIs(dense._steps.ffn_input.func, comm._mlp_input_scatter) + self.assertIs( + dense._steps.ffn_input.func, comm_ops._mlp_input_scatter + ) self.assertIsNone(dense._steps.ffn_output.group) for after_dense in layers[1:3]: self.assertIs( @@ -410,7 +417,7 @@ class TestWhichLayersUseDeclarations(CustomTestCase): modes = planned_modes( 2, 4, sparse=True, previous_sparse=True, parallel=parallel, a2a=True ) - with patch.object(comm, "_use_ag_after_qlora", True): + with patch_communicator("_use_ag_after_qlora", True): layer = build(modes, parallel, a2a=True) self.assertIs(layer._steps.attention_input, comm.CommunicateSimpleFn._trivial) @@ -454,7 +461,7 @@ def build_mhc( ) with ( planning(parallel, a2a=a2a, dsa_cp=dsa_cp), - patch.object(comm, "get_exec", lambda: overlap), + patch_communicator("get_exec", lambda: overlap), ): return MHCLayerCommunicator( layer_scatter_modes=modes, @@ -482,20 +489,20 @@ class TestMhcOnTheDeclarations(CustomTestCase): ( "attention TP 1", parallel_of(attn_dp=1, attn_tp=1), - comm._mlp_input_norm, + comm_ops._mlp_input_norm, comm.CommunicateSummableTensorPairFn._trivial, ), ( "the attention-TP sum", parallel_of(attn_dp=1, attn_tp=2), - comm._mlp_input_without_dp, + comm_ops._mlp_input_without_dp, comm.CommunicateSummableTensorPairFn._trivial, ), # The DP gather runs after hc_post, which is not a plain add. ( "a DP gather", parallel_of(attn_dp=2, attn_tp=2), - comm._mlp_input_dp_replicate, + comm_ops._mlp_input_dp_replicate, None, ), ): @@ -520,17 +527,17 @@ class TestMhcOnTheDeclarations(CustomTestCase): ) with ( planning(parallel), - patch.object(comm, "get_forward", lambda: SimpleNamespace(sp_active=False)), - patch.object(comm, "should_use_dp_reduce_scatterv", lambda: False), - patch.object(comm, "can_use_dp_reduce_scatter", lambda: True), + patch_communicator("get_forward", lambda: SimpleNamespace(sp_active=False)), + patch_communicator("should_use_dp_reduce_scatterv", lambda: False), + patch_communicator("can_use_dp_reduce_scatter", lambda: True), ): self.assertTrue(communicator.should_use_reduce_scatter(max_len)) def test_the_exit_runs_the_move_back_over_dp_it_chose(self): parallel = parallel_of(attn_dp=2, attn_tp=2) for name, reduce_scatterv, is_max_len, move in ( - ("SUM_LEN", True, False, comm._reduce_and_redistribute_output_varlen), - ("MAX_LEN", False, True, comm._reduce_and_redistribute_output_max_len), + ("SUM_LEN", True, False, comm_ops._reduce_and_redistribute_output_varlen), + ("MAX_LEN", False, True, comm_ops._reduce_and_redistribute_output_max_len), ): with self.subTest(name): hc_post = MagicMock(side_effect=lambda h, r, h_res, h_post: h + r) @@ -547,16 +554,16 @@ class TestMhcOnTheDeclarations(CustomTestCase): is_context_parallel_extend=lambda: False ), ) - choose = MagicMock(wraps=comm._reduce_and_redistribute_output_step) + choose = MagicMock(wraps=comm_ops._reduce_and_redistribute_output_step) to_local_tokens = MagicMock(side_effect=lambda step, fb, h: h[:2]) with ( planning(parallel), - patch.object( - comm, "should_use_dp_reduce_scatterv", lambda: reduce_scatterv + patch_communicator( + "should_use_dp_reduce_scatterv", lambda: reduce_scatterv ), - patch.object(comm, "can_use_dp_reduce_scatter", lambda: True), - patch.object(comm, "_reduce_and_redistribute_output_step", choose), - patch.object(comm, "_to_local_tokens", to_local_tokens), + patch_communicator("can_use_dp_reduce_scatter", lambda: True), + patch_communicator("_reduce_and_redistribute_output_step", choose), + patch_communicator("_to_local_tokens", to_local_tokens), ): with communicator.ffn_exit(forward_batch) as ffn_exit: self.assertTrue(ffn_exit.mlp_reduce_scatter) @@ -592,13 +599,13 @@ class TestMhcOnTheDeclarations(CustomTestCase): first_a2a, last = layers[1], layers[2] self.assert_step( first_a2a._steps.ffn_input, - comm._mlp_input_scatter, + comm_ops._mlp_input_scatter, first_a2a, scatters_residual=True, ) self.assert_step( last._steps.ffn_input, - comm._mlp_input_scatter, + comm_ops._mlp_input_scatter, last, scatters_residual=False, ) @@ -626,7 +633,7 @@ class TestMhcOnTheDeclarations(CustomTestCase): ) steps = communicator._input_scattered_steps self.assert_step( - steps.ffn_input, comm._mlp_input_on_residual_shard, communicator + steps.ffn_input, comm_ops._mlp_input_on_residual_shard, communicator ) self.assert_step( steps.ffn_output_move, @@ -649,7 +656,7 @@ class TestMhcOnTheDeclarations(CustomTestCase): ) self.assertIs(steps.attention_input, comm.CommunicateSimpleFn._trivial) self.assertIs( - steps.attention_handoff, comm._hand_scattered_input_to_attention + steps.attention_handoff, comm_ops._hand_scattered_input_to_attention ) self.assertIs( communicator._steps.ffn_output_move, @@ -664,7 +671,7 @@ class TestMhcOnTheDeclarations(CustomTestCase): layer_facts(1, 3, next_sparse=True), parallel, two_batch_overlap=True ) self.assert_step( - communicator._steps.ffn_input, comm._mlp_input_scatter, communicator + communicator._steps.ffn_input, comm_ops._mlp_input_scatter, communicator ) self.assert_step( communicator._steps.ffn_output_move, @@ -685,8 +692,8 @@ class TestMhcOnTheDeclarations(CustomTestCase): communicator.mhc.h_res = communicator.mhc.h_post = torch.zeros(2) with ( planning(parallel), - patch.object( - comm, "get_forward", lambda: SimpleNamespace(sp_active=False) + patch_communicator( + "get_forward", lambda: SimpleNamespace(sp_active=False) ), patch.object( mhc_module, "hc_contract", lambda h, hc_mult: h.sum(-1) @@ -772,7 +779,7 @@ class TestTwoBatchOverlap(CustomTestCase): parallel = parallel_of(attn_dp=2, attn_tp=2, moe_dense_tp_size=1) overlap = SimpleNamespace(overlap=SimpleNamespace(enable_two_batch_overlap=tbo)) layers = [] - with planning(parallel), patch.object(comm, "get_exec", lambda: overlap): + with planning(parallel), patch_communicator("get_exec", lambda: overlap): for i, sparse in enumerate((False, False, True, True)): modes = LayerScatterModes.init_new( layer_id=i, @@ -835,7 +842,7 @@ class TestTwoBatchOverlap(CustomTestCase): ) with ( planning(parallel, a2a=a2a), - patch.object(comm, "get_exec", lambda: overlap), + patch_communicator("get_exec", lambda: overlap), ): communicator = LayerCommunicator( layer_scatter_modes=modes, @@ -901,8 +908,7 @@ class TestTwoBatchOverlap(CustomTestCase): steps.ffn_input_rows, communicator._declared.ffn.layout ) published = {} - with patch.object( - comm, + with patch_communicator( "get_forward", lambda: SimpleNamespace(set=published.__setitem__), ): @@ -948,7 +954,7 @@ class TestTheAttentionOutputDecidesItsSum(CustomTestCase): TokenAxis.ATTN_TP_SCATTER: 2, } sides = sides_of(sizes) - steps, _ = comm._select_ffn_input( + steps, _ = comm_boundary._select_ffn_input( produced, residual=sides.input_rows, residual_to=sides.ffn_residual_rows, @@ -959,21 +965,20 @@ class TestTheAttentionOutputDecidesItsSum(CustomTestCase): reduced = [] context = SimpleNamespace(attn_tp_size=2, attn_tp_rank=0) with ( - patch.object( - comm, + patch_communicator( "attention_tensor_model_parallel_all_reduce", lambda x: reduced.append(x) or 2 * x, ), - patch.object( - comm, "_redistribute_input_to_dp", lambda h, fb, cp_shard_counts=None: h + patch_communicator( + "_redistribute_input_to_dp", lambda h, fb, cp_shard_counts=None: h ), - patch.object( - comm, "get_parallel", lambda: parallel_of(attn_dp=2, attn_tp=2) + patch_communicator( + "get_parallel", lambda: parallel_of(attn_dp=2, attn_tp=2) ), - patch.object( - comm, "use_symmetric_memory", lambda g, disabled=False: nullcontext() + patch_communicator( + "use_symmetric_memory", lambda g, disabled=False: nullcontext() ), - patch.object(comm, "is_allocation_symmetric", lambda: False), + patch_communicator("is_allocation_symmetric", lambda: False), ): hidden, residual = steps( torch.full((1, HIDDEN), 7.0), @@ -1057,7 +1062,7 @@ class TestFusedKernelsTakeOnlyTheStepsTheyComplete(CustomTestCase): TokenAxis.ATTN_TP_SCATTER: 2, } ) - return comm._select_ffn_input( + return comm_boundary._select_ffn_input( sides.attention_output, residual=sides.input_rows, residual_to=sides.ffn_residual_rows, @@ -1070,7 +1075,7 @@ class TestFusedKernelsTakeOnlyTheStepsTheyComplete(CustomTestCase): over_tp = self.fused(SumGroup.TP, True) over_attention_tp = self.fused(SumGroup.ATTN_TP, False) steps, chosen = self.select(attn_dp=1, fusions=(over_tp, over_attention_tp)) - self.assertIs(steps.func, comm._mlp_input_without_dp) + self.assertIs(steps.func, comm_ops._mlp_input_without_dp) self.assertEqual(steps.keywords["fusions"], (over_attention_tp.run,)) self.assertEqual(chosen, (over_attention_tp,)) @@ -1078,7 +1083,7 @@ class TestFusedKernelsTakeOnlyTheStepsTheyComplete(CustomTestCase): steps, chosen = self.select( attn_dp=2, fusions=(self.fused(SumGroup.ATTN_TP, True),) ) - self.assertIs(steps.func, comm._mlp_input_dp_partial) + self.assertIs(steps.func, comm_ops._mlp_input_dp_partial) self.assertEqual(chosen, ()) def test_aux_capture_follows_the_chosen_kernels(self): @@ -1113,9 +1118,9 @@ class TestTheSequenceParallelRegion(CustomTestCase): SIZES = {TokenAxis.ATTN_DP: 1, TokenAxis.ATTN_CP: 1, TokenAxis.ATTN_TP_SCATTER: 2} SP_STEPS = comm.BoundarySteps( - attention_prepare=comm._attention_input_step, + attention_prepare=comm_boundary._attention_input_step, attention_input=comm.CommunicateSimpleFn._trivial, - ffn_input=comm._mlp_input_norm, + ffn_input=comm_ops._mlp_input_norm, # The linears gather each rank's slice themselves. ffn_input_rows=comm.Layout(frozenset({TokenAxis.ATTN_TP_SCATTER})), ffn_output=sequence_parallel_layer_sides(axis_sizes=SIZES).ffn_output, @@ -1152,7 +1157,7 @@ class TestTheSequenceParallelRegion(CustomTestCase): self.assertEqual(sides.ffn.gathers_itself, local) self.assertIsNone(sides.attention_output.group) self.assertIsNone(sides.ffn_output.group) - steps = self.unbound(comm._select_boundary_steps(sides)) + steps = self.unbound(comm_boundary._select_boundary_steps(sides)) self.assertEqual( (steps.attention_input, steps.ffn_input, steps.ffn_output_move), ( @@ -1174,7 +1179,7 @@ class TestTheSequenceParallelRegion(CustomTestCase): ) self.assertEqual(self.unbound(communicator._sp_steps), self.SP_STEPS) self.assertIs( - communicator._steps.ffn_input.func, comm._mlp_input_without_dp + communicator._steps.ffn_input.func, comm_ops._mlp_input_without_dp ) def test_what_the_ffn_exit_reads_comes_from_the_batch_s_steps(self): @@ -1185,15 +1190,14 @@ class TestTheSequenceParallelRegion(CustomTestCase): with ( self.subTest(sp_active=active), planning(parallel, sp=True), - patch.object( - comm, "get_forward", lambda: SimpleNamespace(sp_active=active) + patch_communicator( + "get_forward", lambda: SimpleNamespace(sp_active=active) ), - patch.object( - comm, + patch_communicator( "get_attn_tp_context", lambda: SimpleNamespace(input_scattered=False), ), - patch.object(comm, "is_dp_attention_enabled", lambda: False), + patch_communicator("is_dp_attention_enabled", lambda: False), ): self.assertIs( communicator._ffn_sum_can_move_to_next_layer(None), not active @@ -1210,13 +1214,15 @@ class TestTheSequenceParallelRegion(CustomTestCase): def test_the_same_selector_chooses_a_layer_s_ordinary_steps(self): # Under attention DP the FFN exit chooses the move back per batch. - kept = comm._select_boundary_steps(sides_of(self.SIZES, leaves_next=True)) + kept = comm_boundary._select_boundary_steps( + sides_of(self.SIZES, leaves_next=True) + ) self.assertTrue(kept.ffn_output.leaves_for_next_layer) self.assertTrue(kept.ffn_sum_is_movable) self.assertIs( kept.ffn_output_move, comm.CommunicateSummableTensorPairFn._trivial ) - returned = comm._select_boundary_steps( + returned = comm_boundary._select_boundary_steps( sides_of({**self.SIZES, TokenAxis.ATTN_DP: 2}) ) self.assertTrue(returned.returns_over_dp) @@ -1239,13 +1245,13 @@ class TestInputScatteredAttention(CustomTestCase): ffn_group=SumGroup.TP, hands_on_partial=hands_on, ) - steps = comm._select_boundary_steps(sides) + steps = comm_boundary._select_boundary_steps(sides) self.assertIs( steps.attention_prepare.keywords["layer_input"], comm.tp_reduce_scatter, ) self.assertIs(steps.attention_input, comm.CommunicateSimpleFn._trivial) - self.assertIs(steps.ffn_input, comm._mlp_input_residual_into_sum) + self.assertIs(steps.ffn_input, comm_ops._mlp_input_residual_into_sum) self.assertIs( steps.ffn_output_move, comm.CommunicateSummableTensorPairFn._trivial, @@ -1262,11 +1268,11 @@ class TestInputScatteredAttention(CustomTestCase): local = comm.Layout(frozenset({TokenAxis.ATTN_TP_SCATTER})) owed = comm.StageOutput(attention, group=SumGroup.ATTN_TP, always_leaves=True) for joins, step in ( - (True, comm._mlp_input_residual_into_sum), - (False, comm._mlp_input_without_dp), + (True, comm_ops._mlp_input_residual_into_sum), + (False, comm_ops._mlp_input_without_dp), ): with self.subTest(residual_joins_sum=joins): - steps, _ = comm._select_ffn_input( + steps, _ = comm_boundary._select_ffn_input( owed, residual=local, residual_to=attention, @@ -1283,7 +1289,7 @@ class TestInputScatteredAttention(CustomTestCase): sizes = self.SIZES attention = comm.Layout.sharded_over(axis_sizes=sizes) local = comm.Layout(frozenset({TokenAxis.ATTN_TP_SCATTER})) - step, _ = comm._select_ffn_input( + step, _ = comm_boundary._select_ffn_input( comm.StageOutput(attention), residual=attention, residual_to=local, @@ -1291,7 +1297,7 @@ class TestInputScatteredAttention(CustomTestCase): force_layernorm_before_gather=False, fusions=(), ) - self.assertIs(step.func, comm._mlp_input_slice) + self.assertIs(step.func, comm_ops._mlp_input_slice) hidden = torch.arange(4.0)[:, None].expand(4, HIDDEN).clone() residual = torch.ones(4, HIDDEN) context = SimpleNamespace(attn_tp_size=2, attn_tp_rank=1) @@ -1333,13 +1339,12 @@ class TestInputScatteredAttention(CustomTestCase): for scattered in (False, True): with ( self.subTest(input_scattered=scattered), - patch.object( - comm, + patch_communicator( "get_attn_tp_context", lambda: SimpleNamespace(input_scattered=scattered), ), - patch.object( - comm, "get_forward", lambda: SimpleNamespace(sp_active=False) + patch_communicator( + "get_forward", lambda: SimpleNamespace(sp_active=False) ), ): self.assertIs( @@ -1362,13 +1367,12 @@ class TestInputScatteredAttention(CustomTestCase): ) communicator = build(layer_facts(1, 3), parallel, allow_reduce_scatter=True) with ( - patch.object(comm, "get_parallel", lambda: parallel), - patch.object( - comm, + patch_communicator("get_parallel", lambda: parallel), + patch_communicator( "get_attn_tp_context", lambda: SimpleNamespace(input_scattered=True), ), - patch.object(comm, "get_forward", lambda: SimpleNamespace(sp_active=False)), + patch_communicator("get_forward", lambda: SimpleNamespace(sp_active=False)), ): hidden, residual = communicator.prepare_attn( torch.ones(4, HIDDEN), torch.full((4, HIDDEN), 3.0), None @@ -1500,8 +1504,8 @@ class TestPrefillCP(CustomTestCase): ) communicator = build(modes, parallel, dsa_cp=True, allow_reduce_scatter=True) cp, ordinary = communicator._cp_steps, communicator._steps - self.assertIs(cp.ffn_input.func, comm._mlp_input_gather_attention_cp) - self.assertIs(cp.ffn_input.keywords["gather"].func, comm._mlp_input_norm) + self.assertIs(cp.ffn_input.func, comm_ops._mlp_input_gather_attention_cp) + self.assertIs(cp.ffn_input.keywords["gather"].func, comm_ops._mlp_input_norm) self.assertIs( cp.ffn_output_move, comm.CommunicateSummableTensorPairFn._reduce_scatter_over_cp, @@ -1511,14 +1515,13 @@ class TestPrefillCP(CustomTestCase): # The dense layer before it ran on the same shard: nothing to move. self.assertIs(cp.attention_input, comm.CommunicateSimpleFn._trivial) # Other batches hold every token on each CP rank: the MoE sums itself. - self.assertIs(ordinary.ffn_input.func, comm._mlp_input_norm) + self.assertIs(ordinary.ffn_input.func, comm_ops._mlp_input_norm) self.assertIs( ordinary.ffn_output_move, comm.CommunicateSummableTensorPairFn._trivial ) with ( - patch.object(comm, "get_forward", lambda: SimpleNamespace(sp_active=False)), - patch.object( - comm, + patch_communicator("get_forward", lambda: SimpleNamespace(sp_active=False)), + patch_communicator( "get_attn_tp_context", lambda: SimpleNamespace(input_scattered=False), ), @@ -1526,7 +1529,7 @@ class TestPrefillCP(CustomTestCase): for shards, leaves in ((True, True), (False, False)): with ( self.subTest(cp_extend=shards), - patch.object(comm, "_batch_shards_over_cp", lambda fb: shards), + patch_communicator("_batch_shards_over_cp", lambda fb: shards), ): self.assertIs( communicator._ffn_leaves_sum_to_reduce_scatter( @@ -1556,10 +1559,12 @@ class TestPrefillCP(CustomTestCase): ): with ( self.subTest(cp_extend=cp_extend, active=active), - patch.object(comm, "dsa_use_prefill_cp", lambda fb: active), - patch.object(comm, "is_mla_cp_active", lambda fb: False), + patch_communicator("dsa_use_prefill_cp", lambda fb: active), + patch_communicator("is_mla_cp_active", lambda fb: False), ): - self.assertIs(comm._batch_shards_over_cp(batch(cp_extend)), shards) + self.assertIs( + comm_layout._batch_shards_over_cp(batch(cp_extend)), shards + ) def test_dsa_cp_dense_layers_run_on_their_shard(self): parallel = self.dsa_parallel() @@ -1568,7 +1573,7 @@ class TestPrefillCP(CustomTestCase): ) communicator = build(modes, parallel, dsa_cp=True, allow_reduce_scatter=True) for steps in (communicator._steps, communicator._cp_steps): - self.assertIs(steps.ffn_input.func, comm._mlp_input_norm) + self.assertIs(steps.ffn_input.func, comm_ops._mlp_input_norm) self.assertIsNone(steps.ffn_output.group) self.assertIs( steps.ffn_output_move, comm.CommunicateSummableTensorPairFn._trivial @@ -1577,9 +1582,11 @@ class TestPrefillCP(CustomTestCase): def test_a_cp_extend_gathers_over_cp_and_takes_its_chunk_back(self): communicator = build(layer_facts(1, 3), self.cp_parallel()) cp = communicator._cp_steps - self.assertIs(cp.ffn_input.func, comm._mlp_input_gather_moe_cp) + self.assertIs(cp.ffn_input.func, comm_ops._mlp_input_gather_moe_cp) # Each rank completes its own chunk before the gather. - self.assertIs(cp.ffn_input.keywords["gather"].func, comm._mlp_input_without_dp) + self.assertIs( + cp.ffn_input.keywords["gather"].func, comm_ops._mlp_input_without_dp + ) self.assertIs( cp.ffn_output_move, comm.CommunicateSummableTensorPairFn._scatter_hidden_states_moe, @@ -1587,7 +1594,9 @@ class TestPrefillCP(CustomTestCase): self.assertFalse(cp.returns_over_dp) self.assertFalse(cp.ffn_output.leaves_for_next_layer) self.assertFalse(cp.ffn_output.leaves_for_reduce_scatter) - self.assertIs(communicator._steps.ffn_input.func, comm._mlp_input_without_dp) + self.assertIs( + communicator._steps.ffn_input.func, comm_ops._mlp_input_without_dp + ) self.assertIs( communicator._steps.ffn_output_move, comm.CommunicateSummableTensorPairFn._trivial, @@ -1603,13 +1612,13 @@ class TestPrefillCP(CustomTestCase): ) for steps in (communicator._cp_steps, communicator._steps): # The attention's rows are the MoE's: nothing to gather or take back. - self.assertIs(steps.ffn_input.func, comm._mlp_input_without_dp) + self.assertIs(steps.ffn_input.func, comm_ops._mlp_input_without_dp) self.assertIs( steps.ffn_output_move, comm.CommunicateSummableTensorPairFn._trivial ) self.assertTrue(steps.ffn_output.leaves_for_next_layer) dense = build(layer_facts(1, 3), parallel, allow_deferred_ffn_reduction=True) - self.assertIs(dense._cp_steps.ffn_input.func, comm._mlp_input_gather_moe_cp) + self.assertIs(dense._cp_steps.ffn_input.func, comm_ops._mlp_input_gather_moe_cp) self.assertFalse(dense._cp_steps.ffn_output.leaves_for_next_layer) def test_the_fused_kernels_run_on_each_chunk(self): @@ -1721,12 +1730,12 @@ class TestPrefillCP(CustomTestCase): parallel = parallel_of(attn_dp=2, attn_tp=2, attn_cp=2, enable_prefill_cp=True) communicator = build(layer_facts(1, 3), parallel) cp, ordinary = communicator._cp_steps, communicator._steps - self.assertIs(cp.ffn_input.func, comm._mlp_input_dp_partial) + self.assertIs(cp.ffn_input.func, comm_ops._mlp_input_dp_partial) self.assertTrue(cp.ffn_input.keywords["places_cp_shards"]) self.assertIs( cp.ffn_output_move, comm.CommunicateSummableTensorPairFn._take_back_cp_shard ) - self.assertIs(ordinary.ffn_input.func, comm._mlp_input_dp_partial) + self.assertIs(ordinary.ffn_input.func, comm_ops._mlp_input_dp_partial) self.assertFalse(ordinary.ffn_input.keywords["places_cp_shards"]) self.assertTrue(ordinary.returns_over_dp) for steps in (cp, ordinary): @@ -1755,12 +1764,11 @@ class TestPrefillCP(CustomTestCase): with ( self.subTest(expected=expected is communicator._cp_steps), planning(self.cp_parallel()), - patch.object(comm, "moe_cp_gathered_rows", rows), - patch.object( - comm, "get_forward", lambda: SimpleNamespace(sp_active=False) + patch_communicator("moe_cp_gathered_rows", rows), + patch_communicator( + "get_forward", lambda: SimpleNamespace(sp_active=False) ), - patch.object( - comm, + patch_communicator( "get_attn_tp_context", lambda: SimpleNamespace(input_scattered=False), ), @@ -1784,13 +1792,12 @@ class TestBranchRows(CustomTestCase): ) with ( planning(parallel), - patch.object(comm, "get_forward", lambda: SimpleNamespace(sp_active=False)), - patch.object( - comm, + patch_communicator("get_forward", lambda: SimpleNamespace(sp_active=False)), + patch_communicator( "get_attn_tp_context", lambda: SimpleNamespace(input_scattered=False), ), - patch.object(comm, "moe_cp_gathered_rows", lambda fb: [2, 1]), + patch_communicator("moe_cp_gathered_rows", lambda fb: [2, 1]), ): return communicator._branch_rows(batch) @@ -1835,7 +1842,7 @@ class TestBranchRows(CustomTestCase): moves.append((value, rows, to)) return value - with patch.object(comm, "move_rows", recorded): + with patch_communicator("move_rows", recorded): dense.branch_input(moe, "h0", "residual", None) self.assertEqual( moves, @@ -1875,11 +1882,10 @@ class TestBranchRows(CustomTestCase): value = torch.arange(4.0).view(4, 1) with ( - patch.object(comm, "_redistribute_from_attn_tp_shards", tp_gather), - patch.object(comm, "_redistribute_input_to_dp", lambda h, fb: dp_gather(h)), - patch.object(comm, "_to_local_tokens", lambda step, fb, h: dp_scatter(h)), - patch.object( - comm, + patch_communicator("_redistribute_from_attn_tp_shards", tp_gather), + patch_communicator("_redistribute_input_to_dp", lambda h, fb: dp_gather(h)), + patch_communicator("_to_local_tokens", lambda step, fb, h: dp_scatter(h)), + patch_communicator( "get_parallel", lambda: SimpleNamespace(attn_tp_size=2, attn_tp_rank=1), ), @@ -2080,12 +2086,12 @@ def running(*, reduce_scatterv, a2a=False): with ExitStack() as stack: stack.enter_context(planning(lambda: state().parallel, a2a=a2a)) for name, value in replaced.items(): - if hasattr(comm, name): - stack.enter_context(patch.object(comm, name, value)) + if any(hasattr(m, name) for m in COMMUNICATOR_MODULES): + stack.enter_context(patch_communicator(name, value)) # What the MoE declares its skipped reduction leaves, read in its module. for name, value in ( ("get_parallel", lambda: state().parallel), - ("get_moe_a2a_backend", comm.get_moe_a2a_backend), + ("get_moe_a2a_backend", comm_layer.get_moe_a2a_backend), ( "post_experts_output_is_complete", replaced["post_experts_output_is_complete"], @@ -2124,10 +2130,10 @@ def moe(x, state): if ( state.flags.fuse_mlp_allreduce or state.flags.mlp_reduce_scatter - or comm.should_use_dp_reduce_scatterv() + or comm_ops.should_use_dp_reduce_scatterv() ): return partial_sum - return comm.post_experts_reduction_group().all_reduce(partial_sum) + return comm_layout.post_experts_reduction_group().all_reduce(partial_sum) def reference(x): @@ -2357,19 +2363,17 @@ class TestTheFfnInputReduction(CustomTestCase): ) calls = [] with ( - patch.object(comm, "get_exec", lambda: exec_), - patch.object( - comm, + patch_communicator("get_exec", lambda: exec_), + patch_communicator( "attention_tensor_model_parallel_quant_all_reduce", lambda h: calls.append("quant") or h, ), - patch.object( - comm, + patch_communicator( "attention_tensor_model_parallel_all_reduce", lambda h: calls.append("full") or h, ), ): - comm._mlp_input_without_dp( + comm_ops._mlp_input_without_dp( torch.ones(2, HIDDEN), torch.ones(2, HIDDEN), batch, diff --git a/test/registered/unit/layers/test_dp_attention_cp_gather.py b/test/registered/unit/layers/test_dp_attention_cp_gather.py index 317af1e7c7..4030cf2307 100644 --- a/test/registered/unit/layers/test_dp_attention_cp_gather.py +++ b/test/registered/unit/layers/test_dp_attention_cp_gather.py @@ -14,12 +14,14 @@ import torch from sglang.srt.layers import communicator as comm from sglang.srt.layers import dp_attention, layernorm_sp +from sglang.srt.layers.communicator import ops as comm_ops from sglang.srt.layers.cp import base as cp_base from sglang.srt.layers.cp import padding as cp_padding from sglang.srt.layers.cp.zigzag import ZigzagCPStrategy from sglang.srt.layers.dp_attention import DpPaddingMode from sglang.srt.model_executor.forward_batch_info import ForwardMode from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.communicator_patch import patch_communicator from sglang.test.test_utils import CustomTestCase register_cpu_ci(est_time=10, suite="base-a-test-cpu") @@ -237,7 +239,11 @@ class TestDpCpGather(CustomTestCase): ).double(), ), ]: - stack.enter_context(patch.object(*target, value)) + stack.enter_context( + patch_communicator(target[1], value) + if target[0] is comm + else patch.object(*target, value) + ) communicator = comm.LayerCommunicator( layer_scatter_modes=SimpleNamespace( is_first_layer=False, @@ -383,7 +389,7 @@ class TestDpCpGather(CustomTestCase): norm=rms_norm, norm_rows=rms_rows, force_layernorm_before_dp_gather=True, - ffn_input=comm._mlp_input_dp_replicate, + ffn_input=comm_ops._mlp_input_dp_replicate, ) diff --git a/test/registered/unit/layers/test_layer_communicator_fusion_gate.py b/test/registered/unit/layers/test_layer_communicator_fusion_gate.py index 30fe2c1bb7..97c2c4d907 100644 --- a/test/registered/unit/layers/test_layer_communicator_fusion_gate.py +++ b/test/registered/unit/layers/test_layer_communicator_fusion_gate.py @@ -7,14 +7,19 @@ from unittest.mock import MagicMock, patch import torch from sglang.srt.layers import communicator as comm -from sglang.srt.layers.boundary_layout import ( +from sglang.srt.layers.communicator import ( + LayerCommunicator, Layout, + ScatterMode, StageOutput, SumGroup, TokenAxis, +) +from sglang.srt.layers.communicator import boundary as comm_boundary +from sglang.srt.layers.communicator import ops as comm_ops +from sglang.srt.layers.communicator import ( sequence_parallel_layer_sides, ) -from sglang.srt.layers.communicator import LayerCommunicator, ScatterMode from sglang.srt.layers.moe import ( can_merge_post_experts_all_reduce, deferred_post_experts_all_reduce, @@ -23,6 +28,7 @@ from sglang.srt.layers.moe import ( from sglang.srt.layers.moe import utils as moe_utils from sglang.srt.runtime_context import get_forward, get_parallel from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.communicator_patch import patch_communicator from sglang.test.test_utils import CustomTestCase register_cpu_ci(est_time=11, suite="base-a-test-cpu") @@ -30,7 +36,7 @@ register_cpu_ci(est_time=11, suite="base-a-test-cpu") def sp_region_steps(): """The steps a layer runs while a LayerNorm SP region is active.""" - return comm._select_boundary_steps( + return comm_boundary._select_boundary_steps( sequence_parallel_layer_sides( axis_sizes={ TokenAxis.ATTN_DP: 1, @@ -44,14 +50,14 @@ def sp_region_steps(): def _steps(*, ffn_output=None, returns_over_dp=False, ffn_sum_is_movable=True): return comm.BoundarySteps( attention_prepare=partial( - comm._attention_input_step, + comm_boundary._attention_input_step, layer_input=None, fusions=(), enters_stack=False, residual_ops=comm.ADD_AND_NORM, ), attention_input=comm.CommunicateSimpleFn._trivial, - ffn_input=comm._mlp_input_norm, + ffn_input=comm_ops._mlp_input_norm, ffn_input_rows=Layout(frozenset()), ffn_output=ffn_output or StageOutput(Layout(frozenset()), group=SumGroup.TP), ffn_output_move=( @@ -304,10 +310,9 @@ class TestFuseMlpAllReduceGate(CustomTestCase): input_ids=types.SimpleNamespace(shape=(8,)) ) with ( - patch.object(comm, "is_enable_moe_cp_allgather", return_value=False), - patch.object(comm, "apply_flashinfer_allreduce_fusion", return_value=True), - patch.object( - comm, + patch_communicator("is_enable_moe_cp_allgather", return_value=False), + patch_communicator("apply_flashinfer_allreduce_fusion", return_value=True), + patch_communicator( "get_attn_tp_context", return_value=types.SimpleNamespace(input_scattered=False), ), @@ -392,21 +397,19 @@ class TestDeferFfnReduction(CustomTestCase): global_dp_buffer_len=global_tokens, ) with ( - patch.object( - comm, "_reduce_and_redistribute_output_step", return_value=step + patch_communicator( + "_reduce_and_redistribute_output_step", return_value=step ), get_forward().scoped(sp_active=sp_active), - patch.object(comm, "is_enable_moe_cp_allgather", return_value=False), - patch.object(comm, "apply_flashinfer_allreduce_fusion", return_value=fused), - patch.object(comm, "_use_aiter", False), - patch.object( - comm, + patch_communicator("is_enable_moe_cp_allgather", return_value=False), + patch_communicator("apply_flashinfer_allreduce_fusion", return_value=fused), + patch_communicator("_use_aiter", False), + patch_communicator( "get_attn_tp_context", return_value=types.SimpleNamespace(input_scattered=False), ), - patch.object(comm, "is_dp_attention_enabled", return_value=dp_attention), - patch.object( - comm, + patch_communicator("is_dp_attention_enabled", return_value=dp_attention), + patch_communicator( "get_moe_a2a_backend", return_value=types.SimpleNamespace(is_none=lambda: a2a_none), ), @@ -497,10 +500,13 @@ class TestDeferFfnReductionUnderAttentionDp(CustomTestCase): def test_keeps_what_the_next_layer_input_cannot_run(self): for name, condition in ( - ("reduce-scatter", dict(step=comm._reduce_and_redistribute_output_varlen)), + ( + "reduce-scatter", + dict(step=comm_ops._reduce_and_redistribute_output_varlen), + ), ( "MAX_LEN reduce-scatter", - dict(step=comm._reduce_and_redistribute_output_max_len), + dict(step=comm_ops._reduce_and_redistribute_output_max_len), ), ("other postprocess", dict(scatters_to_local_tokens=False)), ("LayerNorm SP", dict(sp_active=True)), diff --git a/test/registered/unit/layers/test_layer_scatter_modes_cp_dense_mlp.py b/test/registered/unit/layers/test_layer_scatter_modes_cp_dense_mlp.py index 3a112232e8..f086b8a47c 100644 --- a/test/registered/unit/layers/test_layer_scatter_modes_cp_dense_mlp.py +++ b/test/registered/unit/layers/test_layer_scatter_modes_cp_dense_mlp.py @@ -1,9 +1,8 @@ import unittest -from unittest.mock import patch -from sglang.srt.layers import communicator as comm from sglang.srt.layers.communicator import LayerScatterModes, ScatterMode from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.communicator_patch import patch_communicator from sglang.test.test_utils import CustomTestCase register_cpu_ci(est_time=7, suite="base-a-test-cpu") @@ -16,10 +15,10 @@ class TestDenseMlpScatterModeUnderPrefillCP(CustomTestCase): def test_dense_mlp_gathers_across_cp(self): with ( - patch.object(comm, "_generic_prefill_cp_shards_tokens", return_value=True), - patch.object(comm, "is_dsa_enable_prefill_cp", return_value=False), - patch.object(comm, "is_mla_cp_enabled", return_value=False), - patch.object(comm, "enable_moe_dense_fully_dp", return_value=False), + patch_communicator("_generic_prefill_cp_shards_tokens", return_value=True), + patch_communicator("is_dsa_enable_prefill_cp", return_value=False), + patch_communicator("is_mla_cp_enabled", return_value=False), + patch_communicator("enable_moe_dense_fully_dp", return_value=False), ): modes = LayerScatterModes.init_new( layer_id=1, diff --git a/test/registered/unit/layers/test_layernorm_sp.py b/test/registered/unit/layers/test_layernorm_sp.py index 4a9bb8021c..03ef34de0a 100644 --- a/test/registered/unit/layers/test_layernorm_sp.py +++ b/test/registered/unit/layers/test_layernorm_sp.py @@ -15,11 +15,14 @@ import torch from sglang.srt.arg_groups.layernorm_sp_hook import validate_layernorm_sp from sglang.srt.layers import communicator as comm from sglang.srt.layers import layernorm_sp -from sglang.srt.layers.boundary_layout import ( +from sglang.srt.layers.communicator import ( Layout, StageOutput, SumGroup, TokenAxis, +) +from sglang.srt.layers.communicator import boundary as comm_boundary +from sglang.srt.layers.communicator import ( sequence_parallel_layer_sides, ) from sglang.srt.model_executor.forward_batch_info import ForwardMode @@ -31,6 +34,7 @@ from sglang.srt.runtime_context import ( ) from sglang.srt.server_args import ServerArgs from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.communicator_patch import patch_communicator from sglang.test.test_utils import CustomTestCase register_cpu_ci(est_time=9, suite="base-a-test-cpu") @@ -38,7 +42,7 @@ register_cpu_ci(est_time=9, suite="base-a-test-cpu") def sp_region_steps(): """The steps a layer runs while a LayerNorm SP region is active.""" - return comm._select_boundary_steps( + return comm_boundary._select_boundary_steps( sequence_parallel_layer_sides( axis_sizes={ TokenAxis.ATTN_DP: 1, @@ -188,7 +192,7 @@ class TestSpRegionSteps(CustomTestCase): """The layer's steps outside the region, which must not run inside it.""" return comm.BoundarySteps( attention_prepare=partial( - comm._attention_input_step, + comm_boundary._attention_input_step, layer_input=None, fusions=(), enters_stack=False, @@ -207,8 +211,7 @@ class TestSpRegionSteps(CustomTestCase): with ( get_flags().sp.override(enabled=True), get_forward().scoped(sp_active=False), - patch.object( - comm, + patch_communicator( "get_attn_tp_context", return_value=SimpleNamespace(input_scattered=False), ), @@ -258,8 +261,7 @@ class TestSpRegionSteps(CustomTestCase): get_flags().sp.override(enabled=True), get_forward().scoped(sp_active=True), ): - with patch.object( - comm, + with patch_communicator( "get_attn_tp_context", return_value=SimpleNamespace(input_scattered=False), ): diff --git a/test/registered/unit/layers/test_postprocess_reduce_scatterv.py b/test/registered/unit/layers/test_postprocess_reduce_scatterv.py index 2c848af163..944a37180c 100644 --- a/test/registered/unit/layers/test_postprocess_reduce_scatterv.py +++ b/test/registered/unit/layers/test_postprocess_reduce_scatterv.py @@ -18,6 +18,8 @@ from sglang.srt.layers.communicator import ( LayerScatterModes, ScatterMode, ) +from sglang.srt.layers.communicator import boundary as comm_boundary +from sglang.srt.layers.communicator import ops as comm_ops from sglang.srt.layers.moe import utils as moe_utils from sglang.srt.layers.moe.utils import ( should_skip_mlp_all_reduce, @@ -25,6 +27,7 @@ from sglang.srt.layers.moe.utils import ( ) from sglang.srt.runtime_context import get_forward from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.communicator_patch import patch_communicator from sglang.test.test_utils import CustomTestCase register_cpu_ci(est_time=5, suite="base-a-test-cpu") @@ -35,14 +38,14 @@ def steps(*, ffn_output_move): back over attention DP.""" return comm.BoundarySteps( attention_prepare=partial( - comm._attention_input_step, + comm_boundary._attention_input_step, layer_input=None, fusions=(), enters_stack=False, residual_ops=comm.ADD_AND_NORM, ), attention_input=comm.CommunicateSimpleFn._trivial, - ffn_input=comm._mlp_input_norm, + ffn_input=comm_ops._mlp_input_norm, ffn_input_rows=comm.Layout(frozenset()), ffn_output=comm.StageOutput(comm.Layout(frozenset())), ffn_output_move=ffn_output_move, @@ -53,7 +56,7 @@ def steps(*, ffn_output_move): @contextlib.contextmanager def reduce_scatterv_applies(): with ( - patch.object(comm, "should_use_dp_reduce_scatterv", return_value=True), + patch_communicator("should_use_dp_reduce_scatterv", return_value=True), patch.object(moe_utils, "should_use_dp_reduce_scatterv", return_value=True), ): yield @@ -73,11 +76,11 @@ def postprocess_sums(*, allow_reduce_scatter, is_layer_sparse): ) with ( reduce_scatterv_applies(), - patch.object(comm, "get_parallel", return_value=parallel), - patch.object(comm, "get_local_dp_buffer", return_value=torch.empty(1, 4)), - patch.object(comm, "get_dp_global_num_tokens", return_value=[1, 1]), - patch.object( - comm, "dp_scatter", side_effect=lambda *args: calls.append("dp_scatter") + patch_communicator("get_parallel", return_value=parallel), + patch_communicator("get_local_dp_buffer", return_value=torch.empty(1, 4)), + patch_communicator("get_dp_global_num_tokens", return_value=[1, 1]), + patch_communicator( + "dp_scatter", side_effect=lambda *args: calls.append("dp_scatter") ), ): CommunicateSummableTensorPairFn._scatter_hidden_states( @@ -141,12 +144,12 @@ class TestPostprocessReduceScatterv(CustomTestCase): for sparse in (False, True): with ( self.subTest(sparse=sparse), - patch.object( - comm, "sparse_mlp_scatter_mode", return_value=ScatterMode.FULL + patch_communicator( + "sparse_mlp_scatter_mode", return_value=ScatterMode.FULL ), - patch.object(comm, "enable_moe_dense_fully_dp", return_value=False), - patch.object( - comm, "_generic_prefill_cp_shards_tokens", return_value=False + patch_communicator("enable_moe_dense_fully_dp", return_value=False), + patch_communicator( + "_generic_prefill_cp_shards_tokens", return_value=False ), ): modes = LayerScatterModes.init_new( @@ -184,14 +187,14 @@ class TestPostprocessReduceScatterv(CustomTestCase): communicator._cp_steps = None communicator._steps = steps(ffn_output_move=None) communicator._postprocess_dp_step = lambda forward_batch: step - with patch.object( - comm, "_to_local_tokens", side_effect=lambda s, fb, h: (s, h) + with patch_communicator( + "_to_local_tokens", side_effect=lambda s, fb, h: (s, h) ): hidden_states, residual = communicator.postprocess_layer( "h", "r", "fb" ) self.assertEqual( - hidden_states, (step or comm._redistribute_output, "h") + hidden_states, (step or comm_ops._redistribute_output, "h") ) self.assertEqual(residual, "r") @@ -205,11 +208,11 @@ class TestReduceAndRedistributeOutputStep(CustomTestCase): dp_padding_mode=types.SimpleNamespace(is_max_len=lambda: max_len) ) with ( - patch.object(comm, "should_use_dp_reduce_scatterv", return_value=varlen), - patch.object(comm, "can_use_dp_reduce_scatter", return_value=tiles), + patch_communicator("should_use_dp_reduce_scatterv", return_value=varlen), + patch_communicator("can_use_dp_reduce_scatter", return_value=tiles), ): # A MoE block leaves its sum to reduce_scatterv whenever it applies. - return comm._reduce_and_redistribute_output_step( + return comm_ops._reduce_and_redistribute_output_step( forward_batch, leaves_for_reduce_scatter=allow, leaves_for_reduce_scatterv=allow or sparse, @@ -220,9 +223,9 @@ class TestReduceAndRedistributeOutputStep(CustomTestCase): (False, True), repeat=5 ): if varlen and (allow or sparse): - expected = comm._reduce_and_redistribute_output_varlen + expected = comm_ops._reduce_and_redistribute_output_varlen elif allow and max_len and tiles: - expected = comm._reduce_and_redistribute_output_max_len + expected = comm_ops._reduce_and_redistribute_output_max_len else: expected = None case = dict( diff --git a/test/registered/unit/layers/test_prepare_attn_steps.py b/test/registered/unit/layers/test_prepare_attn_steps.py index 9272b62768..4b07af3145 100644 --- a/test/registered/unit/layers/test_prepare_attn_steps.py +++ b/test/registered/unit/layers/test_prepare_attn_steps.py @@ -5,13 +5,16 @@ import contextlib import unittest from functools import partial from types import SimpleNamespace -from unittest.mock import MagicMock, patch +from unittest.mock import MagicMock import msgspec import torch from sglang.srt.layers import communicator as comm +from sglang.srt.layers.communicator import boundary as comm_boundary +from sglang.srt.layers.communicator import ops as comm_ops from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.communicator_patch import patch_communicator from sglang.test.test_utils import CustomTestCase register_cpu_ci(est_time=10, suite="base-a-test-cpu") @@ -58,14 +61,14 @@ def communicator(norm): # A layer whose attention takes its input as it is and owes nothing on it. c._steps = comm.BoundarySteps( attention_prepare=partial( - comm._attention_input_step, + comm_boundary._attention_input_step, layer_input=None, fusions=c._attn_input_fusions, enters_stack=False, residual_ops=comm.ADD_AND_NORM, ), attention_input=lambda hidden_states, **_: hidden_states, - ffn_input=comm._mlp_input_norm, + ffn_input=comm_ops._mlp_input_norm, ffn_input_rows=comm.Layout(frozenset()), ffn_output=comm.StageOutput(comm.Layout(frozenset())), ffn_output_move=comm.CommunicateSummableTensorPairFn._trivial, @@ -85,29 +88,27 @@ def platform(*, use_aiter=False, gfx95=False, fusion=False, kernel_group=True): group = SimpleNamespace(all_reduce=all_reduce) with contextlib.ExitStack() as stack: for name, mock in kernels.items(): - stack.enter_context(patch.object(comm, name, mock, create=True)) - stack.enter_context(patch.object(comm, "_use_aiter", use_aiter)) - stack.enter_context(patch.object(comm, "_is_gfx95_supported", gfx95)) + stack.enter_context(patch_communicator(name, mock, create=True)) + stack.enter_context(patch_communicator("_use_aiter", use_aiter)) + stack.enter_context(patch_communicator("_is_gfx95_supported", gfx95)) stack.enter_context( - patch.object(comm, "_use_aiter_bpreshuffle_gfx95", False, create=True) + patch_communicator("_use_aiter_bpreshuffle_gfx95", False, create=True) ) stack.enter_context( - patch.object(comm, "apply_aiter_all_reduce_fusion", return_value=False) + patch_communicator("apply_aiter_all_reduce_fusion", return_value=False) ) stack.enter_context( - patch.object(comm, "apply_flashinfer_allreduce_fusion", return_value=fusion) + patch_communicator("apply_flashinfer_allreduce_fusion", return_value=fusion) ) # The group the fused kernel reduces over. stack.enter_context( - patch.object( - comm, + patch_communicator( "post_experts_reduction_group", return_value=group if kernel_group else SimpleNamespace(), ) ) stack.enter_context( - patch.object( - comm, + patch_communicator( "get_attn_tp_context", return_value=SimpleNamespace(input_scattered=False, is_dsa=False), ) @@ -240,7 +241,7 @@ class TestPrepareAttnSteps(CustomTestCase): c._steps = msgspec.structs.replace( c._steps, attention_prepare=partial( - comm._attention_input_step, + comm_boundary._attention_input_step, layer_input=None, fusions=(takes_anything,), enters_stack=False, diff --git a/test/registered/unit/lora/test_triton_dp_attention_unit.py b/test/registered/unit/lora/test_triton_dp_attention_unit.py index 6eed83c573..2bda7e75c2 100644 --- a/test/registered/unit/lora/test_triton_dp_attention_unit.py +++ b/test/registered/unit/lora/test_triton_dp_attention_unit.py @@ -9,12 +9,13 @@ from unittest.mock import patch import pytest import torch -from sglang.srt.layers.boundary_layout import Layout, TokenAxis from sglang.srt.layers.communicator import ( ADD_AND_NORM, LayerCommunicator, - _attention_input_step, + Layout, + TokenAxis, ) +from sglang.srt.layers.communicator.boundary import _attention_input_step from sglang.srt.layers.dp_attention import DpPaddingMode from sglang.srt.lora.backend.base_backend import BaseLoRABackend from sglang.srt.lora.backend.triton_backend import ( @@ -33,6 +34,7 @@ from sglang.srt.runtime_context import ( get_parallel, ) from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.communicator_patch import COMMUNICATOR_MODULES from sglang.test.test_utils import CustomTestCase register_cpu_ci(est_time=5, suite="base-a-test-cpu") @@ -209,10 +211,13 @@ def test_manager_rejects_dp_attention_with_multi_rank_attention_groups( def test_communicator_publishes_layout_at_each_transition( monkeypatch, gathered_over_dp, num_tokens, publish_lora_layout ): - monkeypatch.setattr( - "sglang.srt.layers.communicator.get_parallel", - lambda: SimpleNamespace(enable_dp_attention=publish_lora_layout), - ) + for module in COMMUNICATOR_MODULES: + if hasattr(module, "get_parallel"): + monkeypatch.setattr( + module, + "get_parallel", + lambda: SimpleNamespace(enable_dp_attention=publish_lora_layout), + ) # Start from TP_GLOBAL so an unpublished transition is distinguishable. initial = LoRABatchLayout.TP_GLOBAL expected_mlp = ( diff --git a/test/registered/unit/model_executor/test_num_token_non_padded_localization.py b/test/registered/unit/model_executor/test_num_token_non_padded_localization.py index 4a918b47c4..a51a5073f7 100644 --- a/test/registered/unit/model_executor/test_num_token_non_padded_localization.py +++ b/test/registered/unit/model_executor/test_num_token_non_padded_localization.py @@ -28,7 +28,6 @@ from unittest.mock import patch import torch -from sglang.srt.layers import communicator as comm from sglang.srt.layers.moe import utils as moe_utils from sglang.srt.model_executor.forward_batch_info import ( ForwardBatch, @@ -37,6 +36,7 @@ from sglang.srt.model_executor.forward_batch_info import ( compute_local_num_token_non_padded_cpu, ) from sglang.srt.runtime_context import get_parallel +from sglang.test.communicator_patch import patch_communicator from sglang.test.test_utils import CustomTestCase @@ -50,11 +50,11 @@ def sparse_moe_input(rows): patch.object( moe_utils, "is_moe_input_scattered_across_dp_ranks", return_value=local ), - patch.object( - comm, "is_moe_input_scattered_across_dp_ranks", return_value=local + patch_communicator( + "is_moe_input_scattered_across_dp_ranks", return_value=local ), - patch.object(comm, "is_enable_moe_cp_allgather", return_value=rows == "moe_cp"), - patch.object(comm, "_gathers_over_attention_cp", return_value=False), + patch_communicator("is_enable_moe_cp_allgather", return_value=rows == "moe_cp"), + patch_communicator("_gathers_over_attention_cp", return_value=False), ): yield @@ -196,7 +196,7 @@ class TestMoeNumTokenNonPaddedTable(CustomTestCase): self.subTest(case=label), get_parallel().override(attn_dp_size=1, attn_cp_size=2), sparse_moe_input("moe_cp"), - patch.object(comm, "get_moe_cp_size", return_value=2), + patch_communicator("get_moe_cp_size", return_value=2), patch( "sglang.srt.layers.attention.dsa.utils.dsa_use_prefill_cp", return_value=False, diff --git a/test/registered/unit/models/test_aux_capture_deferred_allreduce.py b/test/registered/unit/models/test_aux_capture_deferred_allreduce.py index b230861dfc..434e464816 100644 --- a/test/registered/unit/models/test_aux_capture_deferred_allreduce.py +++ b/test/registered/unit/models/test_aux_capture_deferred_allreduce.py @@ -7,6 +7,8 @@ import torch from torch import nn from sglang.srt.layers import communicator as comm +from sglang.srt.layers.communicator import boundary as comm_boundary +from sglang.srt.layers.communicator import ops as comm_ops from sglang.srt.model_executor.forward_batch_info import ForwardMode from sglang.srt.models.bailing_moe import BailingMoEModel from sglang.srt.models.bailing_moe_v3 import BailingMoELinearModel @@ -49,14 +51,14 @@ class DeferringLayer(nn.Module): self.layer_communicator = comm.LayerCommunicator.__new__(comm.LayerCommunicator) self.layer_communicator._steps = comm.BoundarySteps( attention_prepare=partial( - comm._attention_input_step, + comm_boundary._attention_input_step, layer_input=None, fusions=(), enters_stack=False, residual_ops=comm.ADD_AND_NORM, ), attention_input=comm.CommunicateSimpleFn._trivial, - ffn_input=comm._mlp_input_norm, + ffn_input=comm_ops._mlp_input_norm, ffn_input_rows=comm.Layout(frozenset()), ffn_output=comm.StageOutput( comm.Layout(frozenset()), diff --git a/test/registered/unit/models/test_last_layer_communicator.py b/test/registered/unit/models/test_last_layer_communicator.py index 89e6cdd117..de93b53d35 100644 --- a/test/registered/unit/models/test_last_layer_communicator.py +++ b/test/registered/unit/models/test_last_layer_communicator.py @@ -8,6 +8,7 @@ from unittest.mock import MagicMock, patch import torch from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.communicator_patch import patch_communicator from sglang.test.test_utils import CustomTestCase register_cpu_ci(est_time=15, suite="base-a-test-cpu") @@ -272,10 +273,10 @@ class TestLayerScatterModesLastLayer(CustomTestCase): from sglang.srt.layers import communicator as comm with ( - patch.object(comm, "enable_moe_dense_fully_dp", return_value=False), - patch.object(comm, "_generic_prefill_cp_shards_tokens", return_value=False), - patch.object(comm, "is_dsa_enable_prefill_cp", return_value=False), - patch.object(comm, "is_mla_cp_enabled", return_value=False), + patch_communicator("enable_moe_dense_fully_dp", return_value=False), + patch_communicator("_generic_prefill_cp_shards_tokens", return_value=False), + patch_communicator("is_dsa_enable_prefill_cp", return_value=False), + patch_communicator("is_mla_cp_enabled", return_value=False), ): for num_layers, layer_id in ( (NUM_LAYERS, 0), diff --git a/test/registered/unit/models/test_longcat_flash_shortcut.py b/test/registered/unit/models/test_longcat_flash_shortcut.py index 207a2bf656..33f491a7c3 100644 --- a/test/registered/unit/models/test_longcat_flash_shortcut.py +++ b/test/registered/unit/models/test_longcat_flash_shortcut.py @@ -1,6 +1,6 @@ import unittest from types import SimpleNamespace -from unittest.mock import Mock, patch +from unittest.mock import Mock import torch from torch import nn @@ -8,6 +8,7 @@ from torch import nn from sglang.srt.layers import communicator as comm from sglang.srt.models.longcat_flash import LongcatFlashDecoderLayer from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.communicator_patch import patch_communicator from sglang.test.test_utils import CustomTestCase register_cpu_ci(est_time=3, suite="base-a-test-cpu") @@ -51,18 +52,15 @@ class TestLongcatShortcut(CustomTestCase): ) ) with ( - patch.object( - comm, + patch_communicator( "get_local_dp_buffer", side_effect=lambda group, hidden_size=None: torch.empty(rows, 3), ), - patch.object( - comm, + patch_communicator( "attn_tp_all_gather_into_tensor", side_effect=lambda out, x: out.copy_(x.repeat(tp, 1)), ), - patch.object( - comm, + patch_communicator( "get_parallel", return_value=SimpleNamespace(attn_tp_group=object()), ), diff --git a/test/registered/unit/models/test_nemotron_h_aux_capture.py b/test/registered/unit/models/test_nemotron_h_aux_capture.py index 7f8550fa38..e99f74ccce 100644 --- a/test/registered/unit/models/test_nemotron_h_aux_capture.py +++ b/test/registered/unit/models/test_nemotron_h_aux_capture.py @@ -1,16 +1,15 @@ import unittest from types import SimpleNamespace -from unittest.mock import patch import torch from torch import nn -from sglang.srt.layers import communicator as comm from sglang.srt.layers.moe.utils import should_skip_mlp_all_reduce from sglang.srt.model_executor.forward_batch_info import ForwardMode from sglang.srt.models import nemotron_h as model from sglang.srt.runtime_context import get_context, get_flags, get_parallel from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.communicator_patch import patch_communicator from sglang.test.test_utils import CustomTestCase register_cpu_ci(est_time=5, suite="base-a-test-cpu") @@ -121,9 +120,9 @@ class TestNemotronAuxCapture(CustomTestCase): moe_dp_rank=0, moe_dp_size=1, ), - patch.object(comm, "get_moe_cp_size", return_value=1), - patch.object(comm, "apply_flashinfer_allreduce_fusion", return_value=False), - patch.object(comm, "apply_aiter_all_reduce_fusion", return_value=False), + patch_communicator("get_moe_cp_size", return_value=1), + patch_communicator("apply_flashinfer_allreduce_fusion", return_value=False), + patch_communicator("apply_aiter_all_reduce_fusion", return_value=False), ): baseline = _build(pattern, tp, False)( batch.input_ids, torch.arange(2), batch, inputs_embeds=inputs.clone() diff --git a/test/registered/unit/models/test_nemotron_h_mtp_reduction.py b/test/registered/unit/models/test_nemotron_h_mtp_reduction.py index f6d6395d80..720bf76ad2 100644 --- a/test/registered/unit/models/test_nemotron_h_mtp_reduction.py +++ b/test/registered/unit/models/test_nemotron_h_mtp_reduction.py @@ -1,15 +1,15 @@ import unittest from types import SimpleNamespace -from unittest.mock import Mock, patch +from unittest.mock import Mock import torch from torch import nn -from sglang.srt.layers import communicator as comm from sglang.srt.model_executor.forward_batch_info import ForwardMode from sglang.srt.models import nemotron_h_mtp from sglang.srt.runtime_context import get_context, get_flags, get_parallel from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.communicator_patch import patch_communicator from sglang.test.test_utils import CustomTestCase register_cpu_ci(est_time=3, suite="base-a-test-cpu") @@ -53,12 +53,12 @@ class TestNemotronMTPReduction(CustomTestCase): moe_dp_rank=0, moe_dp_size=1, ), - patch.object(comm, "get_moe_cp_size", return_value=1), - patch.object( - comm, "apply_flashinfer_allreduce_fusion", return_value=False + patch_communicator("get_moe_cp_size", return_value=1), + patch_communicator( + "apply_flashinfer_allreduce_fusion", return_value=False ), - patch.object( - comm, "apply_aiter_all_reduce_fusion", return_value=False + patch_communicator( + "apply_aiter_all_reduce_fusion", return_value=False ), ): layer = nemotron_h_mtp.NemotronHMTPMoEDecoderLayer.__new__( diff --git a/test/registered/unit/models/test_nemotron_h_stages.py b/test/registered/unit/models/test_nemotron_h_stages.py index 2b8096efb7..65824661f3 100644 --- a/test/registered/unit/models/test_nemotron_h_stages.py +++ b/test/registered/unit/models/test_nemotron_h_stages.py @@ -5,16 +5,17 @@ from unittest.mock import MagicMock, patch import torch -from sglang.srt.layers.boundary_layout import ( +from sglang.srt.layers.communicator import ( Layout, + MixerExit, StageDecl, StageInput, StageOutput, SumGroup, TokenAxis, + UnreducedOutput, stage_edges, ) -from sglang.srt.layers.communicator import MixerExit, UnreducedOutput from sglang.srt.layers.moe.utils import should_skip_mlp_all_reduce from sglang.srt.models import nemotron_h_utils as utils from sglang.srt.runtime_context import get_parallel