diff --git a/docs/docs/developer_guide/layer_boundary.mdx b/docs/docs/developer_guide/layer_boundary.mdx index 2969a9c5f7..94643cece3 100644 --- a/docs/docs/developer_guide/layer_boundary.mdx +++ b/docs/docs/developer_guide/layer_boundary.mdx @@ -178,6 +178,8 @@ Compute never sums a stage's output: build every output projection, MLP and MoE - `FfnExit` decides before compute runs whether the exit completes the sum, a reduce-scatter on the way back completes it, or the next input does. It publishes only `defer_moe_finalize` on `get_forward()`: a MoE may then return a `DeferredFinalize`. - `MixerExit`: before an FFN, the mixer always leaves its attention-TP sum to the FFN's input, including under attention DP. Before another mixer the exit may carry the sum to that mixer's input, but never under attention DP; otherwise it completes the sum itself. +One exception: when an FFN's output sum is fused with a reduction its computation needs anyway, so the two cannot be separated without an extra collective, the computation completes it and the stage declares `declare_ffn(output_complete=True)`; the exit then owes no sum. Kimi K3's latent MoE reduces its latent before the latent norm and completes the shared experts' sum beside it. + A component every TP rank computes in full (a shared expert replicated with `tp_size=1`, LoRA-B on a partial sum) is added on TP rank 0 only, so that the one sum counts it once. Model loading fails if a module under a stage boundary is built with `reduce_results=True`. Returning a handoff while `defer_moe_finalize` is false fails in `finish()`. A complete tensor after `finish()` can still have a pending **residual update**. An incomplete reduction is exposed to the model as an opaque `OwedOutput`; pass it to the next boundary or a supported access method. Do not inspect it as a tensor, slice it, or unwrap its contribution in model code. diff --git a/python/sglang/srt/arg_groups/boundary_reduction.py b/python/sglang/srt/arg_groups/boundary_reduction.py index 21ffc570c5..b6f26bc780 100644 --- a/python/sglang/srt/arg_groups/boundary_reduction.py +++ b/python/sglang/srt/arg_groups/boundary_reduction.py @@ -12,6 +12,8 @@ _RSV_ARCHITECTURES = { "Step3VLForConditionalGeneration", } _AR_ARCHITECTURES = { + "KimiK3ForConditionalGeneration", + "KimiK3LinearForCausalLM", "Qwen3ForCausalLM", "Qwen3ForSequenceClassification", "Qwen3Model", diff --git a/python/sglang/srt/arg_groups/parallel_hook.py b/python/sglang/srt/arg_groups/parallel_hook.py index 563112688e..c081a28ba6 100644 --- a/python/sglang/srt/arg_groups/parallel_hook.py +++ b/python/sglang/srt/arg_groups/parallel_hook.py @@ -61,22 +61,25 @@ def handle_deprecated_dp_attention(server_args: Any): ) -def _boundary_parallelism_overrides(cfg, model_type: str) -> dict: +def _boundary_parallelism_overrides(cfg, model_type: str, architecture: str) -> dict: """Resolve the token-row modes the model's forward actually enters.""" nemotron = model_type in ("nemotron_h", "nemotron_h_puzzle") longcat = model_type == "longcat_flash" + kimi_k3 = architecture in ( + "KimiK3ForConditionalGeneration", + "KimiK3LinearForCausalLM", + ) if nemotron and cfg.attn_cp_size > 1: raise ValueError("Nemotron-H does not support --attn-cp-size > 1") if longcat and cfg.enable_prefill_cp and cfg.attn_cp_size > 1: raise ValueError( "LongCat-Flash does not support --enable-prefill-cp with --attn-cp-size > 1" ) - if (nemotron or longcat) and cfg.enable_attn_tp_input_scattered: - # Neither model enters the input-scattered attention scope. Preserve - # their existing ordinary execution and make the effective flag explicit. + if (nemotron or longcat or kimi_k3) and cfg.enable_attn_tp_input_scattered: + # None of these models enters the input-scattered attention scope. logger.warning( "Disabling input-scattered attention for %s: its forward does not enter that scope", - model_type, + architecture, ) return {"enable_attn_tp_input_scattered": False} return {} @@ -99,7 +102,7 @@ def handle_context_parallelism(server_args: Any): server_args, "boundary_parallelism", **_boundary_parallelism_overrides( - cfg, model_config.hf_text_config.model_type + cfg, model_config.hf_text_config.model_type, model_arch ), ) if ( diff --git a/python/sglang/srt/layers/layer_boundary/factories.py b/python/sglang/srt/layers/layer_boundary/factories.py index 1a9045bd7b..ae8add4bf8 100644 --- a/python/sglang/srt/layers/layer_boundary/factories.py +++ b/python/sglang/srt/layers/layer_boundary/factories.py @@ -96,6 +96,7 @@ def _resolve_ffn( read=NORM_READOUT, update=PLAIN_ADD, dense_tp_size=None, + output_complete=False, ): parallel = get_parallel() if dense_tp_size not in (None, 1, parallel.tp_size): @@ -167,7 +168,7 @@ def _resolve_ffn( ) ) # An FFN that writes the next stream itself hands on a complete output. - complete = on_rank_rows or update is REPLACE_AT_EXIT + complete = on_rank_rows or update is REPLACE_AT_EXIT or output_complete produced = ( OutputContract(rows, update=update, transform=output_transform) if complete @@ -222,6 +223,9 @@ class StageDeclaration: exit_rows: Required FFN output rows at the layer or branch exit. writes_at_handoff: Whether this FFN writes its output into the residual at its exit because the next stage is on another pipeline rank. + output_complete: Whether the FFN's compute completes its own output + sum, so the exit owes none. For an FFN whose output sum is fused + with a reduction its computation needs anyway. previous: Declaration whose output this stage consumes, as the stack records it; across pipeline ranks it is built locally. prepared_from: Declaration whose already-read input a branch reuses. @@ -242,6 +246,7 @@ class StageDeclaration: dense_tp_size: Optional[int] = None exit_rows: Optional[ExitRows] = None writes_at_handoff: bool = False + output_complete: bool = False # Only declarations participate in construction, never executable stages. previous: Optional[StageDeclaration] = None prepared_from: Optional[StageDeclaration] = None @@ -329,6 +334,7 @@ def declare_ffn( next_layer_sparse=False, dense_tp_size=None, exit_rows=None, + output_complete=False, ): """Declare a dense or MoE FFN independently of its compute module. @@ -343,6 +349,8 @@ def declare_ffn( compute, or the full TP size. exit_rows: Explicit output-row requirement; otherwise derived from the adjacent FFN kinds and TBO configuration. + output_complete: Whether the compute completes its own output sum, + fused with a reduction it needs anyway; the exit then owes none. Returns: A StageDeclaration with no norm, tensors or execution plan. @@ -355,6 +363,7 @@ def declare_ffn( output_transform=output_transform, dense_tp_size=dense_tp_size, exit_rows=exit_rows or tbo_exit_rows(sparse, next_layer_sparse), + output_complete=output_complete, ) @@ -378,6 +387,7 @@ def _resolve_stage(stage, variant, following=None): read=stage.read, update=stage.update, dense_tp_size=stage.dense_tp_size, + output_complete=stage.output_complete, ) if resolve_exit_rows(stage.exit_rows) is ExitRows.ATTENTION: returned = attention diff --git a/python/sglang/srt/layers/layer_boundary/prepare.py b/python/sglang/srt/layers/layer_boundary/prepare.py index 605df1bea4..983949a187 100644 --- a/python/sglang/srt/layers/layer_boundary/prepare.py +++ b/python/sglang/srt/layers/layer_boundary/prepare.py @@ -343,7 +343,7 @@ def _update_read( def _attn_tp_reduce_scatter_update_read( hidden_states: torch.Tensor, - residual: torch.Tensor, + residual: Optional[torch.Tensor], forward_batch: ForwardBatch, norm: torch.nn.Module, *, @@ -353,7 +353,7 @@ def _attn_tp_reduce_scatter_update_read( update: ResidualUpdate = PLAIN_ADD, ): hidden_states = attn_tp_reduce_scatter(hidden_states) - if scatters_residual: + if scatters_residual and residual is not None: residual = update.slice_residual_attn_tp(residual) return read.update_and_read(update, hidden_states, residual, norm) diff --git a/python/sglang/srt/layers/layer_boundary/residual/attn_bank.py b/python/sglang/srt/layers/layer_boundary/residual/attn_bank.py new file mode 100644 index 0000000000..f88f6ced43 --- /dev/null +++ b/python/sglang/srt/layers/layer_boundary/residual/attn_bank.py @@ -0,0 +1,223 @@ +# Copyright 2023-2024 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""A residual read that aggregates a bank of snapshots of itself.""" + +from dataclasses import dataclass +from typing import Optional + +import torch + +from sglang.srt.layers.attn_residual import AttnResidual +from sglang.srt.layers.layer_boundary.residual import LayerResidualOps +from sglang.srt.layers.layer_boundary.residual.add_norm import PLAIN_ADD +from sglang.srt.runtime_context import get_parallel + + +class AttnBank: + """The snapshot bank of one layer-stack invocation. + + Every layer's reads aggregate the same bank, so the bank outlives a layer + while the reads that consume it are per-layer. The stack's entry opens one + and the layers reach it through this holder; a pipeline rank that did not + open the stack inherits the rows the previous rank banked. + """ + + __slots__ = ("current",) + + def __init__(self): + self.current = None + + def open(self, hidden_states, block_num, inherited=None) -> AttnResidual: + self.current = AttnResidual(hidden_states, block_num, inherited) + return self.current + + def close(self) -> None: + self.current = None + + def require(self) -> AttnResidual: + if self.current is None: + raise RuntimeError("open the attention-residual bank before a stage") + return self.current + + +def _bank_rows(bank: AttnResidual, hidden_states) -> Optional[slice]: + """The bank rows that go with ``hidden_states``: all of them, or this + rank's contiguous attention-TP shard of them, the rows a reduce-scatter + over attention TP leaves it.""" + total, rows = bank.block_residual.shape[0], hidden_states.shape[0] + if rows == total: + return None + parallel = get_parallel() + if rows * parallel.attn_tp_size != total: + raise RuntimeError( + f"{rows} rows are neither the bank's {total} rows nor an " + f"attention-TP shard of them over {parallel.attn_tp_size} ranks" + ) + rank = parallel.attn_tp_rank + return slice(rank * rows, (rank + 1) * rows) + + +@dataclass +class AttnBankState: + """One layer's pair of reads into the bank an `AttnBank` holds. + + A read aggregates the banked snapshots with the running residual and + normalizes the result; the update that precedes it is an ordinary add, so + both stages declare `PLAIN_ADD` and each read folds the pending add into + its aggregation kernel. A read on this rank's attention-TP shard of the + rows aggregates the matching shard of the bank. A write layer also + snapshots the residual it just formed into the bank's next row, where it + now lives: the running residual restarts empty, and the next contribution + alone is the new head.""" + + bank: AttnBank + attn_score_proj: torch.nn.Module + attn_score_norm: torch.nn.Module + ffn_score_proj: torch.nn.Module + ffn_score_norm: torch.nn.Module + writes_block: bool = False + + def _aggregate( + self, contribution, residual, norm, *, score_proj, score_norm, write + ): + bank = self.bank.require() + return bank.forward( + contribution, + residual, + score_proj, + score_norm, + norm, + rows=_bank_rows(bank, contribution), + write=write, + ) + + def read_attn_input(self, residual, norm): + # No pending add: the residual alone is the head (the stack's input, + # or the residual of a stream that was written). + return self._attn_aggregate(residual, None, norm) + + def update_and_read_attn_input(self, contribution, residual, norm): + return self._attn_aggregate(contribution, residual, norm) + + def _attn_aggregate(self, contribution, residual, norm): + normed, residual = self._aggregate( + contribution, + residual, + norm, + score_proj=self.attn_score_proj, + score_norm=self.attn_score_norm, + write=self.writes_block, + ) + return normed, None if self.writes_block else residual + + def update_and_read_ffn_input(self, contribution, residual, norm): + return self._aggregate( + contribution, + residual, + norm, + score_proj=self.ffn_score_proj, + score_norm=self.ffn_score_norm, + write=False, + ) + + def residual_ops(self) -> LayerResidualOps: + return LayerResidualOps( + attn_readout=_AttnReadout(self), + attn_update=PLAIN_ADD, + ffn_readout=_FfnReadout(self), + ffn_update=PLAIN_ADD, + ) + + +class _BankReadout: + """A read that mixes the banked snapshots into the residual's norm, so it + is not the residual's plain norm and no add+norm kernel may claim it. It + reads this rank's own rows, which the bank holds, before any DP gather.""" + + is_plain_norm = False + reads_before_dp_gather = True + + def __init__(self, state: AttnBankState): + self.state = state + + def init_residual(self, hidden_states): + return hidden_states + + def _reject(self, update, quant_format): + if quant_format: + raise NotImplementedError( + f"an attention-residual bank read in {quant_format=}" + ) + if update is not None and not update.is_plain_add: + # The read folds the pending add into its aggregation, which only + # reproduces an ordinary add. + raise NotImplementedError( + f"an attention-residual bank read after {update=}" + ) + + +class _AttnReadout(_BankReadout): + """The attention input: the bank aggregation and this layer's input norm. + A write layer snapshots the residual this read forms.""" + + def read(self, residual, norm, quant_format="", post_residual_addition=None): + self._reject(None, quant_format) + if post_residual_addition is not None: + raise NotImplementedError( + "an attention-residual bank read with a separate addition" + ) + return self.state.read_attn_input(residual, norm) + + def update_and_read(self, update, hidden_states, residual, norm, **kwargs): + self._reject(update, kwargs.get("quant_format", "")) + return self.state.update_and_read_attn_input(hidden_states, residual, norm) + + +class _FfnReadout(_BankReadout): + """The FFN input: the attention output's add folded into the bank + aggregation, and this layer's post-attention norm.""" + + def read(self, residual, norm, quant_format="", post_residual_addition=None): + # The attention read of a write layer leaves the residual empty, so the + # attention output alone is the head. + self._reject(None, quant_format) + return self.state.update_and_read_ffn_input(residual, None, norm) + + def update_and_read(self, update, hidden_states, residual, norm, **kwargs): + self._reject(update, kwargs.get("quant_format", "")) + return self.state.update_and_read_ffn_input(hidden_states, residual, norm) + + +class AttnBankOutputRead: + """The layer stack's terminal read: the bank aggregation with the output + side's scoring parameters, then the final norm. A callable final norm for + `residual_batch.final_norm`.""" + + def __init__(self, bank: AttnBank, score_proj, score_norm, norm): + self.bank = bank + self.score_proj = score_proj + self.score_norm = score_norm + self.norm = norm + + def __call__(self, hidden_states, residual=None): + bank = self.bank.require() + normed, head = bank.forward( + hidden_states, + residual, + self.score_proj, + self.score_norm, + self.norm, + rows=_bank_rows(bank, hidden_states), + ) + return normed if residual is None else (normed, head) diff --git a/python/sglang/srt/layers/layer_boundary/residual/batch.py b/python/sglang/srt/layers/layer_boundary/residual/batch.py index 1d9b31a59d..915cc1aaa7 100644 --- a/python/sglang/srt/layers/layer_boundary/residual/batch.py +++ b/python/sglang/srt/layers/layer_boundary/residual/batch.py @@ -106,6 +106,8 @@ def final_norm( ) return hidden_states if skip_empty and hidden_states.shape[0] == 0: + if capture is not None: + capture(hidden_states) return hidden_states return access.final_norm_pair( hidden_states, residual, layernorm, capture, **read_kwargs diff --git a/python/sglang/srt/managers/scheduler_components/dynamic_chunk_sizer.py b/python/sglang/srt/managers/scheduler_components/dynamic_chunk_sizer.py index 93b67c4b9a..c6ce743e0c 100644 --- a/python/sglang/srt/managers/scheduler_components/dynamic_chunk_sizer.py +++ b/python/sglang/srt/managers/scheduler_components/dynamic_chunk_sizer.py @@ -131,6 +131,10 @@ class DynamicChunkSizer: return predicted_size def _profile_prefill_latency(self) -> Tuple[List[int], List[float]]: + from sglang.srt.model_executor.runner_utils.buffers import ( + allocate_pp_proxy_tensors, + ) + seq_lens: List[int] = [] latencies: List[float] = [] model_runner = self.model_runner @@ -193,28 +197,18 @@ class DynamicChunkSizer: batch.global_num_tokens = global_num_tokens batch.global_num_tokens_for_logprob = global_num_tokens - hs = ( - getattr(model_config, "hc_hidden_size", None) - or model_config.hidden_size - ) - proxy_tensors = { - "hidden_states": torch.zeros( - (current_seq_len, hs), + # Keyed and shaped like the runner's PP buffers. + with torch.device(self.device): + proxy_tensors = allocate_pp_proxy_tensors( + max_num_tokens=current_seq_len, + max_hidden_tokens=current_seq_len, + hidden_size=model_config.hidden_size, dtype=model_config.dtype, - device=self.device, - ), - "residual": torch.zeros( - (current_seq_len, model_config.hidden_size), - dtype=model_config.dtype, - device=self.device, - ), - } - pp_proxy_topk_size = model_runner.get_pp_proxy_topk_size() - if pp_proxy_topk_size is not None: - proxy_tensors["topk_indices"] = torch.zeros( - (current_seq_len, pp_proxy_topk_size), - dtype=torch.int32, - device=self.device, + hc_hidden_size=getattr(model_config, "hc_hidden_size", None), + pp_proxy_topk_size=model_runner.get_pp_proxy_topk_size(), + pp_proxy_residual_num_blocks=( + model_runner.get_pp_proxy_residual_num_blocks() + ), ) pp_proxy = PPProxyTensors(proxy_tensors) diff --git a/python/sglang/srt/model_executor/runner/base_runner.py b/python/sglang/srt/model_executor/runner/base_runner.py index e0fcc11148..d613bb0346 100644 --- a/python/sglang/srt/model_executor/runner/base_runner.py +++ b/python/sglang/srt/model_executor/runner/base_runner.py @@ -138,15 +138,17 @@ def _allocate_decode_buffers( pp_proxy_tensors = { "hidden_states": torch.zeros((max_num_token, hs), dtype=dtype), } - if not is_mhc: - # Only Kimi K3 supplies num_blocks: its PP bank is token-major - # [T, blocks, H]. Other models use [T, H]. - residual_shape = ( - (max_num_token, pp_proxy_residual_num_blocks, hidden_size) - if pp_proxy_residual_num_blocks is not None - else (max_num_token, hidden_size) + if pp_proxy_residual_num_blocks is not None: + # Only Kimi K3 supplies num_blocks: its attention-residual bank + # is token-major [T, blocks, H] and takes the residual's place. + pp_proxy_tensors["attn_res_bank"] = torch.zeros( + (max_num_token, pp_proxy_residual_num_blocks, hidden_size), + dtype=dtype, + ) + elif not is_mhc: + pp_proxy_tensors["residual"] = torch.zeros( + (max_num_token, hidden_size), dtype=dtype ) - pp_proxy_tensors["residual"] = torch.zeros(residual_shape, dtype=dtype) if pp_proxy_topk_size is not None: pp_proxy_tensors["topk_indices"] = torch.zeros( (max_num_token, pp_proxy_topk_size), dtype=torch.int32 diff --git a/python/sglang/srt/model_executor/runner_utils/buffers.py b/python/sglang/srt/model_executor/runner_utils/buffers.py index d9157905af..93d6a3b685 100644 --- a/python/sglang/srt/model_executor/runner_utils/buffers.py +++ b/python/sglang/srt/model_executor/runner_utils/buffers.py @@ -64,7 +64,7 @@ def _grouped_foreach_copy_(dsts: List[torch.Tensor], srcs: List[torch.Tensor]) - foreach_copy(group_dsts, group_srcs) -def _allocate_pp_proxy_tensors( +def allocate_pp_proxy_tensors( *, max_num_tokens: int, max_hidden_tokens: int, @@ -81,15 +81,17 @@ def _allocate_pp_proxy_tensors( pp_proxy_tensors = { "hidden_states": torch.zeros((max_hidden_tokens, pp_hidden_size), dtype=dtype), } - if not is_mhc: - # Only Kimi K3 supplies num_blocks: its PP bank is token-major - # [T, blocks, H]. Other models use the phase-specific hidden-token bound. - residual_shape = ( - (max_num_tokens, pp_proxy_residual_num_blocks, hidden_size) - if pp_proxy_residual_num_blocks is not None - else (max_hidden_tokens, hidden_size) + if pp_proxy_residual_num_blocks is not None: + # Only Kimi K3 supplies num_blocks: its attention-residual bank is + # token-major [T, blocks, H] and takes the residual's place. + pp_proxy_tensors["attn_res_bank"] = torch.zeros( + (max_num_tokens, pp_proxy_residual_num_blocks, hidden_size), dtype=dtype + ) + elif not is_mhc: + # Sized by the phase-specific hidden-token bound, like hidden_states. + pp_proxy_tensors["residual"] = torch.zeros( + (max_hidden_tokens, hidden_size), dtype=dtype ) - pp_proxy_tensors["residual"] = torch.zeros(residual_shape, dtype=dtype) if pp_proxy_topk_size is not None: pp_proxy_tensors["topk_indices"] = torch.zeros( (max_num_tokens, pp_proxy_topk_size), dtype=torch.int32 @@ -185,7 +187,7 @@ class DecodeInputBuffers(ForwardInputBuffers): ) pp_proxy_tensors = ( - _allocate_pp_proxy_tensors( + allocate_pp_proxy_tensors( max_num_tokens=max_num_token, max_hidden_tokens=max_num_token, hidden_size=hidden_size, @@ -333,7 +335,7 @@ class PrefillInputBuffers(ForwardInputBuffers): mrope_positions = None pp_proxy_tensors = ( - _allocate_pp_proxy_tensors( + allocate_pp_proxy_tensors( max_num_tokens=max_num_tokens, max_hidden_tokens=max_num_tokens, hidden_size=hidden_size, diff --git a/python/sglang/srt/models/kimi_k3.py b/python/sglang/srt/models/kimi_k3.py index 770687205e..b1c83dac49 100644 --- a/python/sglang/srt/models/kimi_k3.py +++ b/python/sglang/srt/models/kimi_k3.py @@ -28,6 +28,7 @@ from sglang.srt.distributed import ( from sglang.srt.distributed.device_communicators.pynccl_allocator import ( use_symmetric_memory, ) +from sglang.srt.distributed.utils import get_group_rank_size from sglang.srt.environ import envs from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder from sglang.srt.layers import zero_copy_context @@ -35,6 +36,7 @@ from sglang.srt.layers.activation import SiluAndMul, SituAndMul from sglang.srt.layers.attn_residual import AttnResidual, aggregate_stream, get_cw from sglang.srt.layers.aux_hidden_states import ( AuxHiddenStateAccumulator, + AuxHiddenStateList, AuxHiddenStatePacker, ) from sglang.srt.layers.communication import k3_ar_fusion, k3_sp_collective @@ -47,6 +49,22 @@ from sglang.srt.layers.dp_attention import ( is_allocation_symmetric, is_dp_attention_enabled, ) +from sglang.srt.layers.layer_boundary import ( + ExitRows, + append_stages, + declare_attn, + declare_ffn, +) +from sglang.srt.layers.layer_boundary.residual import batch as residual_batch +from sglang.srt.layers.layer_boundary.residual.add_norm import ( + REPLACE_AT_EXIT, + NormReadout, +) +from sglang.srt.layers.layer_boundary.residual.attn_bank import ( + AttnBank, + AttnBankOutputRead, + AttnBankState, +) from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import ( ColumnParallelBatchedLinear, @@ -429,8 +447,13 @@ class KimiK3MoE(nn.Module): prefix: str = "", layer_idx: int = 0, alt_stream: Optional[torch.cuda.Stream] = None, + reduce_results: bool = True, ): super().__init__() + # Whether this block completes the TP sum of a non-latent output; a + # stage boundary completes it otherwise. A latent block completes its + # sums together with the latent reduction either way. + self.reduce_results = reduce_results hidden_size = config.hidden_size moe_intermediate_size = config.moe_intermediate_size moe_renormalize = config.moe_renormalize @@ -993,6 +1016,15 @@ class KimiK3MoE(nn.Module): value.record_stream(current_stream) return topk_output, routed_input + def _reduce_shared(self, shared_output: torch.Tensor) -> torch.Tensor: + """The TP-sharded shared experts' sum, completed beside a routed + output the latent reduction already completed.""" + return tensor_model_parallel_all_reduce(shared_output) + + def _reduce_latent_and_shared(self, buf: torch.Tensor) -> torch.Tensor: + """The fused front's [latent | shared] pair in one collective.""" + return tensor_model_parallel_all_reduce(buf) + def _reduce_latent(self, latent: torch.Tensor) -> torch.Tensor: """Unfused-front latent tail: TP-partial routed sums must be reduced in latent space BEFORE the RMSNorm (sum(norm(x_i)) != norm(sum(x_i))).""" @@ -1193,7 +1225,7 @@ class KimiK3MoE(nn.Module): expert_output = expert_output + shared_output # EP combine and the shared-expert subgroup have already completed # each source token. A global TP reduction would mix token shards. - if self.tp_size > 1 and not self._ep_a2a: + if self.tp_size > 1 and not self._ep_a2a and self.reduce_results: expert_output = tensor_model_parallel_all_reduce(expert_output) if prefix_sum is not None: expert_output = expert_output + prefix_sum @@ -1234,7 +1266,7 @@ class KimiK3MoE(nn.Module): and not self._shared_experts_tp1 and not self._shared_experts_tp_comm ): - shared_output = tensor_model_parallel_all_reduce(shared_output) + shared_output = self._reduce_shared(shared_output) out = _add3(out, shared_output, prefix_sum) return out out = out if prefix_sum is None else out + prefix_sum @@ -1443,7 +1475,7 @@ class KimiK3MoE(nn.Module): elif k3_ar_fusion.enabled(): k3_ar_fusion.all_reduce(buf) else: - buf = tensor_model_parallel_all_reduce(buf) + buf = self._reduce_latent_and_shared(buf) latent = buf[:latent_numel].view(num_tokens, self.moe_hidden_size) shared_output = buf[latent_numel:].view(num_tokens, hidden_size) @@ -1510,6 +1542,7 @@ class KimiK3DeltaAttention(nn.Module): prefix: str = "", all_reduce_fusion: bool = False, bfa_alt_stream: Optional[torch.cuda.Stream] = None, + reduce_results: bool = True, **kwargs, ) -> None: super().__init__() @@ -1766,7 +1799,7 @@ class KimiK3DeltaAttention(nn.Module): # all-reduce (which can fold the attn-res prefix add in). Only # valid when the attn TP group is the full TP group (the fused # comm lives there). - reduce_results=not self.all_reduce_fusion, + reduce_results=reduce_results and not self.all_reduce_fusion, quant_config=quant_config, parallel_group="attn_tp", # Reduce within the attn-TP group: the default full-TP collective @@ -2163,6 +2196,7 @@ class KimiK3MLAAttention(DeepseekV2AttentionMLA): prefix: str = "", alt_stream: Optional[torch.cuda.Stream] = None, gate_alt_stream: Optional[torch.cuda.Stream] = None, + reduce_results: bool = True, ) -> None: # ModelSlim can quantize K3 latent projections while still storing # MLA kv_b_proj as one dense tensor; only GGUF expert packs split K/V. @@ -2185,7 +2219,7 @@ class KimiK3MLAAttention(DeepseekV2AttentionMLA): q_lora_rank=config.q_lora_rank, kv_lora_rank=config.kv_lora_rank, skip_rope=True, - reduce_results=not self.all_reduce_fusion, + reduce_results=reduce_results and not self.all_reduce_fusion, alt_stream=alt_stream, ) if split_gguf_kv_b: @@ -2372,6 +2406,61 @@ class KimiK3MLAAttention(DeepseekV2AttentionMLA): ) +def _is_moe_layer(config: KimiLinearConfig, layer_idx: int) -> bool: + """Whether a layer runs a MoE rather than the dense MLP: past the dense + prefix and on the MoE cadence.""" + return ( + config.is_moe + and config.num_experts is not None + and layer_idx >= config.first_k_dense_replace + and layer_idx % config.moe_layer_freq == 0 + ) + + +def _shards_moe_rows() -> bool: + """Whether an MoE layer runs on its attention-TP token shard (SP-MoE): an + EP a2a backend with attention TP.""" + backend = get_moe_a2a_backend() + return ( + backend.is_megamoe() + or backend.is_deepep() + or backend.is_mooncake() + or backend.is_ascend_fuseep() + or backend.is_mori() + ) and get_parallel().attn_tp_group.world_size > 1 + + +def _fuses_attn_all_reduce(config: KimiLinearConfig) -> bool: + """Whether the attention-residual bank asks for o_proj's all-reduce fused + with the pending residual add.""" + attn_tp_size = get_parallel().attn_tp_size + return ( + attn_tp_size > 1 + and attn_tp_size == get_parallel().tp_size + and config.attn_res_block_size is not None + and k3_ar_fusion.enabled() + ) + + +def _uses_stage_boundaries(config: KimiLinearConfig) -> bool: + """Whether the layers build stage boundaries, which is the same for every + layer of a stack. The layer's own communication still runs for: a dense + MLP sharded over attention TP; an attention-residual bank whose o_proj + all-reduce is fused with the pending add, or whose MoE on its attention-TP + token shard (SP-MoE) uses K3's tuned SP collectives, which the sharded + carry also needs; and SP-MoE on batches that are not padded to a multiple + of attention TP (--disable-attn-tp-gather).""" + if get_parallel().enable_dense_mlp_attn_tp and is_dp_attention_enabled(): + return False + if _fuses_attn_all_reduce(config): + return False + if not _shards_moe_rows(): + return True + if not require_mlp_sync(): + return False + return config.attn_res_block_size is None or not k3_sp_collective.enabled() + + class KimiK3DecoderLayer(nn.Module): """Decoder layer carrying the K3 attention-residual stream.""" @@ -2382,24 +2471,17 @@ class KimiK3DecoderLayer(nn.Module): quant_config: Optional[QuantizationConfig] = None, prefix: str = "", alt_streams: Optional[List[torch.cuda.Stream]] = None, + attn_bank: Optional[AttnBank] = None, ) -> None: super().__init__() self.hidden_size = config.hidden_size - self.is_moe = config.is_moe self.layer_idx = layer_idx self._dp_attention = is_dp_attention_enabled() # mlp-sync (DP attention OR MoE a2a/EP) pads extend batches to # attn_tp multiples; attention must then run on the real rows only. self._trim_padded_attn = require_mlp_sync() - # A layer runs MoE (vs a plain dense MLP) iff it is past the dense - # prefix and on the MoE cadence — same predicate the mlp construction - # below uses. - self._is_moe_layer = ( - self.is_moe - and config.num_experts is not None - and layer_idx >= config.first_k_dense_replace - and layer_idx % config.moe_layer_freq == 0 - ) + self._is_moe_layer = _is_moe_layer(config, layer_idx) + self._stage_boundaries = _uses_stage_boundaries(config) # SP-MoE (EP a2a backend): o_proj defers its attention-TP reduction; # this layer completes it as a reduce-scatter so the whole MoE region # runs on 1/attn_tp of the rows, then all-gathers back after the tail @@ -2407,30 +2489,12 @@ class KimiK3DecoderLayer(nn.Module): # disappears via tp1 weights, and each rank dispatches only its shard # through the a2a. Same under DP attention. Dense layers excluded: no # per-token decomposition survives a token shard. - _a2a_backend = get_moe_a2a_backend() - self._sp_moe = ( - ( - _a2a_backend.is_megamoe() - or _a2a_backend.is_deepep() - or _a2a_backend.is_mooncake() - or _a2a_backend.is_ascend_fuseep() - or _a2a_backend.is_mori() - ) - and self._is_moe_layer - and get_parallel().attn_tp_group.world_size > 1 - ) + self._sp_moe = self._is_moe_layer and _shards_moe_rows() # Mutually exclusive with SP-MoE: both complete o_proj's deferred # reduction, but SP-MoE reduce-scatters to a shard whereas the fusion # produces the full batch in a symm buffer. - attn_tp_size = get_parallel().attn_tp_size - self.all_reduce_fusion = ( - not self._sp_moe - and attn_tp_size > 1 - and attn_tp_size == get_parallel().tp_size - and config.attn_res_block_size is not None - and k3_ar_fusion.enabled() - ) + self.all_reduce_fusion = not self._sp_moe and _fuses_attn_all_reduce(config) # Attention if config.is_kda_layer(layer_idx): @@ -2441,6 +2505,7 @@ class KimiK3DecoderLayer(nn.Module): quant_config=quant_config, prefix=f"{prefix}.self_attn", all_reduce_fusion=self.all_reduce_fusion, + reduce_results=not self._stage_boundaries, # Shared with the MLA gate stream: KDA and MLA layers never # run concurrently within one forward, so the stream is free. bfa_alt_stream=(alt_streams[2] if alt_streams is not None else None), @@ -2452,6 +2517,7 @@ class KimiK3DecoderLayer(nn.Module): quant_config=quant_config, prefix=f"{prefix}.self_attn", all_reduce_fusion=self.all_reduce_fusion, + reduce_results=not self._stage_boundaries, alt_stream=alt_streams[1] if alt_streams is not None else None, gate_alt_stream=alt_streams[2] if alt_streams is not None else None, ) @@ -2468,6 +2534,7 @@ class KimiK3DecoderLayer(nn.Module): layer_idx=layer_idx, prefix=f"{prefix}.mlp", alt_stream=alt_streams[0] if alt_streams is not None else None, + reduce_results=not self._stage_boundaries, ) else: self.mlp = KimiK3MLP( @@ -2475,6 +2542,7 @@ class KimiK3DecoderLayer(nn.Module): intermediate_size=config.intermediate_size, hidden_act=config.hidden_act, quant_config=quant_config, + reduce_results=not self._stage_boundaries, prefix=f"{prefix}.mlp", activation_situ_beta=config.activation_situ_beta, activation_situ_linear_beta=config.activation_situ_linear_beta, @@ -2538,6 +2606,68 @@ class KimiK3DecoderLayer(nn.Module): o_proj.forward = _sp_o_proj_forward + # A latent MoE on this rank's rows adds the residual in its tail add and + # writes the next stream itself; one gathered over attention DP returns + # its output to this rank's tokens first. + self._ffn_writes_stream = ( + self._stage_boundaries + and self.use_attn_residuals + and self._is_moe_layer + and self.mlp.use_latent_moe + and not (self._dp_attention and not self.mlp._ep_a2a) + ) + if self._stage_boundaries: + # Under attention DP the FFN input is read on this rank's rows + # once the attention's sum is complete, then gathered. + attn_ops = {} + ffn_ops = dict(read=NormReadout(reads_before_dp_gather=True)) + if self.use_attn_residuals: + bank_ops = AttnBankState( + attn_bank, + self.self_attention_res_proj, + self.self_attention_res_norm, + self.mlp_res_proj, + self.mlp_res_norm, + writes_block=self.is_block_write_layer, + ).residual_ops() + attn_ops = dict(read=bank_ops.attn_readout, update=bank_ops.attn_update) + ffn_ops = dict( + read=bank_ops.ffn_readout, + update=( + REPLACE_AT_EXIT + if self._ffn_writes_stream + else bank_ops.ffn_update + ), + ) + self.attn_boundary, self.ffn_boundary = append_stages( + (declare_attn(**attn_ops), self.input_layernorm), + ( + declare_ffn( + **ffn_ops, + sparse=self._is_moe_layer, + next_layer_sparse=_is_moe_layer(config, layer_idx + 1), + # The TP width the dense MLP is built with. + dense_tp_size=( + None + if self._is_moe_layer + else get_group_rank_size(self.mlp.down_proj.tp_group)[1] + ), + # SP-MoE runs on this rank's attention-TP shard of the + # rows; on the bank path, whose reads write the bank on + # every row, its output returns to all of them. + exit_rows=( + ExitRows.ATTENTION + if self._sp_moe and self.use_attn_residuals + else None + ), + # A latent MoE completes its output sum together with + # the latent reduction its norm needs. + output_complete=self._is_moe_layer and self.mlp.use_latent_moe, + ), + self.post_attention_layernorm, + ), + ) + def _finish_attn_reduce( self, attn_out: torch.Tensor, @@ -2657,6 +2787,7 @@ class KimiK3DecoderLayer(nn.Module): zero_allocator: BumpAllocator, input_sharded: bool = False, keep_sharded: bool = False, + capture_gathered: Optional[AuxHiddenStateAccumulator] = None, ) -> tuple[torch.Tensor, Optional[torch.Tensor], bool]: if attn_res is not None: return self._forward_attn_residual( @@ -2671,6 +2802,25 @@ class KimiK3DecoderLayer(nn.Module): ) assert not input_sharded + if self._stage_boundaries: + hidden_states = self.attn_boundary.prepare( + hidden_states, forward_batch, capture_gathered=capture_gathered + ) + hidden_states = self._run_self_attn( + hidden_states, positions, forward_batch, zero_allocator + ) + hidden_states = self.attn_boundary.finish(hidden_states, forward_batch) + hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch) + hidden_states = self.mlp( + hidden_states, + prefix_sum=( + residual_batch.written_residual(forward_batch) + if self._ffn_writes_stream + else None + ), + ) + return self.ffn_boundary.finish(hidden_states, forward_batch), None, False + # Standard residual path if residual is None: residual = hidden_states @@ -2844,6 +2994,7 @@ class KimiK3LinearModel(nn.Module): self.dspark_layers_to_capture: Optional[list[int]] = None self._dp_attention = is_dp_attention_enabled() self._trim_padded_attn = require_mlp_sync() + self._stage_boundaries = _uses_stage_boundaries(config) if self.pp_group.is_first_rank: embedding_quant_config = ( @@ -2870,6 +3021,13 @@ class KimiK3LinearModel(nn.Module): # Disable on HIP code path. self.alt_streams = None if _is_hip else [torch.cuda.Stream() for _ in range(3)] + # The attention-residual bank of one forward, which every layer's + # stage boundaries read. + self.attn_bank = ( + AttnBank() + if self._stage_boundaries and config.attn_res_block_size is not None + else None + ) self.layers, self.start_layer, self.end_layer = make_pp_layers( config.num_hidden_layers, lambda idx, prefix: KimiK3DecoderLayer( @@ -2878,6 +3036,7 @@ class KimiK3LinearModel(nn.Module): quant_config=quant_config, prefix=prefix, alt_streams=self.alt_streams, + attn_bank=self.attn_bank, ), prefix=f"{prefix}.layers", ) @@ -2895,6 +3054,16 @@ class KimiK3LinearModel(nn.Module): quant_config=None, prefix=f"{prefix}.output_attn_res_proj", ) + self._final_read = ( + self.norm + if self.attn_bank is None + else AttnBankOutputRead( + self.attn_bank, + self.output_attn_res_proj, + self.output_attn_res_norm, + self.norm, + ) + ) else: self.norm = PPMissingLayer() @@ -2906,16 +3075,29 @@ class KimiK3LinearModel(nn.Module): inputs_embeds: torch.Tensor | None = None, pp_proxy_tensors: Optional[PPProxyTensors] = None, ) -> torch.Tensor: + residual = None if get_parallel().pp_group.is_first_rank: if inputs_embeds is not None: hidden_states = inputs_embeds else: hidden_states = self.embed_tokens(input_ids) - residual = None + if self._stage_boundaries: + residual_batch.start(forward_batch) + elif self.attn_bank is not None: + # The bank's wire carries the stream head written: the sending rank + # folds it in before the handoff. + hidden_states = residual_batch.set_written( + pp_proxy_tensors["hidden_states"], forward_batch + ) + elif self._stage_boundaries: + hidden_states = self.layers[self.start_layer].attn_boundary.from_pp( + pp_proxy_tensors, forward_batch + ) else: assert pp_proxy_tensors is not None hidden_states = pp_proxy_tensors["hidden_states"] - residual = pp_proxy_tensors["residual"] + if self.config.attn_res_block_size is None: + residual = pp_proxy_tensors["residual"] if TYPE_CHECKING: assert isinstance(hidden_states, torch.Tensor) assert isinstance(residual, torch.Tensor | None) @@ -2946,12 +3128,18 @@ class KimiK3LinearModel(nn.Module): attn_res = None if self.config.attn_res_block_size is not None: attn_res_block_num = _cdiv(self.end_layer, self.config.attn_res_block_size) - attn_res = AttnResidual( - hidden_states, - attn_res_block_num, - block_residual=residual, + # A later pipeline rank inherits the rows the earlier ones banked. + inherited = ( + None + if self.pp_group.is_first_rank + else pp_proxy_tensors["attn_res_bank"] ) - residual = None + if self.attn_bank is not None: + self.attn_bank.open(hidden_states, attn_res_block_num, inherited) + else: + attn_res = AttnResidual( + hidden_states, attn_res_block_num, block_residual=inherited + ) # Carry the raw residual stream as a token shard across consecutive # SP-MoE layers. PP transfer and dspark capture require full tensors, @@ -2970,18 +3158,28 @@ class KimiK3LinearModel(nn.Module): forward_batch, len(self.dspark_layers_to_capture) ) if packs_aux - else [] + else AuxHiddenStateList() ) + captures = self.dspark_layers_to_capture or () + # On the standard residual path the stream a layer hands on is what the + # next layer's attention input reads, captured there on every row (the + # final norm's, after the last layer). The bank's is the mixture the + # next read would form, captured after the layer. + captures_at_input = self._stage_boundaries and self.attn_bank is None if ( self.dspark_layers_to_capture is not None and not self.pp_group.is_first_rank ): if "dspark_hidden_states" in pp_proxy_tensors.tensors: aux_hidden_states.append(pp_proxy_tensors["dspark_hidden_states"]) - if self.start_layer - 1 in self.dspark_layers_to_capture: + if self.start_layer - 1 in captures and not captures_at_input: aux_hidden_states.append( self._dspark_capture_stream( - self.start_layer - 1, hidden_states, residual, attn_res + self.start_layer - 1, + hidden_states, + residual, + attn_res, + forward_batch, ) ) for i in range(self.start_layer, self.end_layer): @@ -2998,16 +3196,38 @@ class KimiK3LinearModel(nn.Module): zero_allocator=zero_allocator, input_sharded=sp_sharded, keep_sharded=sp_attn_res, + capture_gathered=( + aux_hidden_states + if captures_at_input and i - 1 in captures + else None + ), ) if ( - self.dspark_layers_to_capture is not None - and i in self.dspark_layers_to_capture + not captures_at_input + and i in captures and (i + 1 < self.end_layer or self.pp_group.is_last_rank) ): aux_hidden_states.append( - self._dspark_capture_stream(i, hidden_states, residual, attn_res) + self._dspark_capture_stream( + i, hidden_states, residual, attn_res, forward_batch + ) ) + if not self.pp_group.is_last_rank and self._stage_boundaries: + if self.attn_bank is not None: + # The wire carries the stream head beside the bank. + hidden_states = residual_batch.fold(hidden_states, forward_batch) + proxy_tensors = residual_batch.to_pp(hidden_states, forward_batch) + if self.attn_bank is not None: + proxy_tensors.tensors["attn_res_bank"] = ( + self.attn_bank.require().block_residual + ) + self.attn_bank.close() + if aux_hidden_states: + proxy_tensors.tensors["dspark_hidden_states"] = torch.cat( + aux_hidden_states, dim=-1 + ) + return proxy_tensors if not self.pp_group.is_last_rank: assert not sp_sharded if attn_res is not None: @@ -3015,15 +3235,33 @@ class KimiK3LinearModel(nn.Module): # Materialize the delayed MLP add: the wire carries the # full stream head (bit-identical to the fused fold). hidden_states = residual + hidden_states - residual = attn_res.block_residual # raw bank across ranks - proxy_tensors = {"hidden_states": hidden_states, "residual": residual} + proxy_tensors = { + "hidden_states": hidden_states, + "attn_res_bank": attn_res.block_residual, + } + else: + proxy_tensors = {"hidden_states": hidden_states, "residual": residual} if aux_hidden_states: proxy_tensors["dspark_hidden_states"] = torch.cat( aux_hidden_states, dim=-1 ) return PPProxyTensors(proxy_tensors) - if hidden_states.shape[0] != 0: + if self._stage_boundaries: + hidden_states = residual_batch.final_norm( + hidden_states, + forward_batch, + self._final_read, + capture=( + aux_hidden_states.capture + if captures_at_input and self.end_layer - 1 in captures + else None + ), + skip_empty=True, + ) + if self.attn_bank is not None: + self.attn_bank.close() + elif hidden_states.shape[0] != 0: if attn_res is not None: # ---- Final aggregation (output side, folds delayed add) ---- if sp_sharded: @@ -3081,10 +3319,15 @@ class KimiK3LinearModel(nn.Module): hidden_states: torch.Tensor, residual: Optional[torch.Tensor], attn_res: Optional[AttnResidual], + forward_batch: ForwardBatch, ) -> torch.Tensor: """Stream value after `layer_idx`: the pre-norm mixture its next consumer would compute (next layer's attention side; output side for the last layer).""" + if self._stage_boundaries: + # The bank path; the standard one captures at the next input. + hidden_states = residual_batch.snapshot(hidden_states, forward_batch) + attn_res, residual = self.attn_bank.require(), None if attn_res is None: return hidden_states if residual is None else hidden_states + residual if residual is not None: diff --git a/test/registered/unit/layer_boundary/test_attn_bank_residual_ops.py b/test/registered/unit/layer_boundary/test_attn_bank_residual_ops.py new file mode 100644 index 0000000000..84398bfae6 --- /dev/null +++ b/test/registered/unit/layer_boundary/test_attn_bank_residual_ops.py @@ -0,0 +1,327 @@ +"""A stack read through the attention-residual bank's stage reads computes +what the decoder computed when its MLP folded the pending add itself.""" + +import unittest +from types import SimpleNamespace +from unittest.mock import patch + +import test_declared_decoder_boundary as fixture +import torch + +from sglang.srt.layers import attn_residual +from sglang.srt.layers.layer_boundary import ( + BatchVariant, + ExitRows, + append_stages, + declare_attn, + declare_ffn, + layer_stack, +) +from sglang.srt.layers.layer_boundary import prepare as comm_ops +from sglang.srt.layers.layer_boundary.ops import update_attn_tp_gather_output +from sglang.srt.layers.layer_boundary.residual import attn_bank +from sglang.srt.layers.layer_boundary.residual.add_norm import REPLACE_AT_EXIT +from sglang.srt.layers.layer_boundary.residual.attn_bank import ( + AttnBank, + AttnBankOutputRead, + AttnBankState, +) +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=5, suite="base-a-test-cpu") + +TOKENS = 3 +HIDDEN = 8 +BLOCK = 2 +LAYERS = 5 + + +class _Norm(torch.nn.Module): + def __init__(self, seed): + super().__init__() + self.scale = 0.5 + seed / 7 + self.variance_epsilon = 1e-6 + + def forward(self, x): + return x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + 1e-6) * self.scale + + +class _Proj(torch.nn.Module): + def __init__(self, seed): + super().__init__() + self.weight = torch.randn( + 1, HIDDEN, generator=torch.Generator().manual_seed(seed) + ) + + def forward(self, x): + return x @ self.weight.t(), None + + +def _mix(prefix_sum, bank, nvb, score_proj, score_norm): + # The torch reference for the triton score + combine pair. + return attn_residual.aggregate_stream_torch( + prefix_sum, bank, nvb, score_proj, score_norm + ) + + +class _Layer: + """One layer's parameters and stand-ins for its attention and its MLP.""" + + def __init__(self, idx): + self.writes_block = idx % BLOCK == 0 + self.attn_proj, self.attn_score_norm = _Proj(4 * idx), _Norm(4 * idx) + self.ffn_proj, self.ffn_score_norm = _Proj(4 * idx + 1), _Norm(4 * idx + 1) + self.input_norm, self.post_norm = _Norm(4 * idx + 2), _Norm(4 * idx + 3) + self.attn_weight = 0.3 + 0.1 * idx + self.mlp_weight = -0.2 + 0.05 * idx + + def attention(self, x): + return torch.tanh(x) * self.attn_weight + + def mlp(self, x): + return torch.sin(x) * self.mlp_weight + + def ops(self, bank): + return AttnBankState( + bank, + self.attn_proj, + self.attn_score_norm, + self.ffn_proj, + self.ffn_score_norm, + writes_block=self.writes_block, + ).residual_ops() + + +LAYER_LIST = [_Layer(i) for i in range(LAYERS)] +OUT_PROJ, OUT_SCORE_NORM, FINAL_NORM = _Proj(99), _Norm(99), _Norm(100) + + +def _legacy(hidden, layers, bank_rows=None, *, final=True): + """The decoder's own order: the MLP adds the pending prefix to its output, + so each layer hands on the whole head.""" + bank = attn_residual.AttnResidual( + hidden, -(-(layers.stop) // BLOCK), block_residual=bank_rows + ) + for layer in LAYER_LIST[layers]: + hidden, prefix = bank.forward( + hidden, + None, + layer.attn_proj, + layer.attn_score_norm, + layer.input_norm, + write=layer.writes_block, + ) + if layer.writes_block: + prefix = None + hidden, prefix = bank.forward( + layer.attention(hidden), + prefix, + layer.ffn_proj, + layer.ffn_score_norm, + layer.post_norm, + ) + hidden = layer.mlp(hidden) + prefix + if not final: + return hidden, bank.block_residual + hidden, _ = bank.forward(hidden, None, OUT_PROJ, OUT_SCORE_NORM, FINAL_NORM) + return hidden + + +def _staged( + hidden, layers, bank_rows=None, *, read_after_write, writes_stream=(), final=True +): + """The stage boundaries' order: each layer's output and the residual stay + apart, and the next read folds the add into its aggregation. A layer in + ``writes_stream`` adds the residual into its own output instead (a latent + MoE's tail add) and hands on the written stream.""" + holder = AttnBank() + holder.open(hidden, -(-(layers.stop) // BLOCK), bank_rows) + # A stack's first stage, or a pipeline rank handed the whole head: the + # stream is written and the read takes it alone. + residual, contribution = hidden, None + for idx in range(layers.start, layers.stop): + layer = LAYER_LIST[idx] + ops = layer.ops(holder) + if contribution is None: + hidden, residual = ops.attn_readout.read(residual, layer.input_norm) + else: + hidden, residual = ops.attn_readout.update_and_read( + ops.attn_update, contribution, residual, layer.input_norm + ) + attn_out = layer.attention(hidden) + if residual is None and read_after_write: + # What a stream written by the attention's read hands the FFN. + hidden, residual = ops.ffn_readout.read(attn_out, layer.post_norm) + else: + hidden, residual = ops.ffn_readout.update_and_read( + ops.attn_update, attn_out, residual, layer.post_norm + ) + if idx in writes_stream: + residual, contribution = layer.mlp(hidden) + residual, None + else: + contribution = layer.mlp(hidden) + if contribution is None: + if not final: + return residual, holder.require().block_residual + return AttnBankOutputRead(holder, OUT_PROJ, OUT_SCORE_NORM, FINAL_NORM)( + residual + ) + if not final: + return contribution + residual, holder.require().block_residual + # Called as the final norm of an output and its residual: it returns the + # normalized output and the head it aggregated. + hidden, head = AttnBankOutputRead(holder, OUT_PROJ, OUT_SCORE_NORM, FINAL_NORM)( + contribution, residual + ) + torch.testing.assert_close(head, contribution + residual, rtol=0, atol=0) + return hidden + + +class TestAttnBankResidualOps(CustomTestCase): + def setUp(self): + patcher = patch.object(attn_residual, "_mix_fused", _mix) + patcher.start() + self.addCleanup(patcher.stop) + self.hidden = torch.randn( + TOKENS, HIDDEN, generator=torch.Generator().manual_seed(7) + ) + + def assert_identical(self, got, want): + torch.testing.assert_close(got, want, rtol=0, atol=0) + + def test_the_stack_matches_the_decoders_own_order(self): + want = _legacy(self.hidden, slice(0, LAYERS)) + for read_after_write in (False, True): + for writes_stream in ((), (1, 3), (LAYERS - 1,)): + with self.subTest( + read_after_write=read_after_write, writes_stream=writes_stream + ): + got = _staged( + self.hidden, + slice(0, LAYERS), + read_after_write=read_after_write, + writes_stream=writes_stream, + ) + self.assert_identical(got, want) + + def test_a_pipeline_rank_continues_from_the_head_and_the_bank(self): + want = _legacy(self.hidden, slice(0, LAYERS)) + # Split before a layer that writes a block and before one that reads, + # after a layer that adds the residual itself and after one that + # leaves the add to the next read. + for split in (2, 3): + for writes_stream in ((), (split - 1,)): + with self.subTest(split=split, writes_stream=writes_stream): + head, bank_rows = _staged( + self.hidden, + slice(0, split), + read_after_write=False, + writes_stream=writes_stream, + final=False, + ) + legacy_head, legacy_rows = _legacy( + self.hidden, slice(0, split), final=False + ) + # The wire carries what the decoder's own order sent. + self.assert_identical(head, legacy_head) + self.assert_identical(bank_rows, legacy_rows) + got = _staged( + head, + slice(split, LAYERS), + bank_rows, + read_after_write=False, + ) + self.assert_identical(got, want) + + def test_a_read_on_a_shard_aggregates_its_rows_of_the_bank(self): + """After a reduce-scatter over attention TP each rank reads its + contiguous shard of the rows, against the same shard of the bank.""" + generator = torch.Generator().manual_seed(3) + hidden = torch.randn(4, HIDDEN, generator=generator) + attn_out = torch.randn(4, HIDDEN, generator=generator) + layer = LAYER_LIST[0] + holder = AttnBank() + holder.open(hidden, 1) + ops = layer.ops(holder) + # The write layer's attention read banks every token's row. + ops.attn_readout.read(hidden, layer.input_norm) + want, _ = ops.ffn_readout.read(attn_out, layer.post_norm) + + shard = slice(2, 4) + second_of_two = SimpleNamespace(attn_tp_rank=1, attn_tp_size=2) + with patch.object(attn_bank, "get_parallel", lambda: second_of_two): + got, residual = ops.ffn_readout.read(attn_out[shard], layer.post_norm) + self.assert_identical(got, want[shard]) + self.assert_identical(residual, attn_out[shard]) + with self.assertRaisesRegex(RuntimeError, "attention-TP shard"): + ops.ffn_readout.read(attn_out[:3], layer.post_norm) + + def test_declared_capabilities(self): + ops = LAYER_LIST[1].ops(AttnBank()) + # The updates are ordinary adds, so they may cross a pipeline boundary + # and defer the sum they complete to the next read. + for update in (ops.attn_update, ops.ffn_update): + self.assertTrue(update.is_plain_add) + self.assertFalse(update.applied_at_exit) + self.assertTrue(update.outlives_layer) + # The reads aggregate the bank, which no add+norm kernel computes, on + # this rank's own rows. + for readout in (ops.attn_readout, ops.ffn_readout): + self.assertFalse(readout.is_plain_norm) + self.assertTrue(readout.reads_before_dp_gather) + + +class TestAttnBankSpMoeStages(CustomTestCase): + """A latent MoE dispatched over an a2a backend with attention TP runs on + this rank's shard of the rows: the FFN's entry reduce-scatters the + attention output and slices the residual, and each MoE layer's exit + gathers its stream back to every row, as the decoder did.""" + + def build(self, exit_rows): + holder = AttnBank() + with ( + fixture.planning( + fixture.parallel_of(attn_dp=1, attn_tp=2), + a2a=True, + boundary_reduction="ar", + ), + layer_stack(), + ): + return [ + append_stages( + ( + declare_attn(read=ops.attn_readout, update=ops.attn_update), + fixture.Norm(), + ), + ( + declare_ffn( + read=ops.ffn_readout, + update=REPLACE_AT_EXIT, + sparse=True, + next_layer_sparse=True, + output_complete=True, + exit_rows=exit_rows, + ), + fixture.Norm(), + ), + )[1] + for ops in (LAYER_LIST[i].ops(holder) for i in range(3)) + ] + + def entry_step(self, ffn): + prepare = ffn.plan.paths[BatchVariant.ORDINARY].entry.prepare + return prepare.keywords["step"] + + def test_each_moe_layer_returns_to_every_row(self): + for ffn in self.build(ExitRows.ATTENTION): + step = self.entry_step(ffn) + self.assertIs(step.func, comm_ops._attn_tp_reduce_scatter_update_read) + self.assertTrue(step.keywords["scatters_residual"]) + self.assertIs(step.keywords["read"], ffn.declaration.read) + move = ffn.plan.paths[BatchVariant.ORDINARY].output_move + self.assertIs(move.func, update_attn_tp_gather_output) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/layer_boundary/test_aux_capture_deferred_allreduce.py b/test/registered/unit/layer_boundary/test_aux_capture_deferred_allreduce.py index f4df42625f..9663a86247 100644 --- a/test/registered/unit/layer_boundary/test_aux_capture_deferred_allreduce.py +++ b/test/registered/unit/layer_boundary/test_aux_capture_deferred_allreduce.py @@ -12,6 +12,7 @@ from sglang.srt.layers.layer_boundary import StageKind from sglang.srt.layers.layer_boundary import prepare as comm_ops from sglang.srt.layers.layer_boundary.contracts import BatchVariant from sglang.srt.layers.layer_boundary.ops import keep_output +from sglang.srt.layers.layer_boundary.residual import batch as residual_batch from sglang.srt.layers.layer_boundary.residual.stream import ResidualStream from sglang.srt.model_executor.forward_batch_info import ForwardMode, PPProxyTensors from sglang.srt.models.bailing_moe import BailingMoEModel @@ -209,6 +210,22 @@ class TestAuxCaptureDeferredAllreduce(CustomTestCase): self.assertIsNone(captured) reduce.assert_not_called() + def test_an_empty_final_norm_still_captures(self): + # A rank without rows takes the same number of captures as the others. + stream = ResidualStream(torch.empty(0, 4)) + hidden = stream.record(torch.empty(0, 4), comm.PLAIN_ADD) + batch = SimpleNamespace(residual_stream=stream) + captured = AuxHiddenStateList() + hidden = residual_batch.final_norm( + hidden, + batch, + SumNorm(), + capture=captured.capture, + skip_empty=True, + ) + self.assertEqual(tuple(hidden.shape), (0, 4)) + self.assertEqual([tuple(c.shape) for c in captured], [(0, 4)]) + def test_snapshot_without_a_residual_does_not_alias_the_main_output(self): hidden = torch.ones(2, 4) captured = ResidualStream().snapshot(hidden) diff --git a/test/registered/unit/layer_boundary/test_boundary_output_contracts.py b/test/registered/unit/layer_boundary/test_boundary_output_contracts.py index 63132d483d..1d35a654e7 100644 --- a/test/registered/unit/layer_boundary/test_boundary_output_contracts.py +++ b/test/registered/unit/layer_boundary/test_boundary_output_contracts.py @@ -217,6 +217,33 @@ class TestBoundaryIntegrations(unittest.TestCase): self.assertIsNone(path.output.group, (sparse, variant)) self.assertFalse(path.output.may_defer_to_next) + def test_an_ffn_that_completes_its_own_sum_owes_none(self): + # Its compute completes the sum, so the exit neither sums nor defers it, + # under DP too. + for attn_dp in (1, 2): + with ( + self.subTest(attn_dp=attn_dp), + fixture.planning(fixture.parallel_of(attn_dp=attn_dp, attn_tp=2)), + ): + for sparse in (False, True): + stages = { + complete: build_stages( + (declare_attn(), fixture.Norm()), + ( + declare_ffn(sparse=sparse, output_complete=complete), + fixture.Norm(), + ), + previous=declare_ffn(sparse=sparse), + )[1] + for complete in (False, True) + } + owed = stages[False].plan.paths[BatchVariant.ORDINARY].output + self.assertIsNotNone(owed.group, sparse) + for variant, path in stages[True].plan.paths.items(): + self.assertIsNone(path.output.group, (sparse, variant)) + self.assertFalse(path.output.may_defer_to_next) + self.assertFalse(path.output.may_reduce_scatter) + def test_only_an_ffn_writing_the_next_stream_sums_its_parts(self): summed = Mock(side_effect=lambda value, *args, **kwargs: value * 2) fb = SimpleNamespace() diff --git a/test/registered/unit/layer_boundary/test_stage_producers_leave_sums.py b/test/registered/unit/layer_boundary/test_stage_producers_leave_sums.py index 4dd5ad0f03..0e3d4eefa7 100644 --- a/test/registered/unit/layer_boundary/test_stage_producers_leave_sums.py +++ b/test/registered/unit/layer_boundary/test_stage_producers_leave_sums.py @@ -42,6 +42,16 @@ _BUILDS_STAGES = ( ) +# Producers whose stage declares output_complete: their computation completes +# the stage output's sum. Kimi K3's latent MoE reduces the latent before its +# norm and completes the shared experts' sum beside it. +_COMPLETES_OWN_SUM = { + "kimi_k3.py": frozenset( + {"_reduce_latent", "_reduce_shared", "_reduce_latent_and_shared"} + ), +} + + # A vocabulary-parallel lookup leaves each rank holding only its own shard's # rows, so the class completes that sum itself. An embedding is never a decoder # stage, so no sum inside one is a stage output and the whole class is exempt. @@ -84,16 +94,19 @@ def _reduce_results_names(tree) -> set: return names -def _unguarded_sums(source: str): +def _unguarded_sums(source: str, completes_own_sum=frozenset()): """Sum calls not under ``if ... self.reduce_results ...``: such a branch is a shared class built without stage boundaries by another model. Sums inside - a vocabulary-parallel embedding complete its lookup, not a stage output.""" + a vocabulary-parallel embedding complete its lookup, not a stage output, + and the named functions belong to a producer that completes its own.""" found = [] guards = _reduce_results_names(ast.parse(source)) def visit(node, guarded): if _is_vocab_parallel(node): return + if isinstance(node, ast.FunctionDef) and node.name in completes_own_sum: + return if isinstance(node, ast.If) and _requires(node.test, guards): for child in node.body: visit(child, True) @@ -118,7 +131,12 @@ class TestStageProducersLeaveSums(CustomTestCase): if not any(marker in source for marker in _BUILDS_STAGES): continue checked += 1 - if sums := _unguarded_sums(source): + exempt = _COMPLETES_OWN_SUM.get(path.name, frozenset()) + if exempt and "output_complete=" not in source: + offenders[str(path.relative_to(models))] = [ + "exempt sums, but no stage declares output_complete" + ] + elif sums := _unguarded_sums(source, exempt): offenders[str(path.relative_to(models))] = sums self.assertGreater(checked, 30) self.assertEqual(offenders, {}) diff --git a/test/registered/unit/model_executor/test_prefill_cuda_graph_runner_helpers.py b/test/registered/unit/model_executor/test_prefill_cuda_graph_runner_helpers.py index e9d2a7017e..de0d7273cf 100644 --- a/test/registered/unit/model_executor/test_prefill_cuda_graph_runner_helpers.py +++ b/test/registered/unit/model_executor/test_prefill_cuda_graph_runner_helpers.py @@ -274,7 +274,7 @@ class TestPrefillCudaGraphRunnerHelpers(CustomTestCase): }, { "hidden_states": (16, 8), - "residual": (16, 3, 8), + "attn_res_bank": (16, 3, 8), "dspark_hidden_states": (16, 16), }, ) diff --git a/test/registered/unit/test_model_overrides.py b/test/registered/unit/test_model_overrides.py index 5bc3ab186f..8bd14e2289 100644 --- a/test/registered/unit/test_model_overrides.py +++ b/test/registered/unit/test_model_overrides.py @@ -242,6 +242,8 @@ class TestBoundaryReductionDefaults(CustomTestCase): for architecture, default in ( ("Qwen3ForCausalLM", "ar"), ("Qwen3Model", "ar"), + ("KimiK3ForConditionalGeneration", "ar"), + ("KimiK3LinearForCausalLM", "ar"), ("MossVLForConditionalGeneration", "ar"), ("Qwen4ExpForConditionalGeneration", "ar"), ("Qwen4ExpForCausalLMMTP", "ar"),