[Refactor] Run Kimi K3 through stage boundaries (#42894)

This commit is contained in:
Cheng Wan
2026-10-09 01:25:51 -07:00
committed by GitHub
parent 666da4f56b
commit 125f88cb75
17 changed files with 978 additions and 104 deletions
@@ -12,6 +12,8 @@ _RSV_ARCHITECTURES = {
"Step3VLForConditionalGeneration",
}
_AR_ARCHITECTURES = {
"KimiK3ForConditionalGeneration",
"KimiK3LinearForCausalLM",
"Qwen3ForCausalLM",
"Qwen3ForSequenceClassification",
"Qwen3Model",
@@ -61,22 +61,25 @@ def handle_deprecated_dp_attention(server_args: Any):
)
def _boundary_parallelism_overrides(cfg, model_type: str) -> dict:
def _boundary_parallelism_overrides(cfg, model_type: str, architecture: str) -> dict:
"""Resolve the token-row modes the model's forward actually enters."""
nemotron = model_type in ("nemotron_h", "nemotron_h_puzzle")
longcat = model_type == "longcat_flash"
kimi_k3 = architecture in (
"KimiK3ForConditionalGeneration",
"KimiK3LinearForCausalLM",
)
if nemotron and cfg.attn_cp_size > 1:
raise ValueError("Nemotron-H does not support --attn-cp-size > 1")
if longcat and cfg.enable_prefill_cp and cfg.attn_cp_size > 1:
raise ValueError(
"LongCat-Flash does not support --enable-prefill-cp with --attn-cp-size > 1"
)
if (nemotron or longcat) and cfg.enable_attn_tp_input_scattered:
# Neither model enters the input-scattered attention scope. Preserve
# their existing ordinary execution and make the effective flag explicit.
if (nemotron or longcat or kimi_k3) and cfg.enable_attn_tp_input_scattered:
# None of these models enters the input-scattered attention scope.
logger.warning(
"Disabling input-scattered attention for %s: its forward does not enter that scope",
model_type,
architecture,
)
return {"enable_attn_tp_input_scattered": False}
return {}
@@ -99,7 +102,7 @@ def handle_context_parallelism(server_args: Any):
server_args,
"boundary_parallelism",
**_boundary_parallelism_overrides(
cfg, model_config.hf_text_config.model_type
cfg, model_config.hf_text_config.model_type, model_arch
),
)
if (
@@ -96,6 +96,7 @@ def _resolve_ffn(
read=NORM_READOUT,
update=PLAIN_ADD,
dense_tp_size=None,
output_complete=False,
):
parallel = get_parallel()
if dense_tp_size not in (None, 1, parallel.tp_size):
@@ -167,7 +168,7 @@ def _resolve_ffn(
)
)
# An FFN that writes the next stream itself hands on a complete output.
complete = on_rank_rows or update is REPLACE_AT_EXIT
complete = on_rank_rows or update is REPLACE_AT_EXIT or output_complete
produced = (
OutputContract(rows, update=update, transform=output_transform)
if complete
@@ -222,6 +223,9 @@ class StageDeclaration:
exit_rows: Required FFN output rows at the layer or branch exit.
writes_at_handoff: Whether this FFN writes its output into the residual
at its exit because the next stage is on another pipeline rank.
output_complete: Whether the FFN's compute completes its own output
sum, so the exit owes none. For an FFN whose output sum is fused
with a reduction its computation needs anyway.
previous: Declaration whose output this stage consumes, as the stack
records it; across pipeline ranks it is built locally.
prepared_from: Declaration whose already-read input a branch reuses.
@@ -242,6 +246,7 @@ class StageDeclaration:
dense_tp_size: Optional[int] = None
exit_rows: Optional[ExitRows] = None
writes_at_handoff: bool = False
output_complete: bool = False
# Only declarations participate in construction, never executable stages.
previous: Optional[StageDeclaration] = None
prepared_from: Optional[StageDeclaration] = None
@@ -329,6 +334,7 @@ def declare_ffn(
next_layer_sparse=False,
dense_tp_size=None,
exit_rows=None,
output_complete=False,
):
"""Declare a dense or MoE FFN independently of its compute module.
@@ -343,6 +349,8 @@ def declare_ffn(
compute, or the full TP size.
exit_rows: Explicit output-row requirement; otherwise derived from
the adjacent FFN kinds and TBO configuration.
output_complete: Whether the compute completes its own output sum,
fused with a reduction it needs anyway; the exit then owes none.
Returns:
A StageDeclaration with no norm, tensors or execution plan.
@@ -355,6 +363,7 @@ def declare_ffn(
output_transform=output_transform,
dense_tp_size=dense_tp_size,
exit_rows=exit_rows or tbo_exit_rows(sparse, next_layer_sparse),
output_complete=output_complete,
)
@@ -378,6 +387,7 @@ def _resolve_stage(stage, variant, following=None):
read=stage.read,
update=stage.update,
dense_tp_size=stage.dense_tp_size,
output_complete=stage.output_complete,
)
if resolve_exit_rows(stage.exit_rows) is ExitRows.ATTENTION:
returned = attention
@@ -343,7 +343,7 @@ def _update_read(
def _attn_tp_reduce_scatter_update_read(
hidden_states: torch.Tensor,
residual: torch.Tensor,
residual: Optional[torch.Tensor],
forward_batch: ForwardBatch,
norm: torch.nn.Module,
*,
@@ -353,7 +353,7 @@ def _attn_tp_reduce_scatter_update_read(
update: ResidualUpdate = PLAIN_ADD,
):
hidden_states = attn_tp_reduce_scatter(hidden_states)
if scatters_residual:
if scatters_residual and residual is not None:
residual = update.slice_residual_attn_tp(residual)
return read.update_and_read(update, hidden_states, residual, norm)
@@ -0,0 +1,223 @@
# Copyright 2023-2024 SGLang Team
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""A residual read that aggregates a bank of snapshots of itself."""
from dataclasses import dataclass
from typing import Optional
import torch
from sglang.srt.layers.attn_residual import AttnResidual
from sglang.srt.layers.layer_boundary.residual import LayerResidualOps
from sglang.srt.layers.layer_boundary.residual.add_norm import PLAIN_ADD
from sglang.srt.runtime_context import get_parallel
class AttnBank:
"""The snapshot bank of one layer-stack invocation.
Every layer's reads aggregate the same bank, so the bank outlives a layer
while the reads that consume it are per-layer. The stack's entry opens one
and the layers reach it through this holder; a pipeline rank that did not
open the stack inherits the rows the previous rank banked.
"""
__slots__ = ("current",)
def __init__(self):
self.current = None
def open(self, hidden_states, block_num, inherited=None) -> AttnResidual:
self.current = AttnResidual(hidden_states, block_num, inherited)
return self.current
def close(self) -> None:
self.current = None
def require(self) -> AttnResidual:
if self.current is None:
raise RuntimeError("open the attention-residual bank before a stage")
return self.current
def _bank_rows(bank: AttnResidual, hidden_states) -> Optional[slice]:
"""The bank rows that go with ``hidden_states``: all of them, or this
rank's contiguous attention-TP shard of them, the rows a reduce-scatter
over attention TP leaves it."""
total, rows = bank.block_residual.shape[0], hidden_states.shape[0]
if rows == total:
return None
parallel = get_parallel()
if rows * parallel.attn_tp_size != total:
raise RuntimeError(
f"{rows} rows are neither the bank's {total} rows nor an "
f"attention-TP shard of them over {parallel.attn_tp_size} ranks"
)
rank = parallel.attn_tp_rank
return slice(rank * rows, (rank + 1) * rows)
@dataclass
class AttnBankState:
"""One layer's pair of reads into the bank an `AttnBank` holds.
A read aggregates the banked snapshots with the running residual and
normalizes the result; the update that precedes it is an ordinary add, so
both stages declare `PLAIN_ADD` and each read folds the pending add into
its aggregation kernel. A read on this rank's attention-TP shard of the
rows aggregates the matching shard of the bank. A write layer also
snapshots the residual it just formed into the bank's next row, where it
now lives: the running residual restarts empty, and the next contribution
alone is the new head."""
bank: AttnBank
attn_score_proj: torch.nn.Module
attn_score_norm: torch.nn.Module
ffn_score_proj: torch.nn.Module
ffn_score_norm: torch.nn.Module
writes_block: bool = False
def _aggregate(
self, contribution, residual, norm, *, score_proj, score_norm, write
):
bank = self.bank.require()
return bank.forward(
contribution,
residual,
score_proj,
score_norm,
norm,
rows=_bank_rows(bank, contribution),
write=write,
)
def read_attn_input(self, residual, norm):
# No pending add: the residual alone is the head (the stack's input,
# or the residual of a stream that was written).
return self._attn_aggregate(residual, None, norm)
def update_and_read_attn_input(self, contribution, residual, norm):
return self._attn_aggregate(contribution, residual, norm)
def _attn_aggregate(self, contribution, residual, norm):
normed, residual = self._aggregate(
contribution,
residual,
norm,
score_proj=self.attn_score_proj,
score_norm=self.attn_score_norm,
write=self.writes_block,
)
return normed, None if self.writes_block else residual
def update_and_read_ffn_input(self, contribution, residual, norm):
return self._aggregate(
contribution,
residual,
norm,
score_proj=self.ffn_score_proj,
score_norm=self.ffn_score_norm,
write=False,
)
def residual_ops(self) -> LayerResidualOps:
return LayerResidualOps(
attn_readout=_AttnReadout(self),
attn_update=PLAIN_ADD,
ffn_readout=_FfnReadout(self),
ffn_update=PLAIN_ADD,
)
class _BankReadout:
"""A read that mixes the banked snapshots into the residual's norm, so it
is not the residual's plain norm and no add+norm kernel may claim it. It
reads this rank's own rows, which the bank holds, before any DP gather."""
is_plain_norm = False
reads_before_dp_gather = True
def __init__(self, state: AttnBankState):
self.state = state
def init_residual(self, hidden_states):
return hidden_states
def _reject(self, update, quant_format):
if quant_format:
raise NotImplementedError(
f"an attention-residual bank read in {quant_format=}"
)
if update is not None and not update.is_plain_add:
# The read folds the pending add into its aggregation, which only
# reproduces an ordinary add.
raise NotImplementedError(
f"an attention-residual bank read after {update=}"
)
class _AttnReadout(_BankReadout):
"""The attention input: the bank aggregation and this layer's input norm.
A write layer snapshots the residual this read forms."""
def read(self, residual, norm, quant_format="", post_residual_addition=None):
self._reject(None, quant_format)
if post_residual_addition is not None:
raise NotImplementedError(
"an attention-residual bank read with a separate addition"
)
return self.state.read_attn_input(residual, norm)
def update_and_read(self, update, hidden_states, residual, norm, **kwargs):
self._reject(update, kwargs.get("quant_format", ""))
return self.state.update_and_read_attn_input(hidden_states, residual, norm)
class _FfnReadout(_BankReadout):
"""The FFN input: the attention output's add folded into the bank
aggregation, and this layer's post-attention norm."""
def read(self, residual, norm, quant_format="", post_residual_addition=None):
# The attention read of a write layer leaves the residual empty, so the
# attention output alone is the head.
self._reject(None, quant_format)
return self.state.update_and_read_ffn_input(residual, None, norm)
def update_and_read(self, update, hidden_states, residual, norm, **kwargs):
self._reject(update, kwargs.get("quant_format", ""))
return self.state.update_and_read_ffn_input(hidden_states, residual, norm)
class AttnBankOutputRead:
"""The layer stack's terminal read: the bank aggregation with the output
side's scoring parameters, then the final norm. A callable final norm for
`residual_batch.final_norm`."""
def __init__(self, bank: AttnBank, score_proj, score_norm, norm):
self.bank = bank
self.score_proj = score_proj
self.score_norm = score_norm
self.norm = norm
def __call__(self, hidden_states, residual=None):
bank = self.bank.require()
normed, head = bank.forward(
hidden_states,
residual,
self.score_proj,
self.score_norm,
self.norm,
rows=_bank_rows(bank, hidden_states),
)
return normed if residual is None else (normed, head)
@@ -106,6 +106,8 @@ def final_norm(
)
return hidden_states
if skip_empty and hidden_states.shape[0] == 0:
if capture is not None:
capture(hidden_states)
return hidden_states
return access.final_norm_pair(
hidden_states, residual, layernorm, capture, **read_kwargs
@@ -131,6 +131,10 @@ class DynamicChunkSizer:
return predicted_size
def _profile_prefill_latency(self) -> Tuple[List[int], List[float]]:
from sglang.srt.model_executor.runner_utils.buffers import (
allocate_pp_proxy_tensors,
)
seq_lens: List[int] = []
latencies: List[float] = []
model_runner = self.model_runner
@@ -193,28 +197,18 @@ class DynamicChunkSizer:
batch.global_num_tokens = global_num_tokens
batch.global_num_tokens_for_logprob = global_num_tokens
hs = (
getattr(model_config, "hc_hidden_size", None)
or model_config.hidden_size
)
proxy_tensors = {
"hidden_states": torch.zeros(
(current_seq_len, hs),
# Keyed and shaped like the runner's PP buffers.
with torch.device(self.device):
proxy_tensors = allocate_pp_proxy_tensors(
max_num_tokens=current_seq_len,
max_hidden_tokens=current_seq_len,
hidden_size=model_config.hidden_size,
dtype=model_config.dtype,
device=self.device,
),
"residual": torch.zeros(
(current_seq_len, model_config.hidden_size),
dtype=model_config.dtype,
device=self.device,
),
}
pp_proxy_topk_size = model_runner.get_pp_proxy_topk_size()
if pp_proxy_topk_size is not None:
proxy_tensors["topk_indices"] = torch.zeros(
(current_seq_len, pp_proxy_topk_size),
dtype=torch.int32,
device=self.device,
hc_hidden_size=getattr(model_config, "hc_hidden_size", None),
pp_proxy_topk_size=model_runner.get_pp_proxy_topk_size(),
pp_proxy_residual_num_blocks=(
model_runner.get_pp_proxy_residual_num_blocks()
),
)
pp_proxy = PPProxyTensors(proxy_tensors)
@@ -138,15 +138,17 @@ def _allocate_decode_buffers(
pp_proxy_tensors = {
"hidden_states": torch.zeros((max_num_token, hs), dtype=dtype),
}
if not is_mhc:
# Only Kimi K3 supplies num_blocks: its PP bank is token-major
# [T, blocks, H]. Other models use [T, H].
residual_shape = (
(max_num_token, pp_proxy_residual_num_blocks, hidden_size)
if pp_proxy_residual_num_blocks is not None
else (max_num_token, hidden_size)
if pp_proxy_residual_num_blocks is not None:
# Only Kimi K3 supplies num_blocks: its attention-residual bank
# is token-major [T, blocks, H] and takes the residual's place.
pp_proxy_tensors["attn_res_bank"] = torch.zeros(
(max_num_token, pp_proxy_residual_num_blocks, hidden_size),
dtype=dtype,
)
elif not is_mhc:
pp_proxy_tensors["residual"] = torch.zeros(
(max_num_token, hidden_size), dtype=dtype
)
pp_proxy_tensors["residual"] = torch.zeros(residual_shape, dtype=dtype)
if pp_proxy_topk_size is not None:
pp_proxy_tensors["topk_indices"] = torch.zeros(
(max_num_token, pp_proxy_topk_size), dtype=torch.int32
@@ -64,7 +64,7 @@ def _grouped_foreach_copy_(dsts: List[torch.Tensor], srcs: List[torch.Tensor]) -
foreach_copy(group_dsts, group_srcs)
def _allocate_pp_proxy_tensors(
def allocate_pp_proxy_tensors(
*,
max_num_tokens: int,
max_hidden_tokens: int,
@@ -81,15 +81,17 @@ def _allocate_pp_proxy_tensors(
pp_proxy_tensors = {
"hidden_states": torch.zeros((max_hidden_tokens, pp_hidden_size), dtype=dtype),
}
if not is_mhc:
# Only Kimi K3 supplies num_blocks: its PP bank is token-major
# [T, blocks, H]. Other models use the phase-specific hidden-token bound.
residual_shape = (
(max_num_tokens, pp_proxy_residual_num_blocks, hidden_size)
if pp_proxy_residual_num_blocks is not None
else (max_hidden_tokens, hidden_size)
if pp_proxy_residual_num_blocks is not None:
# Only Kimi K3 supplies num_blocks: its attention-residual bank is
# token-major [T, blocks, H] and takes the residual's place.
pp_proxy_tensors["attn_res_bank"] = torch.zeros(
(max_num_tokens, pp_proxy_residual_num_blocks, hidden_size), dtype=dtype
)
elif not is_mhc:
# Sized by the phase-specific hidden-token bound, like hidden_states.
pp_proxy_tensors["residual"] = torch.zeros(
(max_hidden_tokens, hidden_size), dtype=dtype
)
pp_proxy_tensors["residual"] = torch.zeros(residual_shape, dtype=dtype)
if pp_proxy_topk_size is not None:
pp_proxy_tensors["topk_indices"] = torch.zeros(
(max_num_tokens, pp_proxy_topk_size), dtype=torch.int32
@@ -185,7 +187,7 @@ class DecodeInputBuffers(ForwardInputBuffers):
)
pp_proxy_tensors = (
_allocate_pp_proxy_tensors(
allocate_pp_proxy_tensors(
max_num_tokens=max_num_token,
max_hidden_tokens=max_num_token,
hidden_size=hidden_size,
@@ -333,7 +335,7 @@ class PrefillInputBuffers(ForwardInputBuffers):
mrope_positions = None
pp_proxy_tensors = (
_allocate_pp_proxy_tensors(
allocate_pp_proxy_tensors(
max_num_tokens=max_num_tokens,
max_hidden_tokens=max_num_tokens,
hidden_size=hidden_size,
+294 -51
View File
@@ -28,6 +28,7 @@ from sglang.srt.distributed import (
from sglang.srt.distributed.device_communicators.pynccl_allocator import (
use_symmetric_memory,
)
from sglang.srt.distributed.utils import get_group_rank_size
from sglang.srt.environ import envs
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.layers import zero_copy_context
@@ -35,6 +36,7 @@ from sglang.srt.layers.activation import SiluAndMul, SituAndMul
from sglang.srt.layers.attn_residual import AttnResidual, aggregate_stream, get_cw
from sglang.srt.layers.aux_hidden_states import (
AuxHiddenStateAccumulator,
AuxHiddenStateList,
AuxHiddenStatePacker,
)
from sglang.srt.layers.communication import k3_ar_fusion, k3_sp_collective
@@ -47,6 +49,22 @@ from sglang.srt.layers.dp_attention import (
is_allocation_symmetric,
is_dp_attention_enabled,
)
from sglang.srt.layers.layer_boundary import (
ExitRows,
append_stages,
declare_attn,
declare_ffn,
)
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layer_boundary.residual.add_norm import (
REPLACE_AT_EXIT,
NormReadout,
)
from sglang.srt.layers.layer_boundary.residual.attn_bank import (
AttnBank,
AttnBankOutputRead,
AttnBankState,
)
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.linear import (
ColumnParallelBatchedLinear,
@@ -429,8 +447,13 @@ class KimiK3MoE(nn.Module):
prefix: str = "",
layer_idx: int = 0,
alt_stream: Optional[torch.cuda.Stream] = None,
reduce_results: bool = True,
):
super().__init__()
# Whether this block completes the TP sum of a non-latent output; a
# stage boundary completes it otherwise. A latent block completes its
# sums together with the latent reduction either way.
self.reduce_results = reduce_results
hidden_size = config.hidden_size
moe_intermediate_size = config.moe_intermediate_size
moe_renormalize = config.moe_renormalize
@@ -993,6 +1016,15 @@ class KimiK3MoE(nn.Module):
value.record_stream(current_stream)
return topk_output, routed_input
def _reduce_shared(self, shared_output: torch.Tensor) -> torch.Tensor:
"""The TP-sharded shared experts' sum, completed beside a routed
output the latent reduction already completed."""
return tensor_model_parallel_all_reduce(shared_output)
def _reduce_latent_and_shared(self, buf: torch.Tensor) -> torch.Tensor:
"""The fused front's [latent | shared] pair in one collective."""
return tensor_model_parallel_all_reduce(buf)
def _reduce_latent(self, latent: torch.Tensor) -> torch.Tensor:
"""Unfused-front latent tail: TP-partial routed sums must be reduced
in latent space BEFORE the RMSNorm (sum(norm(x_i)) != norm(sum(x_i)))."""
@@ -1193,7 +1225,7 @@ class KimiK3MoE(nn.Module):
expert_output = expert_output + shared_output
# EP combine and the shared-expert subgroup have already completed
# each source token. A global TP reduction would mix token shards.
if self.tp_size > 1 and not self._ep_a2a:
if self.tp_size > 1 and not self._ep_a2a and self.reduce_results:
expert_output = tensor_model_parallel_all_reduce(expert_output)
if prefix_sum is not None:
expert_output = expert_output + prefix_sum
@@ -1234,7 +1266,7 @@ class KimiK3MoE(nn.Module):
and not self._shared_experts_tp1
and not self._shared_experts_tp_comm
):
shared_output = tensor_model_parallel_all_reduce(shared_output)
shared_output = self._reduce_shared(shared_output)
out = _add3(out, shared_output, prefix_sum)
return out
out = out if prefix_sum is None else out + prefix_sum
@@ -1443,7 +1475,7 @@ class KimiK3MoE(nn.Module):
elif k3_ar_fusion.enabled():
k3_ar_fusion.all_reduce(buf)
else:
buf = tensor_model_parallel_all_reduce(buf)
buf = self._reduce_latent_and_shared(buf)
latent = buf[:latent_numel].view(num_tokens, self.moe_hidden_size)
shared_output = buf[latent_numel:].view(num_tokens, hidden_size)
@@ -1510,6 +1542,7 @@ class KimiK3DeltaAttention(nn.Module):
prefix: str = "",
all_reduce_fusion: bool = False,
bfa_alt_stream: Optional[torch.cuda.Stream] = None,
reduce_results: bool = True,
**kwargs,
) -> None:
super().__init__()
@@ -1766,7 +1799,7 @@ class KimiK3DeltaAttention(nn.Module):
# all-reduce (which can fold the attn-res prefix add in). Only
# valid when the attn TP group is the full TP group (the fused
# comm lives there).
reduce_results=not self.all_reduce_fusion,
reduce_results=reduce_results and not self.all_reduce_fusion,
quant_config=quant_config,
parallel_group="attn_tp",
# Reduce within the attn-TP group: the default full-TP collective
@@ -2163,6 +2196,7 @@ class KimiK3MLAAttention(DeepseekV2AttentionMLA):
prefix: str = "",
alt_stream: Optional[torch.cuda.Stream] = None,
gate_alt_stream: Optional[torch.cuda.Stream] = None,
reduce_results: bool = True,
) -> None:
# ModelSlim can quantize K3 latent projections while still storing
# MLA kv_b_proj as one dense tensor; only GGUF expert packs split K/V.
@@ -2185,7 +2219,7 @@ class KimiK3MLAAttention(DeepseekV2AttentionMLA):
q_lora_rank=config.q_lora_rank,
kv_lora_rank=config.kv_lora_rank,
skip_rope=True,
reduce_results=not self.all_reduce_fusion,
reduce_results=reduce_results and not self.all_reduce_fusion,
alt_stream=alt_stream,
)
if split_gguf_kv_b:
@@ -2372,6 +2406,61 @@ class KimiK3MLAAttention(DeepseekV2AttentionMLA):
)
def _is_moe_layer(config: KimiLinearConfig, layer_idx: int) -> bool:
"""Whether a layer runs a MoE rather than the dense MLP: past the dense
prefix and on the MoE cadence."""
return (
config.is_moe
and config.num_experts is not None
and layer_idx >= config.first_k_dense_replace
and layer_idx % config.moe_layer_freq == 0
)
def _shards_moe_rows() -> bool:
"""Whether an MoE layer runs on its attention-TP token shard (SP-MoE): an
EP a2a backend with attention TP."""
backend = get_moe_a2a_backend()
return (
backend.is_megamoe()
or backend.is_deepep()
or backend.is_mooncake()
or backend.is_ascend_fuseep()
or backend.is_mori()
) and get_parallel().attn_tp_group.world_size > 1
def _fuses_attn_all_reduce(config: KimiLinearConfig) -> bool:
"""Whether the attention-residual bank asks for o_proj's all-reduce fused
with the pending residual add."""
attn_tp_size = get_parallel().attn_tp_size
return (
attn_tp_size > 1
and attn_tp_size == get_parallel().tp_size
and config.attn_res_block_size is not None
and k3_ar_fusion.enabled()
)
def _uses_stage_boundaries(config: KimiLinearConfig) -> bool:
"""Whether the layers build stage boundaries, which is the same for every
layer of a stack. The layer's own communication still runs for: a dense
MLP sharded over attention TP; an attention-residual bank whose o_proj
all-reduce is fused with the pending add, or whose MoE on its attention-TP
token shard (SP-MoE) uses K3's tuned SP collectives, which the sharded
carry also needs; and SP-MoE on batches that are not padded to a multiple
of attention TP (--disable-attn-tp-gather)."""
if get_parallel().enable_dense_mlp_attn_tp and is_dp_attention_enabled():
return False
if _fuses_attn_all_reduce(config):
return False
if not _shards_moe_rows():
return True
if not require_mlp_sync():
return False
return config.attn_res_block_size is None or not k3_sp_collective.enabled()
class KimiK3DecoderLayer(nn.Module):
"""Decoder layer carrying the K3 attention-residual stream."""
@@ -2382,24 +2471,17 @@ class KimiK3DecoderLayer(nn.Module):
quant_config: Optional[QuantizationConfig] = None,
prefix: str = "",
alt_streams: Optional[List[torch.cuda.Stream]] = None,
attn_bank: Optional[AttnBank] = None,
) -> None:
super().__init__()
self.hidden_size = config.hidden_size
self.is_moe = config.is_moe
self.layer_idx = layer_idx
self._dp_attention = is_dp_attention_enabled()
# mlp-sync (DP attention OR MoE a2a/EP) pads extend batches to
# attn_tp multiples; attention must then run on the real rows only.
self._trim_padded_attn = require_mlp_sync()
# A layer runs MoE (vs a plain dense MLP) iff it is past the dense
# prefix and on the MoE cadence — same predicate the mlp construction
# below uses.
self._is_moe_layer = (
self.is_moe
and config.num_experts is not None
and layer_idx >= config.first_k_dense_replace
and layer_idx % config.moe_layer_freq == 0
)
self._is_moe_layer = _is_moe_layer(config, layer_idx)
self._stage_boundaries = _uses_stage_boundaries(config)
# SP-MoE (EP a2a backend): o_proj defers its attention-TP reduction;
# this layer completes it as a reduce-scatter so the whole MoE region
# runs on 1/attn_tp of the rows, then all-gathers back after the tail
@@ -2407,30 +2489,12 @@ class KimiK3DecoderLayer(nn.Module):
# disappears via tp1 weights, and each rank dispatches only its shard
# through the a2a. Same under DP attention. Dense layers excluded: no
# per-token decomposition survives a token shard.
_a2a_backend = get_moe_a2a_backend()
self._sp_moe = (
(
_a2a_backend.is_megamoe()
or _a2a_backend.is_deepep()
or _a2a_backend.is_mooncake()
or _a2a_backend.is_ascend_fuseep()
or _a2a_backend.is_mori()
)
and self._is_moe_layer
and get_parallel().attn_tp_group.world_size > 1
)
self._sp_moe = self._is_moe_layer and _shards_moe_rows()
# Mutually exclusive with SP-MoE: both complete o_proj's deferred
# reduction, but SP-MoE reduce-scatters to a shard whereas the fusion
# produces the full batch in a symm buffer.
attn_tp_size = get_parallel().attn_tp_size
self.all_reduce_fusion = (
not self._sp_moe
and attn_tp_size > 1
and attn_tp_size == get_parallel().tp_size
and config.attn_res_block_size is not None
and k3_ar_fusion.enabled()
)
self.all_reduce_fusion = not self._sp_moe and _fuses_attn_all_reduce(config)
# Attention
if config.is_kda_layer(layer_idx):
@@ -2441,6 +2505,7 @@ class KimiK3DecoderLayer(nn.Module):
quant_config=quant_config,
prefix=f"{prefix}.self_attn",
all_reduce_fusion=self.all_reduce_fusion,
reduce_results=not self._stage_boundaries,
# Shared with the MLA gate stream: KDA and MLA layers never
# run concurrently within one forward, so the stream is free.
bfa_alt_stream=(alt_streams[2] if alt_streams is not None else None),
@@ -2452,6 +2517,7 @@ class KimiK3DecoderLayer(nn.Module):
quant_config=quant_config,
prefix=f"{prefix}.self_attn",
all_reduce_fusion=self.all_reduce_fusion,
reduce_results=not self._stage_boundaries,
alt_stream=alt_streams[1] if alt_streams is not None else None,
gate_alt_stream=alt_streams[2] if alt_streams is not None else None,
)
@@ -2468,6 +2534,7 @@ class KimiK3DecoderLayer(nn.Module):
layer_idx=layer_idx,
prefix=f"{prefix}.mlp",
alt_stream=alt_streams[0] if alt_streams is not None else None,
reduce_results=not self._stage_boundaries,
)
else:
self.mlp = KimiK3MLP(
@@ -2475,6 +2542,7 @@ class KimiK3DecoderLayer(nn.Module):
intermediate_size=config.intermediate_size,
hidden_act=config.hidden_act,
quant_config=quant_config,
reduce_results=not self._stage_boundaries,
prefix=f"{prefix}.mlp",
activation_situ_beta=config.activation_situ_beta,
activation_situ_linear_beta=config.activation_situ_linear_beta,
@@ -2538,6 +2606,68 @@ class KimiK3DecoderLayer(nn.Module):
o_proj.forward = _sp_o_proj_forward
# A latent MoE on this rank's rows adds the residual in its tail add and
# writes the next stream itself; one gathered over attention DP returns
# its output to this rank's tokens first.
self._ffn_writes_stream = (
self._stage_boundaries
and self.use_attn_residuals
and self._is_moe_layer
and self.mlp.use_latent_moe
and not (self._dp_attention and not self.mlp._ep_a2a)
)
if self._stage_boundaries:
# Under attention DP the FFN input is read on this rank's rows
# once the attention's sum is complete, then gathered.
attn_ops = {}
ffn_ops = dict(read=NormReadout(reads_before_dp_gather=True))
if self.use_attn_residuals:
bank_ops = AttnBankState(
attn_bank,
self.self_attention_res_proj,
self.self_attention_res_norm,
self.mlp_res_proj,
self.mlp_res_norm,
writes_block=self.is_block_write_layer,
).residual_ops()
attn_ops = dict(read=bank_ops.attn_readout, update=bank_ops.attn_update)
ffn_ops = dict(
read=bank_ops.ffn_readout,
update=(
REPLACE_AT_EXIT
if self._ffn_writes_stream
else bank_ops.ffn_update
),
)
self.attn_boundary, self.ffn_boundary = append_stages(
(declare_attn(**attn_ops), self.input_layernorm),
(
declare_ffn(
**ffn_ops,
sparse=self._is_moe_layer,
next_layer_sparse=_is_moe_layer(config, layer_idx + 1),
# The TP width the dense MLP is built with.
dense_tp_size=(
None
if self._is_moe_layer
else get_group_rank_size(self.mlp.down_proj.tp_group)[1]
),
# SP-MoE runs on this rank's attention-TP shard of the
# rows; on the bank path, whose reads write the bank on
# every row, its output returns to all of them.
exit_rows=(
ExitRows.ATTENTION
if self._sp_moe and self.use_attn_residuals
else None
),
# A latent MoE completes its output sum together with
# the latent reduction its norm needs.
output_complete=self._is_moe_layer and self.mlp.use_latent_moe,
),
self.post_attention_layernorm,
),
)
def _finish_attn_reduce(
self,
attn_out: torch.Tensor,
@@ -2657,6 +2787,7 @@ class KimiK3DecoderLayer(nn.Module):
zero_allocator: BumpAllocator,
input_sharded: bool = False,
keep_sharded: bool = False,
capture_gathered: Optional[AuxHiddenStateAccumulator] = None,
) -> tuple[torch.Tensor, Optional[torch.Tensor], bool]:
if attn_res is not None:
return self._forward_attn_residual(
@@ -2671,6 +2802,25 @@ class KimiK3DecoderLayer(nn.Module):
)
assert not input_sharded
if self._stage_boundaries:
hidden_states = self.attn_boundary.prepare(
hidden_states, forward_batch, capture_gathered=capture_gathered
)
hidden_states = self._run_self_attn(
hidden_states, positions, forward_batch, zero_allocator
)
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
hidden_states = self.mlp(
hidden_states,
prefix_sum=(
residual_batch.written_residual(forward_batch)
if self._ffn_writes_stream
else None
),
)
return self.ffn_boundary.finish(hidden_states, forward_batch), None, False
# Standard residual path
if residual is None:
residual = hidden_states
@@ -2844,6 +2994,7 @@ class KimiK3LinearModel(nn.Module):
self.dspark_layers_to_capture: Optional[list[int]] = None
self._dp_attention = is_dp_attention_enabled()
self._trim_padded_attn = require_mlp_sync()
self._stage_boundaries = _uses_stage_boundaries(config)
if self.pp_group.is_first_rank:
embedding_quant_config = (
@@ -2870,6 +3021,13 @@ class KimiK3LinearModel(nn.Module):
# Disable on HIP code path.
self.alt_streams = None if _is_hip else [torch.cuda.Stream() for _ in range(3)]
# The attention-residual bank of one forward, which every layer's
# stage boundaries read.
self.attn_bank = (
AttnBank()
if self._stage_boundaries and config.attn_res_block_size is not None
else None
)
self.layers, self.start_layer, self.end_layer = make_pp_layers(
config.num_hidden_layers,
lambda idx, prefix: KimiK3DecoderLayer(
@@ -2878,6 +3036,7 @@ class KimiK3LinearModel(nn.Module):
quant_config=quant_config,
prefix=prefix,
alt_streams=self.alt_streams,
attn_bank=self.attn_bank,
),
prefix=f"{prefix}.layers",
)
@@ -2895,6 +3054,16 @@ class KimiK3LinearModel(nn.Module):
quant_config=None,
prefix=f"{prefix}.output_attn_res_proj",
)
self._final_read = (
self.norm
if self.attn_bank is None
else AttnBankOutputRead(
self.attn_bank,
self.output_attn_res_proj,
self.output_attn_res_norm,
self.norm,
)
)
else:
self.norm = PPMissingLayer()
@@ -2906,16 +3075,29 @@ class KimiK3LinearModel(nn.Module):
inputs_embeds: torch.Tensor | None = None,
pp_proxy_tensors: Optional[PPProxyTensors] = None,
) -> torch.Tensor:
residual = None
if get_parallel().pp_group.is_first_rank:
if inputs_embeds is not None:
hidden_states = inputs_embeds
else:
hidden_states = self.embed_tokens(input_ids)
residual = None
if self._stage_boundaries:
residual_batch.start(forward_batch)
elif self.attn_bank is not None:
# The bank's wire carries the stream head written: the sending rank
# folds it in before the handoff.
hidden_states = residual_batch.set_written(
pp_proxy_tensors["hidden_states"], forward_batch
)
elif self._stage_boundaries:
hidden_states = self.layers[self.start_layer].attn_boundary.from_pp(
pp_proxy_tensors, forward_batch
)
else:
assert pp_proxy_tensors is not None
hidden_states = pp_proxy_tensors["hidden_states"]
residual = pp_proxy_tensors["residual"]
if self.config.attn_res_block_size is None:
residual = pp_proxy_tensors["residual"]
if TYPE_CHECKING:
assert isinstance(hidden_states, torch.Tensor)
assert isinstance(residual, torch.Tensor | None)
@@ -2946,12 +3128,18 @@ class KimiK3LinearModel(nn.Module):
attn_res = None
if self.config.attn_res_block_size is not None:
attn_res_block_num = _cdiv(self.end_layer, self.config.attn_res_block_size)
attn_res = AttnResidual(
hidden_states,
attn_res_block_num,
block_residual=residual,
# A later pipeline rank inherits the rows the earlier ones banked.
inherited = (
None
if self.pp_group.is_first_rank
else pp_proxy_tensors["attn_res_bank"]
)
residual = None
if self.attn_bank is not None:
self.attn_bank.open(hidden_states, attn_res_block_num, inherited)
else:
attn_res = AttnResidual(
hidden_states, attn_res_block_num, block_residual=inherited
)
# Carry the raw residual stream as a token shard across consecutive
# SP-MoE layers. PP transfer and dspark capture require full tensors,
@@ -2970,18 +3158,28 @@ class KimiK3LinearModel(nn.Module):
forward_batch, len(self.dspark_layers_to_capture)
)
if packs_aux
else []
else AuxHiddenStateList()
)
captures = self.dspark_layers_to_capture or ()
# On the standard residual path the stream a layer hands on is what the
# next layer's attention input reads, captured there on every row (the
# final norm's, after the last layer). The bank's is the mixture the
# next read would form, captured after the layer.
captures_at_input = self._stage_boundaries and self.attn_bank is None
if (
self.dspark_layers_to_capture is not None
and not self.pp_group.is_first_rank
):
if "dspark_hidden_states" in pp_proxy_tensors.tensors:
aux_hidden_states.append(pp_proxy_tensors["dspark_hidden_states"])
if self.start_layer - 1 in self.dspark_layers_to_capture:
if self.start_layer - 1 in captures and not captures_at_input:
aux_hidden_states.append(
self._dspark_capture_stream(
self.start_layer - 1, hidden_states, residual, attn_res
self.start_layer - 1,
hidden_states,
residual,
attn_res,
forward_batch,
)
)
for i in range(self.start_layer, self.end_layer):
@@ -2998,16 +3196,38 @@ class KimiK3LinearModel(nn.Module):
zero_allocator=zero_allocator,
input_sharded=sp_sharded,
keep_sharded=sp_attn_res,
capture_gathered=(
aux_hidden_states
if captures_at_input and i - 1 in captures
else None
),
)
if (
self.dspark_layers_to_capture is not None
and i in self.dspark_layers_to_capture
not captures_at_input
and i in captures
and (i + 1 < self.end_layer or self.pp_group.is_last_rank)
):
aux_hidden_states.append(
self._dspark_capture_stream(i, hidden_states, residual, attn_res)
self._dspark_capture_stream(
i, hidden_states, residual, attn_res, forward_batch
)
)
if not self.pp_group.is_last_rank and self._stage_boundaries:
if self.attn_bank is not None:
# The wire carries the stream head beside the bank.
hidden_states = residual_batch.fold(hidden_states, forward_batch)
proxy_tensors = residual_batch.to_pp(hidden_states, forward_batch)
if self.attn_bank is not None:
proxy_tensors.tensors["attn_res_bank"] = (
self.attn_bank.require().block_residual
)
self.attn_bank.close()
if aux_hidden_states:
proxy_tensors.tensors["dspark_hidden_states"] = torch.cat(
aux_hidden_states, dim=-1
)
return proxy_tensors
if not self.pp_group.is_last_rank:
assert not sp_sharded
if attn_res is not None:
@@ -3015,15 +3235,33 @@ class KimiK3LinearModel(nn.Module):
# Materialize the delayed MLP add: the wire carries the
# full stream head (bit-identical to the fused fold).
hidden_states = residual + hidden_states
residual = attn_res.block_residual # raw bank across ranks
proxy_tensors = {"hidden_states": hidden_states, "residual": residual}
proxy_tensors = {
"hidden_states": hidden_states,
"attn_res_bank": attn_res.block_residual,
}
else:
proxy_tensors = {"hidden_states": hidden_states, "residual": residual}
if aux_hidden_states:
proxy_tensors["dspark_hidden_states"] = torch.cat(
aux_hidden_states, dim=-1
)
return PPProxyTensors(proxy_tensors)
if hidden_states.shape[0] != 0:
if self._stage_boundaries:
hidden_states = residual_batch.final_norm(
hidden_states,
forward_batch,
self._final_read,
capture=(
aux_hidden_states.capture
if captures_at_input and self.end_layer - 1 in captures
else None
),
skip_empty=True,
)
if self.attn_bank is not None:
self.attn_bank.close()
elif hidden_states.shape[0] != 0:
if attn_res is not None:
# ---- Final aggregation (output side, folds delayed add) ----
if sp_sharded:
@@ -3081,10 +3319,15 @@ class KimiK3LinearModel(nn.Module):
hidden_states: torch.Tensor,
residual: Optional[torch.Tensor],
attn_res: Optional[AttnResidual],
forward_batch: ForwardBatch,
) -> torch.Tensor:
"""Stream value after `layer_idx`: the pre-norm mixture its next
consumer would compute (next layer's attention side; output side
for the last layer)."""
if self._stage_boundaries:
# The bank path; the standard one captures at the next input.
hidden_states = residual_batch.snapshot(hidden_states, forward_batch)
attn_res, residual = self.attn_bank.require(), None
if attn_res is None:
return hidden_states if residual is None else hidden_states + residual
if residual is not None: