[Refactor] Run Kimi K3 through stage boundaries (#42894)
This commit is contained in:
@@ -12,6 +12,8 @@ _RSV_ARCHITECTURES = {
|
||||
"Step3VLForConditionalGeneration",
|
||||
}
|
||||
_AR_ARCHITECTURES = {
|
||||
"KimiK3ForConditionalGeneration",
|
||||
"KimiK3LinearForCausalLM",
|
||||
"Qwen3ForCausalLM",
|
||||
"Qwen3ForSequenceClassification",
|
||||
"Qwen3Model",
|
||||
|
||||
@@ -61,22 +61,25 @@ def handle_deprecated_dp_attention(server_args: Any):
|
||||
)
|
||||
|
||||
|
||||
def _boundary_parallelism_overrides(cfg, model_type: str) -> dict:
|
||||
def _boundary_parallelism_overrides(cfg, model_type: str, architecture: str) -> dict:
|
||||
"""Resolve the token-row modes the model's forward actually enters."""
|
||||
nemotron = model_type in ("nemotron_h", "nemotron_h_puzzle")
|
||||
longcat = model_type == "longcat_flash"
|
||||
kimi_k3 = architecture in (
|
||||
"KimiK3ForConditionalGeneration",
|
||||
"KimiK3LinearForCausalLM",
|
||||
)
|
||||
if nemotron and cfg.attn_cp_size > 1:
|
||||
raise ValueError("Nemotron-H does not support --attn-cp-size > 1")
|
||||
if longcat and cfg.enable_prefill_cp and cfg.attn_cp_size > 1:
|
||||
raise ValueError(
|
||||
"LongCat-Flash does not support --enable-prefill-cp with --attn-cp-size > 1"
|
||||
)
|
||||
if (nemotron or longcat) and cfg.enable_attn_tp_input_scattered:
|
||||
# Neither model enters the input-scattered attention scope. Preserve
|
||||
# their existing ordinary execution and make the effective flag explicit.
|
||||
if (nemotron or longcat or kimi_k3) and cfg.enable_attn_tp_input_scattered:
|
||||
# None of these models enters the input-scattered attention scope.
|
||||
logger.warning(
|
||||
"Disabling input-scattered attention for %s: its forward does not enter that scope",
|
||||
model_type,
|
||||
architecture,
|
||||
)
|
||||
return {"enable_attn_tp_input_scattered": False}
|
||||
return {}
|
||||
@@ -99,7 +102,7 @@ def handle_context_parallelism(server_args: Any):
|
||||
server_args,
|
||||
"boundary_parallelism",
|
||||
**_boundary_parallelism_overrides(
|
||||
cfg, model_config.hf_text_config.model_type
|
||||
cfg, model_config.hf_text_config.model_type, model_arch
|
||||
),
|
||||
)
|
||||
if (
|
||||
|
||||
@@ -96,6 +96,7 @@ def _resolve_ffn(
|
||||
read=NORM_READOUT,
|
||||
update=PLAIN_ADD,
|
||||
dense_tp_size=None,
|
||||
output_complete=False,
|
||||
):
|
||||
parallel = get_parallel()
|
||||
if dense_tp_size not in (None, 1, parallel.tp_size):
|
||||
@@ -167,7 +168,7 @@ def _resolve_ffn(
|
||||
)
|
||||
)
|
||||
# An FFN that writes the next stream itself hands on a complete output.
|
||||
complete = on_rank_rows or update is REPLACE_AT_EXIT
|
||||
complete = on_rank_rows or update is REPLACE_AT_EXIT or output_complete
|
||||
produced = (
|
||||
OutputContract(rows, update=update, transform=output_transform)
|
||||
if complete
|
||||
@@ -222,6 +223,9 @@ class StageDeclaration:
|
||||
exit_rows: Required FFN output rows at the layer or branch exit.
|
||||
writes_at_handoff: Whether this FFN writes its output into the residual
|
||||
at its exit because the next stage is on another pipeline rank.
|
||||
output_complete: Whether the FFN's compute completes its own output
|
||||
sum, so the exit owes none. For an FFN whose output sum is fused
|
||||
with a reduction its computation needs anyway.
|
||||
previous: Declaration whose output this stage consumes, as the stack
|
||||
records it; across pipeline ranks it is built locally.
|
||||
prepared_from: Declaration whose already-read input a branch reuses.
|
||||
@@ -242,6 +246,7 @@ class StageDeclaration:
|
||||
dense_tp_size: Optional[int] = None
|
||||
exit_rows: Optional[ExitRows] = None
|
||||
writes_at_handoff: bool = False
|
||||
output_complete: bool = False
|
||||
# Only declarations participate in construction, never executable stages.
|
||||
previous: Optional[StageDeclaration] = None
|
||||
prepared_from: Optional[StageDeclaration] = None
|
||||
@@ -329,6 +334,7 @@ def declare_ffn(
|
||||
next_layer_sparse=False,
|
||||
dense_tp_size=None,
|
||||
exit_rows=None,
|
||||
output_complete=False,
|
||||
):
|
||||
"""Declare a dense or MoE FFN independently of its compute module.
|
||||
|
||||
@@ -343,6 +349,8 @@ def declare_ffn(
|
||||
compute, or the full TP size.
|
||||
exit_rows: Explicit output-row requirement; otherwise derived from
|
||||
the adjacent FFN kinds and TBO configuration.
|
||||
output_complete: Whether the compute completes its own output sum,
|
||||
fused with a reduction it needs anyway; the exit then owes none.
|
||||
|
||||
Returns:
|
||||
A StageDeclaration with no norm, tensors or execution plan.
|
||||
@@ -355,6 +363,7 @@ def declare_ffn(
|
||||
output_transform=output_transform,
|
||||
dense_tp_size=dense_tp_size,
|
||||
exit_rows=exit_rows or tbo_exit_rows(sparse, next_layer_sparse),
|
||||
output_complete=output_complete,
|
||||
)
|
||||
|
||||
|
||||
@@ -378,6 +387,7 @@ def _resolve_stage(stage, variant, following=None):
|
||||
read=stage.read,
|
||||
update=stage.update,
|
||||
dense_tp_size=stage.dense_tp_size,
|
||||
output_complete=stage.output_complete,
|
||||
)
|
||||
if resolve_exit_rows(stage.exit_rows) is ExitRows.ATTENTION:
|
||||
returned = attention
|
||||
|
||||
@@ -343,7 +343,7 @@ def _update_read(
|
||||
|
||||
def _attn_tp_reduce_scatter_update_read(
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
residual: Optional[torch.Tensor],
|
||||
forward_batch: ForwardBatch,
|
||||
norm: torch.nn.Module,
|
||||
*,
|
||||
@@ -353,7 +353,7 @@ def _attn_tp_reduce_scatter_update_read(
|
||||
update: ResidualUpdate = PLAIN_ADD,
|
||||
):
|
||||
hidden_states = attn_tp_reduce_scatter(hidden_states)
|
||||
if scatters_residual:
|
||||
if scatters_residual and residual is not None:
|
||||
residual = update.slice_residual_attn_tp(residual)
|
||||
return read.update_and_read(update, hidden_states, residual, norm)
|
||||
|
||||
|
||||
@@ -0,0 +1,223 @@
|
||||
# Copyright 2023-2024 SGLang Team
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ==============================================================================
|
||||
"""A residual read that aggregates a bank of snapshots of itself."""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Optional
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.layers.attn_residual import AttnResidual
|
||||
from sglang.srt.layers.layer_boundary.residual import LayerResidualOps
|
||||
from sglang.srt.layers.layer_boundary.residual.add_norm import PLAIN_ADD
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
|
||||
|
||||
class AttnBank:
|
||||
"""The snapshot bank of one layer-stack invocation.
|
||||
|
||||
Every layer's reads aggregate the same bank, so the bank outlives a layer
|
||||
while the reads that consume it are per-layer. The stack's entry opens one
|
||||
and the layers reach it through this holder; a pipeline rank that did not
|
||||
open the stack inherits the rows the previous rank banked.
|
||||
"""
|
||||
|
||||
__slots__ = ("current",)
|
||||
|
||||
def __init__(self):
|
||||
self.current = None
|
||||
|
||||
def open(self, hidden_states, block_num, inherited=None) -> AttnResidual:
|
||||
self.current = AttnResidual(hidden_states, block_num, inherited)
|
||||
return self.current
|
||||
|
||||
def close(self) -> None:
|
||||
self.current = None
|
||||
|
||||
def require(self) -> AttnResidual:
|
||||
if self.current is None:
|
||||
raise RuntimeError("open the attention-residual bank before a stage")
|
||||
return self.current
|
||||
|
||||
|
||||
def _bank_rows(bank: AttnResidual, hidden_states) -> Optional[slice]:
|
||||
"""The bank rows that go with ``hidden_states``: all of them, or this
|
||||
rank's contiguous attention-TP shard of them, the rows a reduce-scatter
|
||||
over attention TP leaves it."""
|
||||
total, rows = bank.block_residual.shape[0], hidden_states.shape[0]
|
||||
if rows == total:
|
||||
return None
|
||||
parallel = get_parallel()
|
||||
if rows * parallel.attn_tp_size != total:
|
||||
raise RuntimeError(
|
||||
f"{rows} rows are neither the bank's {total} rows nor an "
|
||||
f"attention-TP shard of them over {parallel.attn_tp_size} ranks"
|
||||
)
|
||||
rank = parallel.attn_tp_rank
|
||||
return slice(rank * rows, (rank + 1) * rows)
|
||||
|
||||
|
||||
@dataclass
|
||||
class AttnBankState:
|
||||
"""One layer's pair of reads into the bank an `AttnBank` holds.
|
||||
|
||||
A read aggregates the banked snapshots with the running residual and
|
||||
normalizes the result; the update that precedes it is an ordinary add, so
|
||||
both stages declare `PLAIN_ADD` and each read folds the pending add into
|
||||
its aggregation kernel. A read on this rank's attention-TP shard of the
|
||||
rows aggregates the matching shard of the bank. A write layer also
|
||||
snapshots the residual it just formed into the bank's next row, where it
|
||||
now lives: the running residual restarts empty, and the next contribution
|
||||
alone is the new head."""
|
||||
|
||||
bank: AttnBank
|
||||
attn_score_proj: torch.nn.Module
|
||||
attn_score_norm: torch.nn.Module
|
||||
ffn_score_proj: torch.nn.Module
|
||||
ffn_score_norm: torch.nn.Module
|
||||
writes_block: bool = False
|
||||
|
||||
def _aggregate(
|
||||
self, contribution, residual, norm, *, score_proj, score_norm, write
|
||||
):
|
||||
bank = self.bank.require()
|
||||
return bank.forward(
|
||||
contribution,
|
||||
residual,
|
||||
score_proj,
|
||||
score_norm,
|
||||
norm,
|
||||
rows=_bank_rows(bank, contribution),
|
||||
write=write,
|
||||
)
|
||||
|
||||
def read_attn_input(self, residual, norm):
|
||||
# No pending add: the residual alone is the head (the stack's input,
|
||||
# or the residual of a stream that was written).
|
||||
return self._attn_aggregate(residual, None, norm)
|
||||
|
||||
def update_and_read_attn_input(self, contribution, residual, norm):
|
||||
return self._attn_aggregate(contribution, residual, norm)
|
||||
|
||||
def _attn_aggregate(self, contribution, residual, norm):
|
||||
normed, residual = self._aggregate(
|
||||
contribution,
|
||||
residual,
|
||||
norm,
|
||||
score_proj=self.attn_score_proj,
|
||||
score_norm=self.attn_score_norm,
|
||||
write=self.writes_block,
|
||||
)
|
||||
return normed, None if self.writes_block else residual
|
||||
|
||||
def update_and_read_ffn_input(self, contribution, residual, norm):
|
||||
return self._aggregate(
|
||||
contribution,
|
||||
residual,
|
||||
norm,
|
||||
score_proj=self.ffn_score_proj,
|
||||
score_norm=self.ffn_score_norm,
|
||||
write=False,
|
||||
)
|
||||
|
||||
def residual_ops(self) -> LayerResidualOps:
|
||||
return LayerResidualOps(
|
||||
attn_readout=_AttnReadout(self),
|
||||
attn_update=PLAIN_ADD,
|
||||
ffn_readout=_FfnReadout(self),
|
||||
ffn_update=PLAIN_ADD,
|
||||
)
|
||||
|
||||
|
||||
class _BankReadout:
|
||||
"""A read that mixes the banked snapshots into the residual's norm, so it
|
||||
is not the residual's plain norm and no add+norm kernel may claim it. It
|
||||
reads this rank's own rows, which the bank holds, before any DP gather."""
|
||||
|
||||
is_plain_norm = False
|
||||
reads_before_dp_gather = True
|
||||
|
||||
def __init__(self, state: AttnBankState):
|
||||
self.state = state
|
||||
|
||||
def init_residual(self, hidden_states):
|
||||
return hidden_states
|
||||
|
||||
def _reject(self, update, quant_format):
|
||||
if quant_format:
|
||||
raise NotImplementedError(
|
||||
f"an attention-residual bank read in {quant_format=}"
|
||||
)
|
||||
if update is not None and not update.is_plain_add:
|
||||
# The read folds the pending add into its aggregation, which only
|
||||
# reproduces an ordinary add.
|
||||
raise NotImplementedError(
|
||||
f"an attention-residual bank read after {update=}"
|
||||
)
|
||||
|
||||
|
||||
class _AttnReadout(_BankReadout):
|
||||
"""The attention input: the bank aggregation and this layer's input norm.
|
||||
A write layer snapshots the residual this read forms."""
|
||||
|
||||
def read(self, residual, norm, quant_format="", post_residual_addition=None):
|
||||
self._reject(None, quant_format)
|
||||
if post_residual_addition is not None:
|
||||
raise NotImplementedError(
|
||||
"an attention-residual bank read with a separate addition"
|
||||
)
|
||||
return self.state.read_attn_input(residual, norm)
|
||||
|
||||
def update_and_read(self, update, hidden_states, residual, norm, **kwargs):
|
||||
self._reject(update, kwargs.get("quant_format", ""))
|
||||
return self.state.update_and_read_attn_input(hidden_states, residual, norm)
|
||||
|
||||
|
||||
class _FfnReadout(_BankReadout):
|
||||
"""The FFN input: the attention output's add folded into the bank
|
||||
aggregation, and this layer's post-attention norm."""
|
||||
|
||||
def read(self, residual, norm, quant_format="", post_residual_addition=None):
|
||||
# The attention read of a write layer leaves the residual empty, so the
|
||||
# attention output alone is the head.
|
||||
self._reject(None, quant_format)
|
||||
return self.state.update_and_read_ffn_input(residual, None, norm)
|
||||
|
||||
def update_and_read(self, update, hidden_states, residual, norm, **kwargs):
|
||||
self._reject(update, kwargs.get("quant_format", ""))
|
||||
return self.state.update_and_read_ffn_input(hidden_states, residual, norm)
|
||||
|
||||
|
||||
class AttnBankOutputRead:
|
||||
"""The layer stack's terminal read: the bank aggregation with the output
|
||||
side's scoring parameters, then the final norm. A callable final norm for
|
||||
`residual_batch.final_norm`."""
|
||||
|
||||
def __init__(self, bank: AttnBank, score_proj, score_norm, norm):
|
||||
self.bank = bank
|
||||
self.score_proj = score_proj
|
||||
self.score_norm = score_norm
|
||||
self.norm = norm
|
||||
|
||||
def __call__(self, hidden_states, residual=None):
|
||||
bank = self.bank.require()
|
||||
normed, head = bank.forward(
|
||||
hidden_states,
|
||||
residual,
|
||||
self.score_proj,
|
||||
self.score_norm,
|
||||
self.norm,
|
||||
rows=_bank_rows(bank, hidden_states),
|
||||
)
|
||||
return normed if residual is None else (normed, head)
|
||||
@@ -106,6 +106,8 @@ def final_norm(
|
||||
)
|
||||
return hidden_states
|
||||
if skip_empty and hidden_states.shape[0] == 0:
|
||||
if capture is not None:
|
||||
capture(hidden_states)
|
||||
return hidden_states
|
||||
return access.final_norm_pair(
|
||||
hidden_states, residual, layernorm, capture, **read_kwargs
|
||||
|
||||
@@ -131,6 +131,10 @@ class DynamicChunkSizer:
|
||||
return predicted_size
|
||||
|
||||
def _profile_prefill_latency(self) -> Tuple[List[int], List[float]]:
|
||||
from sglang.srt.model_executor.runner_utils.buffers import (
|
||||
allocate_pp_proxy_tensors,
|
||||
)
|
||||
|
||||
seq_lens: List[int] = []
|
||||
latencies: List[float] = []
|
||||
model_runner = self.model_runner
|
||||
@@ -193,28 +197,18 @@ class DynamicChunkSizer:
|
||||
batch.global_num_tokens = global_num_tokens
|
||||
batch.global_num_tokens_for_logprob = global_num_tokens
|
||||
|
||||
hs = (
|
||||
getattr(model_config, "hc_hidden_size", None)
|
||||
or model_config.hidden_size
|
||||
)
|
||||
proxy_tensors = {
|
||||
"hidden_states": torch.zeros(
|
||||
(current_seq_len, hs),
|
||||
# Keyed and shaped like the runner's PP buffers.
|
||||
with torch.device(self.device):
|
||||
proxy_tensors = allocate_pp_proxy_tensors(
|
||||
max_num_tokens=current_seq_len,
|
||||
max_hidden_tokens=current_seq_len,
|
||||
hidden_size=model_config.hidden_size,
|
||||
dtype=model_config.dtype,
|
||||
device=self.device,
|
||||
),
|
||||
"residual": torch.zeros(
|
||||
(current_seq_len, model_config.hidden_size),
|
||||
dtype=model_config.dtype,
|
||||
device=self.device,
|
||||
),
|
||||
}
|
||||
pp_proxy_topk_size = model_runner.get_pp_proxy_topk_size()
|
||||
if pp_proxy_topk_size is not None:
|
||||
proxy_tensors["topk_indices"] = torch.zeros(
|
||||
(current_seq_len, pp_proxy_topk_size),
|
||||
dtype=torch.int32,
|
||||
device=self.device,
|
||||
hc_hidden_size=getattr(model_config, "hc_hidden_size", None),
|
||||
pp_proxy_topk_size=model_runner.get_pp_proxy_topk_size(),
|
||||
pp_proxy_residual_num_blocks=(
|
||||
model_runner.get_pp_proxy_residual_num_blocks()
|
||||
),
|
||||
)
|
||||
|
||||
pp_proxy = PPProxyTensors(proxy_tensors)
|
||||
|
||||
@@ -138,15 +138,17 @@ def _allocate_decode_buffers(
|
||||
pp_proxy_tensors = {
|
||||
"hidden_states": torch.zeros((max_num_token, hs), dtype=dtype),
|
||||
}
|
||||
if not is_mhc:
|
||||
# Only Kimi K3 supplies num_blocks: its PP bank is token-major
|
||||
# [T, blocks, H]. Other models use [T, H].
|
||||
residual_shape = (
|
||||
(max_num_token, pp_proxy_residual_num_blocks, hidden_size)
|
||||
if pp_proxy_residual_num_blocks is not None
|
||||
else (max_num_token, hidden_size)
|
||||
if pp_proxy_residual_num_blocks is not None:
|
||||
# Only Kimi K3 supplies num_blocks: its attention-residual bank
|
||||
# is token-major [T, blocks, H] and takes the residual's place.
|
||||
pp_proxy_tensors["attn_res_bank"] = torch.zeros(
|
||||
(max_num_token, pp_proxy_residual_num_blocks, hidden_size),
|
||||
dtype=dtype,
|
||||
)
|
||||
elif not is_mhc:
|
||||
pp_proxy_tensors["residual"] = torch.zeros(
|
||||
(max_num_token, hidden_size), dtype=dtype
|
||||
)
|
||||
pp_proxy_tensors["residual"] = torch.zeros(residual_shape, dtype=dtype)
|
||||
if pp_proxy_topk_size is not None:
|
||||
pp_proxy_tensors["topk_indices"] = torch.zeros(
|
||||
(max_num_token, pp_proxy_topk_size), dtype=torch.int32
|
||||
|
||||
@@ -64,7 +64,7 @@ def _grouped_foreach_copy_(dsts: List[torch.Tensor], srcs: List[torch.Tensor]) -
|
||||
foreach_copy(group_dsts, group_srcs)
|
||||
|
||||
|
||||
def _allocate_pp_proxy_tensors(
|
||||
def allocate_pp_proxy_tensors(
|
||||
*,
|
||||
max_num_tokens: int,
|
||||
max_hidden_tokens: int,
|
||||
@@ -81,15 +81,17 @@ def _allocate_pp_proxy_tensors(
|
||||
pp_proxy_tensors = {
|
||||
"hidden_states": torch.zeros((max_hidden_tokens, pp_hidden_size), dtype=dtype),
|
||||
}
|
||||
if not is_mhc:
|
||||
# Only Kimi K3 supplies num_blocks: its PP bank is token-major
|
||||
# [T, blocks, H]. Other models use the phase-specific hidden-token bound.
|
||||
residual_shape = (
|
||||
(max_num_tokens, pp_proxy_residual_num_blocks, hidden_size)
|
||||
if pp_proxy_residual_num_blocks is not None
|
||||
else (max_hidden_tokens, hidden_size)
|
||||
if pp_proxy_residual_num_blocks is not None:
|
||||
# Only Kimi K3 supplies num_blocks: its attention-residual bank is
|
||||
# token-major [T, blocks, H] and takes the residual's place.
|
||||
pp_proxy_tensors["attn_res_bank"] = torch.zeros(
|
||||
(max_num_tokens, pp_proxy_residual_num_blocks, hidden_size), dtype=dtype
|
||||
)
|
||||
elif not is_mhc:
|
||||
# Sized by the phase-specific hidden-token bound, like hidden_states.
|
||||
pp_proxy_tensors["residual"] = torch.zeros(
|
||||
(max_hidden_tokens, hidden_size), dtype=dtype
|
||||
)
|
||||
pp_proxy_tensors["residual"] = torch.zeros(residual_shape, dtype=dtype)
|
||||
if pp_proxy_topk_size is not None:
|
||||
pp_proxy_tensors["topk_indices"] = torch.zeros(
|
||||
(max_num_tokens, pp_proxy_topk_size), dtype=torch.int32
|
||||
@@ -185,7 +187,7 @@ class DecodeInputBuffers(ForwardInputBuffers):
|
||||
)
|
||||
|
||||
pp_proxy_tensors = (
|
||||
_allocate_pp_proxy_tensors(
|
||||
allocate_pp_proxy_tensors(
|
||||
max_num_tokens=max_num_token,
|
||||
max_hidden_tokens=max_num_token,
|
||||
hidden_size=hidden_size,
|
||||
@@ -333,7 +335,7 @@ class PrefillInputBuffers(ForwardInputBuffers):
|
||||
mrope_positions = None
|
||||
|
||||
pp_proxy_tensors = (
|
||||
_allocate_pp_proxy_tensors(
|
||||
allocate_pp_proxy_tensors(
|
||||
max_num_tokens=max_num_tokens,
|
||||
max_hidden_tokens=max_num_tokens,
|
||||
hidden_size=hidden_size,
|
||||
|
||||
@@ -28,6 +28,7 @@ from sglang.srt.distributed import (
|
||||
from sglang.srt.distributed.device_communicators.pynccl_allocator import (
|
||||
use_symmetric_memory,
|
||||
)
|
||||
from sglang.srt.distributed.utils import get_group_rank_size
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.layers import zero_copy_context
|
||||
@@ -35,6 +36,7 @@ from sglang.srt.layers.activation import SiluAndMul, SituAndMul
|
||||
from sglang.srt.layers.attn_residual import AttnResidual, aggregate_stream, get_cw
|
||||
from sglang.srt.layers.aux_hidden_states import (
|
||||
AuxHiddenStateAccumulator,
|
||||
AuxHiddenStateList,
|
||||
AuxHiddenStatePacker,
|
||||
)
|
||||
from sglang.srt.layers.communication import k3_ar_fusion, k3_sp_collective
|
||||
@@ -47,6 +49,22 @@ from sglang.srt.layers.dp_attention import (
|
||||
is_allocation_symmetric,
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
ExitRows,
|
||||
append_stages,
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layer_boundary.residual.add_norm import (
|
||||
REPLACE_AT_EXIT,
|
||||
NormReadout,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.residual.attn_bank import (
|
||||
AttnBank,
|
||||
AttnBankOutputRead,
|
||||
AttnBankState,
|
||||
)
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.linear import (
|
||||
ColumnParallelBatchedLinear,
|
||||
@@ -429,8 +447,13 @@ class KimiK3MoE(nn.Module):
|
||||
prefix: str = "",
|
||||
layer_idx: int = 0,
|
||||
alt_stream: Optional[torch.cuda.Stream] = None,
|
||||
reduce_results: bool = True,
|
||||
):
|
||||
super().__init__()
|
||||
# Whether this block completes the TP sum of a non-latent output; a
|
||||
# stage boundary completes it otherwise. A latent block completes its
|
||||
# sums together with the latent reduction either way.
|
||||
self.reduce_results = reduce_results
|
||||
hidden_size = config.hidden_size
|
||||
moe_intermediate_size = config.moe_intermediate_size
|
||||
moe_renormalize = config.moe_renormalize
|
||||
@@ -993,6 +1016,15 @@ class KimiK3MoE(nn.Module):
|
||||
value.record_stream(current_stream)
|
||||
return topk_output, routed_input
|
||||
|
||||
def _reduce_shared(self, shared_output: torch.Tensor) -> torch.Tensor:
|
||||
"""The TP-sharded shared experts' sum, completed beside a routed
|
||||
output the latent reduction already completed."""
|
||||
return tensor_model_parallel_all_reduce(shared_output)
|
||||
|
||||
def _reduce_latent_and_shared(self, buf: torch.Tensor) -> torch.Tensor:
|
||||
"""The fused front's [latent | shared] pair in one collective."""
|
||||
return tensor_model_parallel_all_reduce(buf)
|
||||
|
||||
def _reduce_latent(self, latent: torch.Tensor) -> torch.Tensor:
|
||||
"""Unfused-front latent tail: TP-partial routed sums must be reduced
|
||||
in latent space BEFORE the RMSNorm (sum(norm(x_i)) != norm(sum(x_i)))."""
|
||||
@@ -1193,7 +1225,7 @@ class KimiK3MoE(nn.Module):
|
||||
expert_output = expert_output + shared_output
|
||||
# EP combine and the shared-expert subgroup have already completed
|
||||
# each source token. A global TP reduction would mix token shards.
|
||||
if self.tp_size > 1 and not self._ep_a2a:
|
||||
if self.tp_size > 1 and not self._ep_a2a and self.reduce_results:
|
||||
expert_output = tensor_model_parallel_all_reduce(expert_output)
|
||||
if prefix_sum is not None:
|
||||
expert_output = expert_output + prefix_sum
|
||||
@@ -1234,7 +1266,7 @@ class KimiK3MoE(nn.Module):
|
||||
and not self._shared_experts_tp1
|
||||
and not self._shared_experts_tp_comm
|
||||
):
|
||||
shared_output = tensor_model_parallel_all_reduce(shared_output)
|
||||
shared_output = self._reduce_shared(shared_output)
|
||||
out = _add3(out, shared_output, prefix_sum)
|
||||
return out
|
||||
out = out if prefix_sum is None else out + prefix_sum
|
||||
@@ -1443,7 +1475,7 @@ class KimiK3MoE(nn.Module):
|
||||
elif k3_ar_fusion.enabled():
|
||||
k3_ar_fusion.all_reduce(buf)
|
||||
else:
|
||||
buf = tensor_model_parallel_all_reduce(buf)
|
||||
buf = self._reduce_latent_and_shared(buf)
|
||||
|
||||
latent = buf[:latent_numel].view(num_tokens, self.moe_hidden_size)
|
||||
shared_output = buf[latent_numel:].view(num_tokens, hidden_size)
|
||||
@@ -1510,6 +1542,7 @@ class KimiK3DeltaAttention(nn.Module):
|
||||
prefix: str = "",
|
||||
all_reduce_fusion: bool = False,
|
||||
bfa_alt_stream: Optional[torch.cuda.Stream] = None,
|
||||
reduce_results: bool = True,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
@@ -1766,7 +1799,7 @@ class KimiK3DeltaAttention(nn.Module):
|
||||
# all-reduce (which can fold the attn-res prefix add in). Only
|
||||
# valid when the attn TP group is the full TP group (the fused
|
||||
# comm lives there).
|
||||
reduce_results=not self.all_reduce_fusion,
|
||||
reduce_results=reduce_results and not self.all_reduce_fusion,
|
||||
quant_config=quant_config,
|
||||
parallel_group="attn_tp",
|
||||
# Reduce within the attn-TP group: the default full-TP collective
|
||||
@@ -2163,6 +2196,7 @@ class KimiK3MLAAttention(DeepseekV2AttentionMLA):
|
||||
prefix: str = "",
|
||||
alt_stream: Optional[torch.cuda.Stream] = None,
|
||||
gate_alt_stream: Optional[torch.cuda.Stream] = None,
|
||||
reduce_results: bool = True,
|
||||
) -> None:
|
||||
# ModelSlim can quantize K3 latent projections while still storing
|
||||
# MLA kv_b_proj as one dense tensor; only GGUF expert packs split K/V.
|
||||
@@ -2185,7 +2219,7 @@ class KimiK3MLAAttention(DeepseekV2AttentionMLA):
|
||||
q_lora_rank=config.q_lora_rank,
|
||||
kv_lora_rank=config.kv_lora_rank,
|
||||
skip_rope=True,
|
||||
reduce_results=not self.all_reduce_fusion,
|
||||
reduce_results=reduce_results and not self.all_reduce_fusion,
|
||||
alt_stream=alt_stream,
|
||||
)
|
||||
if split_gguf_kv_b:
|
||||
@@ -2372,6 +2406,61 @@ class KimiK3MLAAttention(DeepseekV2AttentionMLA):
|
||||
)
|
||||
|
||||
|
||||
def _is_moe_layer(config: KimiLinearConfig, layer_idx: int) -> bool:
|
||||
"""Whether a layer runs a MoE rather than the dense MLP: past the dense
|
||||
prefix and on the MoE cadence."""
|
||||
return (
|
||||
config.is_moe
|
||||
and config.num_experts is not None
|
||||
and layer_idx >= config.first_k_dense_replace
|
||||
and layer_idx % config.moe_layer_freq == 0
|
||||
)
|
||||
|
||||
|
||||
def _shards_moe_rows() -> bool:
|
||||
"""Whether an MoE layer runs on its attention-TP token shard (SP-MoE): an
|
||||
EP a2a backend with attention TP."""
|
||||
backend = get_moe_a2a_backend()
|
||||
return (
|
||||
backend.is_megamoe()
|
||||
or backend.is_deepep()
|
||||
or backend.is_mooncake()
|
||||
or backend.is_ascend_fuseep()
|
||||
or backend.is_mori()
|
||||
) and get_parallel().attn_tp_group.world_size > 1
|
||||
|
||||
|
||||
def _fuses_attn_all_reduce(config: KimiLinearConfig) -> bool:
|
||||
"""Whether the attention-residual bank asks for o_proj's all-reduce fused
|
||||
with the pending residual add."""
|
||||
attn_tp_size = get_parallel().attn_tp_size
|
||||
return (
|
||||
attn_tp_size > 1
|
||||
and attn_tp_size == get_parallel().tp_size
|
||||
and config.attn_res_block_size is not None
|
||||
and k3_ar_fusion.enabled()
|
||||
)
|
||||
|
||||
|
||||
def _uses_stage_boundaries(config: KimiLinearConfig) -> bool:
|
||||
"""Whether the layers build stage boundaries, which is the same for every
|
||||
layer of a stack. The layer's own communication still runs for: a dense
|
||||
MLP sharded over attention TP; an attention-residual bank whose o_proj
|
||||
all-reduce is fused with the pending add, or whose MoE on its attention-TP
|
||||
token shard (SP-MoE) uses K3's tuned SP collectives, which the sharded
|
||||
carry also needs; and SP-MoE on batches that are not padded to a multiple
|
||||
of attention TP (--disable-attn-tp-gather)."""
|
||||
if get_parallel().enable_dense_mlp_attn_tp and is_dp_attention_enabled():
|
||||
return False
|
||||
if _fuses_attn_all_reduce(config):
|
||||
return False
|
||||
if not _shards_moe_rows():
|
||||
return True
|
||||
if not require_mlp_sync():
|
||||
return False
|
||||
return config.attn_res_block_size is None or not k3_sp_collective.enabled()
|
||||
|
||||
|
||||
class KimiK3DecoderLayer(nn.Module):
|
||||
"""Decoder layer carrying the K3 attention-residual stream."""
|
||||
|
||||
@@ -2382,24 +2471,17 @@ class KimiK3DecoderLayer(nn.Module):
|
||||
quant_config: Optional[QuantizationConfig] = None,
|
||||
prefix: str = "",
|
||||
alt_streams: Optional[List[torch.cuda.Stream]] = None,
|
||||
attn_bank: Optional[AttnBank] = None,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.hidden_size = config.hidden_size
|
||||
self.is_moe = config.is_moe
|
||||
self.layer_idx = layer_idx
|
||||
self._dp_attention = is_dp_attention_enabled()
|
||||
# mlp-sync (DP attention OR MoE a2a/EP) pads extend batches to
|
||||
# attn_tp multiples; attention must then run on the real rows only.
|
||||
self._trim_padded_attn = require_mlp_sync()
|
||||
# A layer runs MoE (vs a plain dense MLP) iff it is past the dense
|
||||
# prefix and on the MoE cadence — same predicate the mlp construction
|
||||
# below uses.
|
||||
self._is_moe_layer = (
|
||||
self.is_moe
|
||||
and config.num_experts is not None
|
||||
and layer_idx >= config.first_k_dense_replace
|
||||
and layer_idx % config.moe_layer_freq == 0
|
||||
)
|
||||
self._is_moe_layer = _is_moe_layer(config, layer_idx)
|
||||
self._stage_boundaries = _uses_stage_boundaries(config)
|
||||
# SP-MoE (EP a2a backend): o_proj defers its attention-TP reduction;
|
||||
# this layer completes it as a reduce-scatter so the whole MoE region
|
||||
# runs on 1/attn_tp of the rows, then all-gathers back after the tail
|
||||
@@ -2407,30 +2489,12 @@ class KimiK3DecoderLayer(nn.Module):
|
||||
# disappears via tp1 weights, and each rank dispatches only its shard
|
||||
# through the a2a. Same under DP attention. Dense layers excluded: no
|
||||
# per-token decomposition survives a token shard.
|
||||
_a2a_backend = get_moe_a2a_backend()
|
||||
self._sp_moe = (
|
||||
(
|
||||
_a2a_backend.is_megamoe()
|
||||
or _a2a_backend.is_deepep()
|
||||
or _a2a_backend.is_mooncake()
|
||||
or _a2a_backend.is_ascend_fuseep()
|
||||
or _a2a_backend.is_mori()
|
||||
)
|
||||
and self._is_moe_layer
|
||||
and get_parallel().attn_tp_group.world_size > 1
|
||||
)
|
||||
self._sp_moe = self._is_moe_layer and _shards_moe_rows()
|
||||
|
||||
# Mutually exclusive with SP-MoE: both complete o_proj's deferred
|
||||
# reduction, but SP-MoE reduce-scatters to a shard whereas the fusion
|
||||
# produces the full batch in a symm buffer.
|
||||
attn_tp_size = get_parallel().attn_tp_size
|
||||
self.all_reduce_fusion = (
|
||||
not self._sp_moe
|
||||
and attn_tp_size > 1
|
||||
and attn_tp_size == get_parallel().tp_size
|
||||
and config.attn_res_block_size is not None
|
||||
and k3_ar_fusion.enabled()
|
||||
)
|
||||
self.all_reduce_fusion = not self._sp_moe and _fuses_attn_all_reduce(config)
|
||||
|
||||
# Attention
|
||||
if config.is_kda_layer(layer_idx):
|
||||
@@ -2441,6 +2505,7 @@ class KimiK3DecoderLayer(nn.Module):
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.self_attn",
|
||||
all_reduce_fusion=self.all_reduce_fusion,
|
||||
reduce_results=not self._stage_boundaries,
|
||||
# Shared with the MLA gate stream: KDA and MLA layers never
|
||||
# run concurrently within one forward, so the stream is free.
|
||||
bfa_alt_stream=(alt_streams[2] if alt_streams is not None else None),
|
||||
@@ -2452,6 +2517,7 @@ class KimiK3DecoderLayer(nn.Module):
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.self_attn",
|
||||
all_reduce_fusion=self.all_reduce_fusion,
|
||||
reduce_results=not self._stage_boundaries,
|
||||
alt_stream=alt_streams[1] if alt_streams is not None else None,
|
||||
gate_alt_stream=alt_streams[2] if alt_streams is not None else None,
|
||||
)
|
||||
@@ -2468,6 +2534,7 @@ class KimiK3DecoderLayer(nn.Module):
|
||||
layer_idx=layer_idx,
|
||||
prefix=f"{prefix}.mlp",
|
||||
alt_stream=alt_streams[0] if alt_streams is not None else None,
|
||||
reduce_results=not self._stage_boundaries,
|
||||
)
|
||||
else:
|
||||
self.mlp = KimiK3MLP(
|
||||
@@ -2475,6 +2542,7 @@ class KimiK3DecoderLayer(nn.Module):
|
||||
intermediate_size=config.intermediate_size,
|
||||
hidden_act=config.hidden_act,
|
||||
quant_config=quant_config,
|
||||
reduce_results=not self._stage_boundaries,
|
||||
prefix=f"{prefix}.mlp",
|
||||
activation_situ_beta=config.activation_situ_beta,
|
||||
activation_situ_linear_beta=config.activation_situ_linear_beta,
|
||||
@@ -2538,6 +2606,68 @@ class KimiK3DecoderLayer(nn.Module):
|
||||
|
||||
o_proj.forward = _sp_o_proj_forward
|
||||
|
||||
# A latent MoE on this rank's rows adds the residual in its tail add and
|
||||
# writes the next stream itself; one gathered over attention DP returns
|
||||
# its output to this rank's tokens first.
|
||||
self._ffn_writes_stream = (
|
||||
self._stage_boundaries
|
||||
and self.use_attn_residuals
|
||||
and self._is_moe_layer
|
||||
and self.mlp.use_latent_moe
|
||||
and not (self._dp_attention and not self.mlp._ep_a2a)
|
||||
)
|
||||
if self._stage_boundaries:
|
||||
# Under attention DP the FFN input is read on this rank's rows
|
||||
# once the attention's sum is complete, then gathered.
|
||||
attn_ops = {}
|
||||
ffn_ops = dict(read=NormReadout(reads_before_dp_gather=True))
|
||||
if self.use_attn_residuals:
|
||||
bank_ops = AttnBankState(
|
||||
attn_bank,
|
||||
self.self_attention_res_proj,
|
||||
self.self_attention_res_norm,
|
||||
self.mlp_res_proj,
|
||||
self.mlp_res_norm,
|
||||
writes_block=self.is_block_write_layer,
|
||||
).residual_ops()
|
||||
attn_ops = dict(read=bank_ops.attn_readout, update=bank_ops.attn_update)
|
||||
ffn_ops = dict(
|
||||
read=bank_ops.ffn_readout,
|
||||
update=(
|
||||
REPLACE_AT_EXIT
|
||||
if self._ffn_writes_stream
|
||||
else bank_ops.ffn_update
|
||||
),
|
||||
)
|
||||
self.attn_boundary, self.ffn_boundary = append_stages(
|
||||
(declare_attn(**attn_ops), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(
|
||||
**ffn_ops,
|
||||
sparse=self._is_moe_layer,
|
||||
next_layer_sparse=_is_moe_layer(config, layer_idx + 1),
|
||||
# The TP width the dense MLP is built with.
|
||||
dense_tp_size=(
|
||||
None
|
||||
if self._is_moe_layer
|
||||
else get_group_rank_size(self.mlp.down_proj.tp_group)[1]
|
||||
),
|
||||
# SP-MoE runs on this rank's attention-TP shard of the
|
||||
# rows; on the bank path, whose reads write the bank on
|
||||
# every row, its output returns to all of them.
|
||||
exit_rows=(
|
||||
ExitRows.ATTENTION
|
||||
if self._sp_moe and self.use_attn_residuals
|
||||
else None
|
||||
),
|
||||
# A latent MoE completes its output sum together with
|
||||
# the latent reduction its norm needs.
|
||||
output_complete=self._is_moe_layer and self.mlp.use_latent_moe,
|
||||
),
|
||||
self.post_attention_layernorm,
|
||||
),
|
||||
)
|
||||
|
||||
def _finish_attn_reduce(
|
||||
self,
|
||||
attn_out: torch.Tensor,
|
||||
@@ -2657,6 +2787,7 @@ class KimiK3DecoderLayer(nn.Module):
|
||||
zero_allocator: BumpAllocator,
|
||||
input_sharded: bool = False,
|
||||
keep_sharded: bool = False,
|
||||
capture_gathered: Optional[AuxHiddenStateAccumulator] = None,
|
||||
) -> tuple[torch.Tensor, Optional[torch.Tensor], bool]:
|
||||
if attn_res is not None:
|
||||
return self._forward_attn_residual(
|
||||
@@ -2671,6 +2802,25 @@ class KimiK3DecoderLayer(nn.Module):
|
||||
)
|
||||
|
||||
assert not input_sharded
|
||||
if self._stage_boundaries:
|
||||
hidden_states = self.attn_boundary.prepare(
|
||||
hidden_states, forward_batch, capture_gathered=capture_gathered
|
||||
)
|
||||
hidden_states = self._run_self_attn(
|
||||
hidden_states, positions, forward_batch, zero_allocator
|
||||
)
|
||||
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
|
||||
hidden_states = self.mlp(
|
||||
hidden_states,
|
||||
prefix_sum=(
|
||||
residual_batch.written_residual(forward_batch)
|
||||
if self._ffn_writes_stream
|
||||
else None
|
||||
),
|
||||
)
|
||||
return self.ffn_boundary.finish(hidden_states, forward_batch), None, False
|
||||
|
||||
# Standard residual path
|
||||
if residual is None:
|
||||
residual = hidden_states
|
||||
@@ -2844,6 +2994,7 @@ class KimiK3LinearModel(nn.Module):
|
||||
self.dspark_layers_to_capture: Optional[list[int]] = None
|
||||
self._dp_attention = is_dp_attention_enabled()
|
||||
self._trim_padded_attn = require_mlp_sync()
|
||||
self._stage_boundaries = _uses_stage_boundaries(config)
|
||||
|
||||
if self.pp_group.is_first_rank:
|
||||
embedding_quant_config = (
|
||||
@@ -2870,6 +3021,13 @@ class KimiK3LinearModel(nn.Module):
|
||||
# Disable on HIP code path.
|
||||
self.alt_streams = None if _is_hip else [torch.cuda.Stream() for _ in range(3)]
|
||||
|
||||
# The attention-residual bank of one forward, which every layer's
|
||||
# stage boundaries read.
|
||||
self.attn_bank = (
|
||||
AttnBank()
|
||||
if self._stage_boundaries and config.attn_res_block_size is not None
|
||||
else None
|
||||
)
|
||||
self.layers, self.start_layer, self.end_layer = make_pp_layers(
|
||||
config.num_hidden_layers,
|
||||
lambda idx, prefix: KimiK3DecoderLayer(
|
||||
@@ -2878,6 +3036,7 @@ class KimiK3LinearModel(nn.Module):
|
||||
quant_config=quant_config,
|
||||
prefix=prefix,
|
||||
alt_streams=self.alt_streams,
|
||||
attn_bank=self.attn_bank,
|
||||
),
|
||||
prefix=f"{prefix}.layers",
|
||||
)
|
||||
@@ -2895,6 +3054,16 @@ class KimiK3LinearModel(nn.Module):
|
||||
quant_config=None,
|
||||
prefix=f"{prefix}.output_attn_res_proj",
|
||||
)
|
||||
self._final_read = (
|
||||
self.norm
|
||||
if self.attn_bank is None
|
||||
else AttnBankOutputRead(
|
||||
self.attn_bank,
|
||||
self.output_attn_res_proj,
|
||||
self.output_attn_res_norm,
|
||||
self.norm,
|
||||
)
|
||||
)
|
||||
else:
|
||||
self.norm = PPMissingLayer()
|
||||
|
||||
@@ -2906,16 +3075,29 @@ class KimiK3LinearModel(nn.Module):
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
pp_proxy_tensors: Optional[PPProxyTensors] = None,
|
||||
) -> torch.Tensor:
|
||||
residual = None
|
||||
if get_parallel().pp_group.is_first_rank:
|
||||
if inputs_embeds is not None:
|
||||
hidden_states = inputs_embeds
|
||||
else:
|
||||
hidden_states = self.embed_tokens(input_ids)
|
||||
residual = None
|
||||
if self._stage_boundaries:
|
||||
residual_batch.start(forward_batch)
|
||||
elif self.attn_bank is not None:
|
||||
# The bank's wire carries the stream head written: the sending rank
|
||||
# folds it in before the handoff.
|
||||
hidden_states = residual_batch.set_written(
|
||||
pp_proxy_tensors["hidden_states"], forward_batch
|
||||
)
|
||||
elif self._stage_boundaries:
|
||||
hidden_states = self.layers[self.start_layer].attn_boundary.from_pp(
|
||||
pp_proxy_tensors, forward_batch
|
||||
)
|
||||
else:
|
||||
assert pp_proxy_tensors is not None
|
||||
hidden_states = pp_proxy_tensors["hidden_states"]
|
||||
residual = pp_proxy_tensors["residual"]
|
||||
if self.config.attn_res_block_size is None:
|
||||
residual = pp_proxy_tensors["residual"]
|
||||
if TYPE_CHECKING:
|
||||
assert isinstance(hidden_states, torch.Tensor)
|
||||
assert isinstance(residual, torch.Tensor | None)
|
||||
@@ -2946,12 +3128,18 @@ class KimiK3LinearModel(nn.Module):
|
||||
attn_res = None
|
||||
if self.config.attn_res_block_size is not None:
|
||||
attn_res_block_num = _cdiv(self.end_layer, self.config.attn_res_block_size)
|
||||
attn_res = AttnResidual(
|
||||
hidden_states,
|
||||
attn_res_block_num,
|
||||
block_residual=residual,
|
||||
# A later pipeline rank inherits the rows the earlier ones banked.
|
||||
inherited = (
|
||||
None
|
||||
if self.pp_group.is_first_rank
|
||||
else pp_proxy_tensors["attn_res_bank"]
|
||||
)
|
||||
residual = None
|
||||
if self.attn_bank is not None:
|
||||
self.attn_bank.open(hidden_states, attn_res_block_num, inherited)
|
||||
else:
|
||||
attn_res = AttnResidual(
|
||||
hidden_states, attn_res_block_num, block_residual=inherited
|
||||
)
|
||||
|
||||
# Carry the raw residual stream as a token shard across consecutive
|
||||
# SP-MoE layers. PP transfer and dspark capture require full tensors,
|
||||
@@ -2970,18 +3158,28 @@ class KimiK3LinearModel(nn.Module):
|
||||
forward_batch, len(self.dspark_layers_to_capture)
|
||||
)
|
||||
if packs_aux
|
||||
else []
|
||||
else AuxHiddenStateList()
|
||||
)
|
||||
captures = self.dspark_layers_to_capture or ()
|
||||
# On the standard residual path the stream a layer hands on is what the
|
||||
# next layer's attention input reads, captured there on every row (the
|
||||
# final norm's, after the last layer). The bank's is the mixture the
|
||||
# next read would form, captured after the layer.
|
||||
captures_at_input = self._stage_boundaries and self.attn_bank is None
|
||||
if (
|
||||
self.dspark_layers_to_capture is not None
|
||||
and not self.pp_group.is_first_rank
|
||||
):
|
||||
if "dspark_hidden_states" in pp_proxy_tensors.tensors:
|
||||
aux_hidden_states.append(pp_proxy_tensors["dspark_hidden_states"])
|
||||
if self.start_layer - 1 in self.dspark_layers_to_capture:
|
||||
if self.start_layer - 1 in captures and not captures_at_input:
|
||||
aux_hidden_states.append(
|
||||
self._dspark_capture_stream(
|
||||
self.start_layer - 1, hidden_states, residual, attn_res
|
||||
self.start_layer - 1,
|
||||
hidden_states,
|
||||
residual,
|
||||
attn_res,
|
||||
forward_batch,
|
||||
)
|
||||
)
|
||||
for i in range(self.start_layer, self.end_layer):
|
||||
@@ -2998,16 +3196,38 @@ class KimiK3LinearModel(nn.Module):
|
||||
zero_allocator=zero_allocator,
|
||||
input_sharded=sp_sharded,
|
||||
keep_sharded=sp_attn_res,
|
||||
capture_gathered=(
|
||||
aux_hidden_states
|
||||
if captures_at_input and i - 1 in captures
|
||||
else None
|
||||
),
|
||||
)
|
||||
if (
|
||||
self.dspark_layers_to_capture is not None
|
||||
and i in self.dspark_layers_to_capture
|
||||
not captures_at_input
|
||||
and i in captures
|
||||
and (i + 1 < self.end_layer or self.pp_group.is_last_rank)
|
||||
):
|
||||
aux_hidden_states.append(
|
||||
self._dspark_capture_stream(i, hidden_states, residual, attn_res)
|
||||
self._dspark_capture_stream(
|
||||
i, hidden_states, residual, attn_res, forward_batch
|
||||
)
|
||||
)
|
||||
|
||||
if not self.pp_group.is_last_rank and self._stage_boundaries:
|
||||
if self.attn_bank is not None:
|
||||
# The wire carries the stream head beside the bank.
|
||||
hidden_states = residual_batch.fold(hidden_states, forward_batch)
|
||||
proxy_tensors = residual_batch.to_pp(hidden_states, forward_batch)
|
||||
if self.attn_bank is not None:
|
||||
proxy_tensors.tensors["attn_res_bank"] = (
|
||||
self.attn_bank.require().block_residual
|
||||
)
|
||||
self.attn_bank.close()
|
||||
if aux_hidden_states:
|
||||
proxy_tensors.tensors["dspark_hidden_states"] = torch.cat(
|
||||
aux_hidden_states, dim=-1
|
||||
)
|
||||
return proxy_tensors
|
||||
if not self.pp_group.is_last_rank:
|
||||
assert not sp_sharded
|
||||
if attn_res is not None:
|
||||
@@ -3015,15 +3235,33 @@ class KimiK3LinearModel(nn.Module):
|
||||
# Materialize the delayed MLP add: the wire carries the
|
||||
# full stream head (bit-identical to the fused fold).
|
||||
hidden_states = residual + hidden_states
|
||||
residual = attn_res.block_residual # raw bank across ranks
|
||||
proxy_tensors = {"hidden_states": hidden_states, "residual": residual}
|
||||
proxy_tensors = {
|
||||
"hidden_states": hidden_states,
|
||||
"attn_res_bank": attn_res.block_residual,
|
||||
}
|
||||
else:
|
||||
proxy_tensors = {"hidden_states": hidden_states, "residual": residual}
|
||||
if aux_hidden_states:
|
||||
proxy_tensors["dspark_hidden_states"] = torch.cat(
|
||||
aux_hidden_states, dim=-1
|
||||
)
|
||||
return PPProxyTensors(proxy_tensors)
|
||||
|
||||
if hidden_states.shape[0] != 0:
|
||||
if self._stage_boundaries:
|
||||
hidden_states = residual_batch.final_norm(
|
||||
hidden_states,
|
||||
forward_batch,
|
||||
self._final_read,
|
||||
capture=(
|
||||
aux_hidden_states.capture
|
||||
if captures_at_input and self.end_layer - 1 in captures
|
||||
else None
|
||||
),
|
||||
skip_empty=True,
|
||||
)
|
||||
if self.attn_bank is not None:
|
||||
self.attn_bank.close()
|
||||
elif hidden_states.shape[0] != 0:
|
||||
if attn_res is not None:
|
||||
# ---- Final aggregation (output side, folds delayed add) ----
|
||||
if sp_sharded:
|
||||
@@ -3081,10 +3319,15 @@ class KimiK3LinearModel(nn.Module):
|
||||
hidden_states: torch.Tensor,
|
||||
residual: Optional[torch.Tensor],
|
||||
attn_res: Optional[AttnResidual],
|
||||
forward_batch: ForwardBatch,
|
||||
) -> torch.Tensor:
|
||||
"""Stream value after `layer_idx`: the pre-norm mixture its next
|
||||
consumer would compute (next layer's attention side; output side
|
||||
for the last layer)."""
|
||||
if self._stage_boundaries:
|
||||
# The bank path; the standard one captures at the next input.
|
||||
hidden_states = residual_batch.snapshot(hidden_states, forward_batch)
|
||||
attn_res, residual = self.attn_bank.require(), None
|
||||
if attn_res is None:
|
||||
return hidden_states if residual is None else hidden_states + residual
|
||||
if residual is not None:
|
||||
|
||||
Reference in New Issue
Block a user