feat(kda): integrate FlashInfer prefill checkpoints (#41400)

This commit is contained in:
Khoa Pham
2026-10-08 17:16:33 -07:00
committed by GitHub
parent 1e77db3215
commit fc3385d530
14 changed files with 576 additions and 57 deletions
@@ -319,12 +319,7 @@ class AscendKDAAttnBackend(KDAAttnBackend):
Optional[torch.Tensor],
Optional[torch.Tensor],
]:
"""Apply the Ascend prefill gate contract.
The checkpoint was validated with FP32 gate activation before
``chunk_kda``. Keeping this platform override here leaves the shared
GPU model/backend paths unchanged.
"""
"""Activate raw gate and beta projections in FP32 for Ascend prefill."""
preactivated_g = fused_kda_gate_npu(
g.flatten(-2),
layer.A_log,
@@ -332,7 +327,7 @@ class AscendKDAAttnBackend(KDAAttnBackend):
gate_bias=layer.dt_bias,
lower_bound=layer.lower_bound,
)
return preactivated_g, beta, None, None
return preactivated_g, beta.float().sigmoid(), None, None
def _forward_target_verify(
self,
@@ -34,6 +34,7 @@ elif is_cpu():
causal_conv1d_update = causal_conv1d_update_cpu
from sglang.srt.model_executor.cuda_graph_config import Backend as CudaGraphBackend
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
from sglang.srt.model_executor.model_runner import ModelRunner
from sglang.srt.runtime_context import (
@@ -45,6 +46,24 @@ from sglang.srt.runtime_context import (
)
def _validate_flashinfer_kda_prefill(
*,
chunk_size: int,
enable_two_batch_overlap: bool,
prefill_cuda_graph_backend: CudaGraphBackend,
) -> None:
if chunk_size <= 0 or chunk_size % 32:
raise ValueError(
"FlashInfer KDA checkpoint interval must be positive and divisible by 32"
)
if enable_two_batch_overlap:
raise ValueError("FlashInfer KDA prefill does not support two-batch overlap")
if prefill_cuda_graph_backend == CudaGraphBackend.FULL:
raise ValueError(
"FlashInfer KDA prefill requires eager linear attention; use disabled or breakable prefill CUDA graphs"
)
class KDAKernelDispatcher:
"""Dispatches KDA kernel calls to the appropriate backend per mode."""
@@ -55,6 +74,7 @@ class KDAKernelDispatcher:
verify_backend: LinearAttnKernelBackend,
):
self.verify_backend = verify_backend
self.prefill_backend = prefill_backend
triton_kernel = TritonKDAKernel()
self.triton_kernel = triton_kernel
helion_kernel = None
@@ -86,7 +106,6 @@ class KDAKernelDispatcher:
self.decode_kernel = CuteDSLKDAKernel()
elif decode_backend.is_flashinfer():
# FlashInfer recurrent_kda: SM100 decode + MTP (target_verify).
# Prefill stays on Triton / CuTe DSL (FlashInfer has no KDA chunk kernel).
if not is_cuda():
raise ValueError("KDA FlashInfer backend requires CUDA")
from sglang.srt.layers.attention.linear.kernels.kda_flashinfer import (
@@ -145,6 +164,14 @@ class KDAKernelDispatcher:
)
self.extend_kernel = FlashKDAKernel()
elif prefill_backend.is_flashinfer():
if not is_cuda():
raise ValueError("KDA FlashInfer prefill backend requires CUDA")
from sglang.srt.layers.attention.linear.kernels.kda_flashinfer_prefill import (
FlashInferKDAPrefillKernel,
)
self.extend_kernel = FlashInferKDAPrefillKernel()
elif prefill_backend.is_cutedsl():
if not is_cuda():
raise ValueError("KDA CuTe DSL backend requires CUDA")
@@ -195,7 +222,7 @@ class KDAKernelDispatcher:
else:
raise ValueError(
f"Unsupported KDA prefill backend: {prefill_backend}. "
"KDA supports 'triton', 'helion', 'flashkda', 'cutedsl', "
"KDA supports 'triton', 'helion', 'flashinfer', 'flashkda', 'cutedsl', "
"'nvidia_kda', or 'ptx_kda' (cutedsl/nvidia_kda prefill need "
"SM100, ptx_kda SM100 or SM103)."
)
@@ -332,10 +359,11 @@ class KDAKernelDispatcher:
**kwargs,
)
def effective_extend_kernel(self, lower_bound: Optional[float]):
"""The kernel ``extend`` will actually run: safe-gate models reroute
kernels without ``supports_safe_gate`` to Triton."""
def effective_extend_kernel(self, lower_bound: Optional[float], num_tokens: int):
"""The packed FlashInfer prefill kernel requires more than one token."""
kernel = self.extend_kernel
if self.prefill_backend.is_flashinfer() and num_tokens <= 1:
return self.triton_kernel
if lower_bound is not None and not getattr(kernel, "supports_safe_gate", True):
kernel = self.triton_kernel
return kernel
@@ -353,7 +381,7 @@ class KDAKernelDispatcher:
query_start_loc: torch.Tensor,
**kwargs,
) -> tuple[torch.Tensor, torch.Tensor | None]:
kernel = self.effective_extend_kernel(kwargs.get("lower_bound"))
kernel = self.effective_extend_kernel(kwargs.get("lower_bound"), q.shape[1])
return kernel.extend(
q,
k,
@@ -429,6 +457,12 @@ class KDAAttnBackend(MambaAttnBackendBase):
self.kernel_dispatcher = KDAKernelDispatcher(
decode_backend, prefill_backend, verify_backend
)
if prefill_backend.is_flashinfer():
_validate_flashinfer_kda_prefill(
chunk_size=self.mamba_chunk_size,
enable_two_batch_overlap=get_exec().overlap.enable_two_batch_overlap,
prefill_cuda_graph_backend=get_exec().graph.cuda_graph_config.prefill.backend,
)
# One-shot; emitted at the first fused-decode interception below.
self._fused_override_notice = (
"K3 fused KDA decode engaged: --linear-attn-decode-backend "
@@ -543,6 +577,25 @@ class KDAAttnBackend(MambaAttnBackendBase):
self.forward_metadata.mamba_track_mask_indices
]
)
if (
self.kernel_dispatcher.extend_kernel.uses_state_checkpoints
and forward_batch.forward_mode.is_extend_without_speculative()
and self.forward_metadata.track_ssm_h_src is not None
and self.forward_metadata.track_ssm_h_src.numel() > 0
):
from sglang.srt.layers.attention.linear.kernels.kda_flashinfer_prefill import (
build_flashinfer_kda_checkpoint_plan,
)
assert self._has_cpu_prefill_track_metadata(forward_batch), (
"FlashInfer KDA checkpoints require host prefill tracking metadata"
)
build_flashinfer_kda_checkpoint_plan(
forward_batch,
self.forward_metadata,
self.device,
self.mamba_chunk_size,
)
def forward_decode(
self,
@@ -872,8 +925,7 @@ class KDAAttnBackend(MambaAttnBackendBase):
k = k.unflatten(-1, (-1, layer.head_k_dim)).unsqueeze(0) # n (h d) -> 1 n h d
v = v.unflatten(-1, (-1, layer.head_v_dim)).unsqueeze(0) # n (h d) -> 1 n h d
gate_was_flat = a.ndim == 3
if gate_was_flat:
if a.ndim == 3:
a = a.unflatten(-1, (-1, layer.head_k_dim))
track_ssm = self.forward_metadata.has_mamba_track_mask
@@ -892,7 +944,7 @@ class KDAAttnBackend(MambaAttnBackendBase):
# Check the kernel the dispatcher will actually run (safe-gate
# reroute included), not just the configured one.
extend_kernel = self.kernel_dispatcher.effective_extend_kernel(
layer.lower_bound
layer.lower_bound, q.shape[1]
)
assert extend_kernel.supports_track_state_snapshot, (
f"{type(extend_kernel).__name__} cannot write the fp32 track "
@@ -917,7 +969,7 @@ class KDAAttnBackend(MambaAttnBackendBase):
A_log=layer.A_log,
dt_bias=layer.dt_bias,
lower_bound=layer.lower_bound,
beta_is_raw=gate_was_flat,
beta_is_raw=True,
extend_seq_lens_cpu=forward_batch.extend_seq_lens_cpu,
extend_prefix_lens=forward_batch.extend_prefix_lens,
layer_id=layer.layer_id,
@@ -933,6 +985,13 @@ class KDAAttnBackend(MambaAttnBackendBase):
),
track_state=h_track_buf,
track_chunk_idx=(track_chunk_idx if h_track_buf is not None else None),
state_checkpoint_cu_starts=self.forward_metadata.state_checkpoint_cu_starts,
num_state_checkpoints=self.forward_metadata.num_state_checkpoints,
state_checkpoint_every_n_tokens=(
self.forward_metadata.state_checkpoint_every_n_tokens
),
state_checkpoint_indices=self.forward_metadata.state_checkpoint_indices,
track_ssm_h_batch_src=self.forward_metadata.track_ssm_h_batch_src,
)
if track_ssm:
# Snapshot the SSM state at the last track-aligned chunk boundary
@@ -1,7 +1,7 @@
"""FlashInfer KDA decode/verify wrapper.
Wraps ``flashinfer.kda_decode.recurrent_kda`` (SM100 / Blackwell). FlashInfer has
no KDA prefill kernel, so ``extend`` stays on Triton / CuTe DSL.
Wraps ``flashinfer.kda_decode.recurrent_kda`` (SM100 / Blackwell). KDA prefill
uses a separate adapter in ``kda_flashinfer_prefill``.
Contract with the Triton KDA reference:
- raw per-K gate ``a`` is activated in-kernel as
@@ -85,11 +85,7 @@ def build_fused_accept_indices(
class FlashInferKDAKernel(LinearAttnKernelBase):
"""FlashInfer KDA kernel: SM100 decode + MTP (target_verify), topk=1.
Prefill (``extend``) is intentionally not implemented -- FlashInfer ships no
KDA chunk kernel; the dispatcher keeps prefill on Triton / CuTe DSL.
"""
"""FlashInfer KDA kernel: SM100 decode + MTP (target_verify), topk=1."""
def __init__(self):
available, self._recurrent_kda = _get_flashinfer_kda_kernel()
@@ -381,5 +377,6 @@ class FlashInferKDAKernel(LinearAttnKernelBase):
def extend(self, *args, **kwargs):
raise NotImplementedError(
"FlashInferKDAKernel has no prefill kernel; keep prefill on Triton / CuTe DSL."
"FlashInferKDAKernel is decode/verify-only; use "
"FlashInferKDAPrefillKernel for prefill."
)
@@ -0,0 +1,148 @@
"""FlashInfer 0.7 CuTe DSL KDA prefill with radix-cache checkpoints."""
from __future__ import annotations
from itertools import accumulate
from typing import TYPE_CHECKING, Optional
import torch
from sglang.srt.layers.attention.linear.kernels.kernel_backend import (
LinearAttnKernelBase,
)
if TYPE_CHECKING:
from sglang.srt.layers.attention.mamba.mamba2_metadata import ForwardMetadata
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
def build_flashinfer_kda_checkpoint_plan(
forward_batch: ForwardBatch,
metadata: ForwardMetadata,
device: torch.device,
chunk_size: int,
) -> None:
"""Plan scheduler-generated tracked boundaries, asserting their validity.
Host track metadata follows build_prefill_track_plan's unaligned-row order.
"""
extend_lens = forward_batch.extend_seq_lens_cpu
track_lens = forward_batch.mamba_track_seqlens_cpu
prefix_lens = forward_batch.extend_prefix_lens_cpu
track_mask = forward_batch.mamba_prefill_track_mask_cpu
checkpoint_counts = [length // chunk_size for length in extend_lens]
checkpoint_starts = list(accumulate(checkpoint_counts, initial=0))
# NOTE(kpham-sgl): FlashInfer GDN writes all periodic checkpoints for SGLang
# to select from; FlashInfer KDA writes only mapped destinations (-1 skips).
checkpoint_destinations = [-1] * checkpoint_starts[-1]
num_tracked_checkpoints = 0
for row, tracked in enumerate(track_mask):
if not tracked:
continue
relative_track_len = track_lens[row] - prefix_lens[row]
if relative_track_len % chunk_size == 0:
continue
completed_chunks = relative_track_len // chunk_size
assert 0 < completed_chunks <= checkpoint_counts[row], (
f"Invalid KDA checkpoint metadata: row={row}, "
f"relative_track_len={relative_track_len}, extend_len={extend_lens[row]}, "
f"chunk_size={chunk_size}"
)
checkpoint_destinations[checkpoint_starts[row] + completed_chunks - 1] = (
num_tracked_checkpoints
)
num_tracked_checkpoints += 1
assert num_tracked_checkpoints == metadata.track_ssm_h_batch_src.numel(), (
"KDA checkpoint rows disagree with the prefill tracking metadata"
)
metadata.state_checkpoint_cu_starts = torch.tensor(
checkpoint_starts, dtype=torch.int64, device=device
)
metadata.num_state_checkpoints = num_tracked_checkpoints
metadata.state_checkpoint_every_n_tokens = chunk_size
metadata.state_checkpoint_indices = torch.tensor(
checkpoint_destinations, dtype=torch.int32, device=device
)
class FlashInferKDAPrefillKernel(LinearAttnKernelBase):
"""Execute KDA prefill; FlashInfer validates the tensor contract."""
uses_state_checkpoints = True
supports_track_state_snapshot = True
supports_safe_gate = True
def __init__(self):
if torch.cuda.get_device_capability() not in ((10, 0), (10, 3)):
raise ValueError("FlashInfer KDA prefill requires SM100 or SM103")
from flashinfer.kda import recurrent_kda
self._recurrent_kda = recurrent_kda
def decode(self, *args, **kwargs):
raise NotImplementedError("FlashInferKDAPrefillKernel is prefill-only")
def extend(
self,
q: torch.Tensor,
k: torch.Tensor,
v: torch.Tensor,
g: torch.Tensor,
beta: torch.Tensor,
*,
ssm_states: torch.Tensor,
cache_indices: torch.Tensor,
query_start_loc: torch.Tensor,
A_log: torch.Tensor,
dt_bias: torch.Tensor,
lower_bound: float,
return_intermediate_states: bool = False,
state_checkpoint_cu_starts: Optional[torch.Tensor] = None,
num_state_checkpoints: int = 0,
state_checkpoint_every_n_tokens: int = 0,
state_checkpoint_indices: Optional[torch.Tensor] = None,
track_ssm_h_batch_src: Optional[torch.Tensor] = None,
**kwargs,
) -> tuple[torch.Tensor, Optional[torch.Tensor]]:
needs_checkpoint = (
return_intermediate_states and kwargs.get("track_state") is not None
)
checkpoints = (
ssm_states.new_empty((num_state_checkpoints, *ssm_states.shape[1:]))
if needs_checkpoint
else None
)
result = self._recurrent_kda(
q=q.contiguous(),
k=k.contiguous(),
v=v.contiguous(),
g=g[:, : q.shape[1]].contiguous(),
beta=beta[:, : q.shape[1]].contiguous(),
A_log=A_log.reshape(-1).float().contiguous(),
dt_bias=dt_bias.reshape(q.shape[2], 128).float().contiguous(),
initial_state=ssm_states,
output_final_state=False,
use_qk_l2norm_in_kernel=True,
use_gate_in_kernel=True,
lower_bound=lower_bound,
cu_seqlens=query_start_loc,
ssm_state_indices=cache_indices.to(torch.int32),
beta_is_logit=True,
state_checkpoints=checkpoints,
checkpoint_cu_starts=(
state_checkpoint_cu_starts if needs_checkpoint else None
),
checkpoint_state_indices=(
state_checkpoint_indices if needs_checkpoint else None
),
checkpoint_every_n_tokens=(
state_checkpoint_every_n_tokens if needs_checkpoint else 0
),
backend="cute-dsl",
)
if needs_checkpoint:
kwargs["track_state"][track_ssm_h_batch_src] = checkpoints.float()
output = result[0]
return (output, None) if return_intermediate_states else output
@@ -172,6 +172,7 @@ class HelionKDAKernel(LinearAttnKernelBase):
dt_bias: torch.Tensor | None = None,
lower_bound: float | None = None,
return_intermediate_states: bool = False,
beta_is_raw: bool = False,
**kwargs,
) -> torch.Tensor:
assert self._chunk_kda is not None
@@ -188,6 +189,7 @@ class HelionKDAKernel(LinearAttnKernelBase):
A_log=A_log,
dt_bias=dt_bias,
lower_bound=lower_bound,
beta_is_raw=beta_is_raw,
output_intermediate_states=return_intermediate_states,
track_state=kwargs.get("track_state"),
track_chunk_idx=kwargs.get("track_chunk_idx"),
@@ -237,8 +237,11 @@ class NvidiaKDAKernel(LinearAttnKernelBase):
dt_bias: Optional[torch.Tensor] = None,
lower_bound: Optional[float] = None,
return_intermediate_states: bool = False,
beta_is_raw: bool = False,
**kwargs,
) -> torch.Tensor:
if beta_is_raw:
beta = beta.float().sigmoid()
num_tokens = q.shape[1]
seq_lens_cpu = kwargs.get("extend_seq_lens_cpu")
track_h_src = kwargs.get("track_ssm_h_src")
@@ -64,6 +64,7 @@ class ForwardMetadata:
state_checkpoint_cu_starts: Optional[torch.Tensor] = None
num_state_checkpoints: int = 0
state_checkpoint_every_n_tokens: int = 0
state_checkpoint_indices: Optional[torch.Tensor] = None
track_ssm_seq_idx: Optional[torch.Tensor] = None
track_ssm_end_locs: Optional[torch.Tensor] = None
track_ssm_recompute_dst: Optional[torch.Tensor] = None
-4
View File
@@ -2110,10 +2110,6 @@ class KimiK3DeltaAttention(nn.Module):
if not forward_batch.forward_mode.is_decode():
forget_gate = forget_gate.unflatten(-1, (-1, self.head_dim))
if not forward_batch.forward_mode.is_target_verify():
# Only chunk_kda (extend) wants pre-activated beta; the verify
# kernel sigmoids it in-kernel like decode.
beta = beta.float().sigmoid()
forget_gate = forget_gate.unsqueeze(0)
beta = beta.unsqueeze(0)
-6
View File
@@ -513,8 +513,6 @@ class KimiDeltaAttention(nn.Module):
hidden_states
)
# Prefill passes raw gates to chunk KDA; decode and target-verify kernels
# apply the activation internally.
if (
not forward_batch.forward_mode.is_decode()
and not forward_batch.forward_mode.is_target_verify()
@@ -522,10 +520,6 @@ class KimiDeltaAttention(nn.Module):
forget_gate = forget_gate.unflatten(
-1, (-1, self.head_dim)
) # [T, H*K] -> [T, H, K]
if not forward_batch.forward_mode.is_target_verify():
# Only chunk_kda (extend) wants pre-activated beta; the verify
# kernel sigmoids it in-kernel like decode.
beta = beta.float().sigmoid()
forget_gate = forget_gate.unsqueeze(0)
beta = beta.unsqueeze(0)
@@ -659,9 +659,6 @@ def build_kda_attention_fixture(
device=device,
)
# KDA gate input is per-head-channel ([T, HV*K] raw); beta is per-head ([T, HV]).
# For extend, the production model unflattens gate to [1, T, HV, K] and
# sigmoid-then-unsqueezes beta to [1, T, HV] before calling the attn layer.
# For decode, both stay flat and beta is sigmoid'd inside the fused kernel.
a_raw = torch.randn(
case.num_input_tokens,
case.num_v_heads * head_k_dim,
@@ -673,10 +670,9 @@ def build_kda_attention_fixture(
)
if case.forward_mode.is_decode():
a = a_raw
b = b_raw.unsqueeze(0)
else:
a = a_raw.unflatten(-1, (case.num_v_heads, head_k_dim)).unsqueeze(0)
b = b_raw.float().sigmoid().unsqueeze(0).to(dtype)
b = b_raw.unsqueeze(0)
fixture = KDAAttentionFixture(
case=case,
@@ -886,14 +882,6 @@ def make_kda_case_with_prefix_lens(
def kda_fixture_inputs(fixture: KDAAttentionFixture) -> dict[str, torch.Tensor]:
# `a, b` are the per-forward-mode shaped tensors the actual module
# consumes (see `build_kda_attention_fixture`: for DECODE
# `a = a_raw [T, HV*K]` and `b = b_raw.unsqueeze(0) [1, T, HV]`; for
# non-DECODE `a = a_raw.unflatten(-1, (HV, K)).unsqueeze(0)` and
# `b = b_raw.sigmoid().unsqueeze(0)`). The verify reference
# (`expected_kda_verify_output_from_inputs` →
# `_pure_torch_kda_gating`) expects raw `[T, HV*K]` / `[T, HV]`
# instead, so we expose both shapes through the inputs dict.
return {
"mixed_qkv": fixture.mixed_qkv,
"a": fixture.a,
@@ -925,10 +913,9 @@ def make_kda_random_inputs(
)
if case.forward_mode.is_decode():
a = a_raw
b = b_raw.unsqueeze(0)
else:
a = a_raw.unflatten(-1, (case.num_v_heads, head_k_dim)).unsqueeze(0)
b = b_raw.float().sigmoid().unsqueeze(0).to(dtype)
b = b_raw.unsqueeze(0)
return {
"mixed_qkv": torch.randn(
case.num_input_tokens,