From fc3385d53005945473bca42ae62e9635cea1d6ca Mon Sep 17 00:00:00 2001 From: Khoa Pham Date: Thu, 8 Oct 2026 17:16:33 -0700 Subject: [PATCH] feat(kda): integrate FlashInfer prefill checkpoints (#41400) --- .../npu/attention/ascend_kda_backend.py | 9 +- .../layers/attention/linear/kda_backend.py | 79 +++++- .../linear/kernels/kda_flashinfer.py | 13 +- .../linear/kernels/kda_flashinfer_prefill.py | 148 ++++++++++ .../attention/linear/kernels/kda_helion.py | 2 + .../attention/linear/kernels/kda_nvidia.py | 3 + .../layers/attention/mamba/mamba2_metadata.py | 1 + python/sglang/srt/models/kimi_k3.py | 4 - python/sglang/srt/models/kimi_linear.py | 6 - .../attention_methods/kda_attention.py | 17 +- .../attention/test_kda_prefill_flashinfer.py | 253 ++++++++++++++++++ .../attention/unittests/kda/README.md | 35 ++- .../linear/kernels/test_kda_nvidia.py | 12 +- .../attention/test_linear_attn_config.py | 51 ++++ 14 files changed, 576 insertions(+), 57 deletions(-) create mode 100644 python/sglang/srt/layers/attention/linear/kernels/kda_flashinfer_prefill.py create mode 100644 test/registered/attention/test_kda_prefill_flashinfer.py diff --git a/python/sglang/srt/hardware_backend/npu/attention/ascend_kda_backend.py b/python/sglang/srt/hardware_backend/npu/attention/ascend_kda_backend.py index c23c000747..23830e0747 100644 --- a/python/sglang/srt/hardware_backend/npu/attention/ascend_kda_backend.py +++ b/python/sglang/srt/hardware_backend/npu/attention/ascend_kda_backend.py @@ -319,12 +319,7 @@ class AscendKDAAttnBackend(KDAAttnBackend): Optional[torch.Tensor], Optional[torch.Tensor], ]: - """Apply the Ascend prefill gate contract. - - The checkpoint was validated with FP32 gate activation before - ``chunk_kda``. Keeping this platform override here leaves the shared - GPU model/backend paths unchanged. - """ + """Activate raw gate and beta projections in FP32 for Ascend prefill.""" preactivated_g = fused_kda_gate_npu( g.flatten(-2), layer.A_log, @@ -332,7 +327,7 @@ class AscendKDAAttnBackend(KDAAttnBackend): gate_bias=layer.dt_bias, lower_bound=layer.lower_bound, ) - return preactivated_g, beta, None, None + return preactivated_g, beta.float().sigmoid(), None, None def _forward_target_verify( self, diff --git a/python/sglang/srt/layers/attention/linear/kda_backend.py b/python/sglang/srt/layers/attention/linear/kda_backend.py index 60002e99ba..cf37a8093f 100644 --- a/python/sglang/srt/layers/attention/linear/kda_backend.py +++ b/python/sglang/srt/layers/attention/linear/kda_backend.py @@ -34,6 +34,7 @@ elif is_cpu(): causal_conv1d_update = causal_conv1d_update_cpu +from sglang.srt.model_executor.cuda_graph_config import Backend as CudaGraphBackend from sglang.srt.model_executor.forward_batch_info import ForwardBatch from sglang.srt.model_executor.model_runner import ModelRunner from sglang.srt.runtime_context import ( @@ -45,6 +46,24 @@ from sglang.srt.runtime_context import ( ) +def _validate_flashinfer_kda_prefill( + *, + chunk_size: int, + enable_two_batch_overlap: bool, + prefill_cuda_graph_backend: CudaGraphBackend, +) -> None: + if chunk_size <= 0 or chunk_size % 32: + raise ValueError( + "FlashInfer KDA checkpoint interval must be positive and divisible by 32" + ) + if enable_two_batch_overlap: + raise ValueError("FlashInfer KDA prefill does not support two-batch overlap") + if prefill_cuda_graph_backend == CudaGraphBackend.FULL: + raise ValueError( + "FlashInfer KDA prefill requires eager linear attention; use disabled or breakable prefill CUDA graphs" + ) + + class KDAKernelDispatcher: """Dispatches KDA kernel calls to the appropriate backend per mode.""" @@ -55,6 +74,7 @@ class KDAKernelDispatcher: verify_backend: LinearAttnKernelBackend, ): self.verify_backend = verify_backend + self.prefill_backend = prefill_backend triton_kernel = TritonKDAKernel() self.triton_kernel = triton_kernel helion_kernel = None @@ -86,7 +106,6 @@ class KDAKernelDispatcher: self.decode_kernel = CuteDSLKDAKernel() elif decode_backend.is_flashinfer(): # FlashInfer recurrent_kda: SM100 decode + MTP (target_verify). - # Prefill stays on Triton / CuTe DSL (FlashInfer has no KDA chunk kernel). if not is_cuda(): raise ValueError("KDA FlashInfer backend requires CUDA") from sglang.srt.layers.attention.linear.kernels.kda_flashinfer import ( @@ -145,6 +164,14 @@ class KDAKernelDispatcher: ) self.extend_kernel = FlashKDAKernel() + elif prefill_backend.is_flashinfer(): + if not is_cuda(): + raise ValueError("KDA FlashInfer prefill backend requires CUDA") + from sglang.srt.layers.attention.linear.kernels.kda_flashinfer_prefill import ( + FlashInferKDAPrefillKernel, + ) + + self.extend_kernel = FlashInferKDAPrefillKernel() elif prefill_backend.is_cutedsl(): if not is_cuda(): raise ValueError("KDA CuTe DSL backend requires CUDA") @@ -195,7 +222,7 @@ class KDAKernelDispatcher: else: raise ValueError( f"Unsupported KDA prefill backend: {prefill_backend}. " - "KDA supports 'triton', 'helion', 'flashkda', 'cutedsl', " + "KDA supports 'triton', 'helion', 'flashinfer', 'flashkda', 'cutedsl', " "'nvidia_kda', or 'ptx_kda' (cutedsl/nvidia_kda prefill need " "SM100, ptx_kda SM100 or SM103)." ) @@ -332,10 +359,11 @@ class KDAKernelDispatcher: **kwargs, ) - def effective_extend_kernel(self, lower_bound: Optional[float]): - """The kernel ``extend`` will actually run: safe-gate models reroute - kernels without ``supports_safe_gate`` to Triton.""" + def effective_extend_kernel(self, lower_bound: Optional[float], num_tokens: int): + """The packed FlashInfer prefill kernel requires more than one token.""" kernel = self.extend_kernel + if self.prefill_backend.is_flashinfer() and num_tokens <= 1: + return self.triton_kernel if lower_bound is not None and not getattr(kernel, "supports_safe_gate", True): kernel = self.triton_kernel return kernel @@ -353,7 +381,7 @@ class KDAKernelDispatcher: query_start_loc: torch.Tensor, **kwargs, ) -> tuple[torch.Tensor, torch.Tensor | None]: - kernel = self.effective_extend_kernel(kwargs.get("lower_bound")) + kernel = self.effective_extend_kernel(kwargs.get("lower_bound"), q.shape[1]) return kernel.extend( q, k, @@ -429,6 +457,12 @@ class KDAAttnBackend(MambaAttnBackendBase): self.kernel_dispatcher = KDAKernelDispatcher( decode_backend, prefill_backend, verify_backend ) + if prefill_backend.is_flashinfer(): + _validate_flashinfer_kda_prefill( + chunk_size=self.mamba_chunk_size, + enable_two_batch_overlap=get_exec().overlap.enable_two_batch_overlap, + prefill_cuda_graph_backend=get_exec().graph.cuda_graph_config.prefill.backend, + ) # One-shot; emitted at the first fused-decode interception below. self._fused_override_notice = ( "K3 fused KDA decode engaged: --linear-attn-decode-backend " @@ -543,6 +577,25 @@ class KDAAttnBackend(MambaAttnBackendBase): self.forward_metadata.mamba_track_mask_indices ] ) + if ( + self.kernel_dispatcher.extend_kernel.uses_state_checkpoints + and forward_batch.forward_mode.is_extend_without_speculative() + and self.forward_metadata.track_ssm_h_src is not None + and self.forward_metadata.track_ssm_h_src.numel() > 0 + ): + from sglang.srt.layers.attention.linear.kernels.kda_flashinfer_prefill import ( + build_flashinfer_kda_checkpoint_plan, + ) + + assert self._has_cpu_prefill_track_metadata(forward_batch), ( + "FlashInfer KDA checkpoints require host prefill tracking metadata" + ) + build_flashinfer_kda_checkpoint_plan( + forward_batch, + self.forward_metadata, + self.device, + self.mamba_chunk_size, + ) def forward_decode( self, @@ -872,8 +925,7 @@ class KDAAttnBackend(MambaAttnBackendBase): k = k.unflatten(-1, (-1, layer.head_k_dim)).unsqueeze(0) # n (h d) -> 1 n h d v = v.unflatten(-1, (-1, layer.head_v_dim)).unsqueeze(0) # n (h d) -> 1 n h d - gate_was_flat = a.ndim == 3 - if gate_was_flat: + if a.ndim == 3: a = a.unflatten(-1, (-1, layer.head_k_dim)) track_ssm = self.forward_metadata.has_mamba_track_mask @@ -892,7 +944,7 @@ class KDAAttnBackend(MambaAttnBackendBase): # Check the kernel the dispatcher will actually run (safe-gate # reroute included), not just the configured one. extend_kernel = self.kernel_dispatcher.effective_extend_kernel( - layer.lower_bound + layer.lower_bound, q.shape[1] ) assert extend_kernel.supports_track_state_snapshot, ( f"{type(extend_kernel).__name__} cannot write the fp32 track " @@ -917,7 +969,7 @@ class KDAAttnBackend(MambaAttnBackendBase): A_log=layer.A_log, dt_bias=layer.dt_bias, lower_bound=layer.lower_bound, - beta_is_raw=gate_was_flat, + beta_is_raw=True, extend_seq_lens_cpu=forward_batch.extend_seq_lens_cpu, extend_prefix_lens=forward_batch.extend_prefix_lens, layer_id=layer.layer_id, @@ -933,6 +985,13 @@ class KDAAttnBackend(MambaAttnBackendBase): ), track_state=h_track_buf, track_chunk_idx=(track_chunk_idx if h_track_buf is not None else None), + state_checkpoint_cu_starts=self.forward_metadata.state_checkpoint_cu_starts, + num_state_checkpoints=self.forward_metadata.num_state_checkpoints, + state_checkpoint_every_n_tokens=( + self.forward_metadata.state_checkpoint_every_n_tokens + ), + state_checkpoint_indices=self.forward_metadata.state_checkpoint_indices, + track_ssm_h_batch_src=self.forward_metadata.track_ssm_h_batch_src, ) if track_ssm: # Snapshot the SSM state at the last track-aligned chunk boundary diff --git a/python/sglang/srt/layers/attention/linear/kernels/kda_flashinfer.py b/python/sglang/srt/layers/attention/linear/kernels/kda_flashinfer.py index 180c3482b3..cb1e6bdcbc 100644 --- a/python/sglang/srt/layers/attention/linear/kernels/kda_flashinfer.py +++ b/python/sglang/srt/layers/attention/linear/kernels/kda_flashinfer.py @@ -1,7 +1,7 @@ """FlashInfer KDA decode/verify wrapper. -Wraps ``flashinfer.kda_decode.recurrent_kda`` (SM100 / Blackwell). FlashInfer has -no KDA prefill kernel, so ``extend`` stays on Triton / CuTe DSL. +Wraps ``flashinfer.kda_decode.recurrent_kda`` (SM100 / Blackwell). KDA prefill +uses a separate adapter in ``kda_flashinfer_prefill``. Contract with the Triton KDA reference: - raw per-K gate ``a`` is activated in-kernel as @@ -85,11 +85,7 @@ def build_fused_accept_indices( class FlashInferKDAKernel(LinearAttnKernelBase): - """FlashInfer KDA kernel: SM100 decode + MTP (target_verify), topk=1. - - Prefill (``extend``) is intentionally not implemented -- FlashInfer ships no - KDA chunk kernel; the dispatcher keeps prefill on Triton / CuTe DSL. - """ + """FlashInfer KDA kernel: SM100 decode + MTP (target_verify), topk=1.""" def __init__(self): available, self._recurrent_kda = _get_flashinfer_kda_kernel() @@ -381,5 +377,6 @@ class FlashInferKDAKernel(LinearAttnKernelBase): def extend(self, *args, **kwargs): raise NotImplementedError( - "FlashInferKDAKernel has no prefill kernel; keep prefill on Triton / CuTe DSL." + "FlashInferKDAKernel is decode/verify-only; use " + "FlashInferKDAPrefillKernel for prefill." ) diff --git a/python/sglang/srt/layers/attention/linear/kernels/kda_flashinfer_prefill.py b/python/sglang/srt/layers/attention/linear/kernels/kda_flashinfer_prefill.py new file mode 100644 index 0000000000..9dad5f64cd --- /dev/null +++ b/python/sglang/srt/layers/attention/linear/kernels/kda_flashinfer_prefill.py @@ -0,0 +1,148 @@ +"""FlashInfer 0.7 CuTe DSL KDA prefill with radix-cache checkpoints.""" + +from __future__ import annotations + +from itertools import accumulate +from typing import TYPE_CHECKING, Optional + +import torch + +from sglang.srt.layers.attention.linear.kernels.kernel_backend import ( + LinearAttnKernelBase, +) + +if TYPE_CHECKING: + from sglang.srt.layers.attention.mamba.mamba2_metadata import ForwardMetadata + from sglang.srt.model_executor.forward_batch_info import ForwardBatch + + +def build_flashinfer_kda_checkpoint_plan( + forward_batch: ForwardBatch, + metadata: ForwardMetadata, + device: torch.device, + chunk_size: int, +) -> None: + """Plan scheduler-generated tracked boundaries, asserting their validity. + + Host track metadata follows build_prefill_track_plan's unaligned-row order. + """ + extend_lens = forward_batch.extend_seq_lens_cpu + track_lens = forward_batch.mamba_track_seqlens_cpu + prefix_lens = forward_batch.extend_prefix_lens_cpu + track_mask = forward_batch.mamba_prefill_track_mask_cpu + + checkpoint_counts = [length // chunk_size for length in extend_lens] + checkpoint_starts = list(accumulate(checkpoint_counts, initial=0)) + # NOTE(kpham-sgl): FlashInfer GDN writes all periodic checkpoints for SGLang + # to select from; FlashInfer KDA writes only mapped destinations (-1 skips). + checkpoint_destinations = [-1] * checkpoint_starts[-1] + num_tracked_checkpoints = 0 + for row, tracked in enumerate(track_mask): + if not tracked: + continue + relative_track_len = track_lens[row] - prefix_lens[row] + if relative_track_len % chunk_size == 0: + continue + completed_chunks = relative_track_len // chunk_size + assert 0 < completed_chunks <= checkpoint_counts[row], ( + f"Invalid KDA checkpoint metadata: row={row}, " + f"relative_track_len={relative_track_len}, extend_len={extend_lens[row]}, " + f"chunk_size={chunk_size}" + ) + checkpoint_destinations[checkpoint_starts[row] + completed_chunks - 1] = ( + num_tracked_checkpoints + ) + num_tracked_checkpoints += 1 + + assert num_tracked_checkpoints == metadata.track_ssm_h_batch_src.numel(), ( + "KDA checkpoint rows disagree with the prefill tracking metadata" + ) + metadata.state_checkpoint_cu_starts = torch.tensor( + checkpoint_starts, dtype=torch.int64, device=device + ) + metadata.num_state_checkpoints = num_tracked_checkpoints + metadata.state_checkpoint_every_n_tokens = chunk_size + metadata.state_checkpoint_indices = torch.tensor( + checkpoint_destinations, dtype=torch.int32, device=device + ) + + +class FlashInferKDAPrefillKernel(LinearAttnKernelBase): + """Execute KDA prefill; FlashInfer validates the tensor contract.""" + + uses_state_checkpoints = True + supports_track_state_snapshot = True + supports_safe_gate = True + + def __init__(self): + if torch.cuda.get_device_capability() not in ((10, 0), (10, 3)): + raise ValueError("FlashInfer KDA prefill requires SM100 or SM103") + from flashinfer.kda import recurrent_kda + + self._recurrent_kda = recurrent_kda + + def decode(self, *args, **kwargs): + raise NotImplementedError("FlashInferKDAPrefillKernel is prefill-only") + + def extend( + self, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + g: torch.Tensor, + beta: torch.Tensor, + *, + ssm_states: torch.Tensor, + cache_indices: torch.Tensor, + query_start_loc: torch.Tensor, + A_log: torch.Tensor, + dt_bias: torch.Tensor, + lower_bound: float, + return_intermediate_states: bool = False, + state_checkpoint_cu_starts: Optional[torch.Tensor] = None, + num_state_checkpoints: int = 0, + state_checkpoint_every_n_tokens: int = 0, + state_checkpoint_indices: Optional[torch.Tensor] = None, + track_ssm_h_batch_src: Optional[torch.Tensor] = None, + **kwargs, + ) -> tuple[torch.Tensor, Optional[torch.Tensor]]: + needs_checkpoint = ( + return_intermediate_states and kwargs.get("track_state") is not None + ) + checkpoints = ( + ssm_states.new_empty((num_state_checkpoints, *ssm_states.shape[1:])) + if needs_checkpoint + else None + ) + result = self._recurrent_kda( + q=q.contiguous(), + k=k.contiguous(), + v=v.contiguous(), + g=g[:, : q.shape[1]].contiguous(), + beta=beta[:, : q.shape[1]].contiguous(), + A_log=A_log.reshape(-1).float().contiguous(), + dt_bias=dt_bias.reshape(q.shape[2], 128).float().contiguous(), + initial_state=ssm_states, + output_final_state=False, + use_qk_l2norm_in_kernel=True, + use_gate_in_kernel=True, + lower_bound=lower_bound, + cu_seqlens=query_start_loc, + ssm_state_indices=cache_indices.to(torch.int32), + beta_is_logit=True, + state_checkpoints=checkpoints, + checkpoint_cu_starts=( + state_checkpoint_cu_starts if needs_checkpoint else None + ), + checkpoint_state_indices=( + state_checkpoint_indices if needs_checkpoint else None + ), + checkpoint_every_n_tokens=( + state_checkpoint_every_n_tokens if needs_checkpoint else 0 + ), + backend="cute-dsl", + ) + if needs_checkpoint: + kwargs["track_state"][track_ssm_h_batch_src] = checkpoints.float() + output = result[0] + return (output, None) if return_intermediate_states else output diff --git a/python/sglang/srt/layers/attention/linear/kernels/kda_helion.py b/python/sglang/srt/layers/attention/linear/kernels/kda_helion.py index b1ecbcf482..6a50d5a6c4 100644 --- a/python/sglang/srt/layers/attention/linear/kernels/kda_helion.py +++ b/python/sglang/srt/layers/attention/linear/kernels/kda_helion.py @@ -172,6 +172,7 @@ class HelionKDAKernel(LinearAttnKernelBase): dt_bias: torch.Tensor | None = None, lower_bound: float | None = None, return_intermediate_states: bool = False, + beta_is_raw: bool = False, **kwargs, ) -> torch.Tensor: assert self._chunk_kda is not None @@ -188,6 +189,7 @@ class HelionKDAKernel(LinearAttnKernelBase): A_log=A_log, dt_bias=dt_bias, lower_bound=lower_bound, + beta_is_raw=beta_is_raw, output_intermediate_states=return_intermediate_states, track_state=kwargs.get("track_state"), track_chunk_idx=kwargs.get("track_chunk_idx"), diff --git a/python/sglang/srt/layers/attention/linear/kernels/kda_nvidia.py b/python/sglang/srt/layers/attention/linear/kernels/kda_nvidia.py index 1171c39360..493ca43757 100644 --- a/python/sglang/srt/layers/attention/linear/kernels/kda_nvidia.py +++ b/python/sglang/srt/layers/attention/linear/kernels/kda_nvidia.py @@ -237,8 +237,11 @@ class NvidiaKDAKernel(LinearAttnKernelBase): dt_bias: Optional[torch.Tensor] = None, lower_bound: Optional[float] = None, return_intermediate_states: bool = False, + beta_is_raw: bool = False, **kwargs, ) -> torch.Tensor: + if beta_is_raw: + beta = beta.float().sigmoid() num_tokens = q.shape[1] seq_lens_cpu = kwargs.get("extend_seq_lens_cpu") track_h_src = kwargs.get("track_ssm_h_src") diff --git a/python/sglang/srt/layers/attention/mamba/mamba2_metadata.py b/python/sglang/srt/layers/attention/mamba/mamba2_metadata.py index b2843a536e..cea43f7ebf 100644 --- a/python/sglang/srt/layers/attention/mamba/mamba2_metadata.py +++ b/python/sglang/srt/layers/attention/mamba/mamba2_metadata.py @@ -64,6 +64,7 @@ class ForwardMetadata: state_checkpoint_cu_starts: Optional[torch.Tensor] = None num_state_checkpoints: int = 0 state_checkpoint_every_n_tokens: int = 0 + state_checkpoint_indices: Optional[torch.Tensor] = None track_ssm_seq_idx: Optional[torch.Tensor] = None track_ssm_end_locs: Optional[torch.Tensor] = None track_ssm_recompute_dst: Optional[torch.Tensor] = None diff --git a/python/sglang/srt/models/kimi_k3.py b/python/sglang/srt/models/kimi_k3.py index 99fb96f783..e432aab340 100644 --- a/python/sglang/srt/models/kimi_k3.py +++ b/python/sglang/srt/models/kimi_k3.py @@ -2110,10 +2110,6 @@ class KimiK3DeltaAttention(nn.Module): if not forward_batch.forward_mode.is_decode(): forget_gate = forget_gate.unflatten(-1, (-1, self.head_dim)) - if not forward_batch.forward_mode.is_target_verify(): - # Only chunk_kda (extend) wants pre-activated beta; the verify - # kernel sigmoids it in-kernel like decode. - beta = beta.float().sigmoid() forget_gate = forget_gate.unsqueeze(0) beta = beta.unsqueeze(0) diff --git a/python/sglang/srt/models/kimi_linear.py b/python/sglang/srt/models/kimi_linear.py index 9ad88d8a16..d1a29f2f9d 100644 --- a/python/sglang/srt/models/kimi_linear.py +++ b/python/sglang/srt/models/kimi_linear.py @@ -513,8 +513,6 @@ class KimiDeltaAttention(nn.Module): hidden_states ) - # Prefill passes raw gates to chunk KDA; decode and target-verify kernels - # apply the activation internally. if ( not forward_batch.forward_mode.is_decode() and not forward_batch.forward_mode.is_target_verify() @@ -522,10 +520,6 @@ class KimiDeltaAttention(nn.Module): forget_gate = forget_gate.unflatten( -1, (-1, self.head_dim) ) # [T, H*K] -> [T, H, K] - if not forward_batch.forward_mode.is_target_verify(): - # Only chunk_kda (extend) wants pre-activated beta; the verify - # kernel sigmoids it in-kernel like decode. - beta = beta.float().sigmoid() forget_gate = forget_gate.unsqueeze(0) beta = beta.unsqueeze(0) diff --git a/python/sglang/test/kits/attention_unittest/attention_methods/kda_attention.py b/python/sglang/test/kits/attention_unittest/attention_methods/kda_attention.py index efc2bacace..f3c312ea4a 100644 --- a/python/sglang/test/kits/attention_unittest/attention_methods/kda_attention.py +++ b/python/sglang/test/kits/attention_unittest/attention_methods/kda_attention.py @@ -659,9 +659,6 @@ def build_kda_attention_fixture( device=device, ) # KDA gate input is per-head-channel ([T, HV*K] raw); beta is per-head ([T, HV]). - # For extend, the production model unflattens gate to [1, T, HV, K] and - # sigmoid-then-unsqueezes beta to [1, T, HV] before calling the attn layer. - # For decode, both stay flat and beta is sigmoid'd inside the fused kernel. a_raw = torch.randn( case.num_input_tokens, case.num_v_heads * head_k_dim, @@ -673,10 +670,9 @@ def build_kda_attention_fixture( ) if case.forward_mode.is_decode(): a = a_raw - b = b_raw.unsqueeze(0) else: a = a_raw.unflatten(-1, (case.num_v_heads, head_k_dim)).unsqueeze(0) - b = b_raw.float().sigmoid().unsqueeze(0).to(dtype) + b = b_raw.unsqueeze(0) fixture = KDAAttentionFixture( case=case, @@ -886,14 +882,6 @@ def make_kda_case_with_prefix_lens( def kda_fixture_inputs(fixture: KDAAttentionFixture) -> dict[str, torch.Tensor]: - # `a, b` are the per-forward-mode shaped tensors the actual module - # consumes (see `build_kda_attention_fixture`: for DECODE - # `a = a_raw [T, HV*K]` and `b = b_raw.unsqueeze(0) [1, T, HV]`; for - # non-DECODE `a = a_raw.unflatten(-1, (HV, K)).unsqueeze(0)` and - # `b = b_raw.sigmoid().unsqueeze(0)`). The verify reference - # (`expected_kda_verify_output_from_inputs` → - # `_pure_torch_kda_gating`) expects raw `[T, HV*K]` / `[T, HV]` - # instead, so we expose both shapes through the inputs dict. return { "mixed_qkv": fixture.mixed_qkv, "a": fixture.a, @@ -925,10 +913,9 @@ def make_kda_random_inputs( ) if case.forward_mode.is_decode(): a = a_raw - b = b_raw.unsqueeze(0) else: a = a_raw.unflatten(-1, (case.num_v_heads, head_k_dim)).unsqueeze(0) - b = b_raw.float().sigmoid().unsqueeze(0).to(dtype) + b = b_raw.unsqueeze(0) return { "mixed_qkv": torch.randn( case.num_input_tokens, diff --git a/test/registered/attention/test_kda_prefill_flashinfer.py b/test/registered/attention/test_kda_prefill_flashinfer.py new file mode 100644 index 0000000000..1f98c9c636 --- /dev/null +++ b/test/registered/attention/test_kda_prefill_flashinfer.py @@ -0,0 +1,253 @@ +"""FlashInfer KDA prefill integration against SGLang's Triton reference.""" + +from itertools import accumulate +from types import SimpleNamespace + +import pytest +import torch +from packaging.version import Version + +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.test_utils import CustomTestCase + +register_cuda_ci(est_time=180, stage="base-b-kernel-unit", runner_config="4-gpu-b200") + +if not torch.cuda.is_available() or torch.cuda.get_device_capability() not in ( + (10, 0), + (10, 3), +): + pytest.skip("FlashInfer KDA prefill requires B200/B300", allow_module_level=True) + +flashinfer = pytest.importorskip("flashinfer") +pytest.importorskip("flashinfer.kda") +if Version(flashinfer.__version__) < Version("0.7.0"): + pytest.skip( + "KDA prefill checkpoints require FlashInfer 0.7.0", allow_module_level=True + ) + +from sglang.srt.layers.attention.linear.kda_backend import ( # noqa: E402 + KDAKernelDispatcher, +) +from sglang.srt.layers.attention.linear.kernels.kda_flashinfer_prefill import ( # noqa: E402 + FlashInferKDAPrefillKernel, + build_flashinfer_kda_checkpoint_plan, +) +from sglang.srt.layers.attention.linear.kernels.kda_triton import ( # noqa: E402 + TritonKDAKernel, +) +from sglang.srt.layers.attention.linear.utils import ( # noqa: E402 + LinearAttnKernelBackend, +) +from sglang.srt.layers.attention.mamba.prefill_track_metadata import ( # noqa: E402 + build_prefill_track_plan, +) +from sglang.srt.model_executor.forward_batch_info import ForwardMode # noqa: E402 +from sglang.srt.runtime_context import get_parallel # noqa: E402 +from sglang.test.kits.attention_unittest.attention_methods.kda_attention import ( # noqa: E402 + KDAAttentionCase, + build_kda_attention_fixture, + run_kda_fixture_eager, +) + + +def flashinfer_dispatcher(): + return KDAKernelDispatcher( + LinearAttnKernelBackend.TRITON, + LinearAttnKernelBackend.FLASHINFER, + LinearAttnKernelBackend.TRITON, + ) + + +@pytest.mark.parametrize( + "state_dtype,layout,prefix_len", + [ + (torch.bfloat16, "dense", 0), + (torch.float32, "dense", 0), + (torch.bfloat16, "padded_slots", 0), + (torch.float32, "strided_gates", 0), + (torch.bfloat16, "padded_row", 0), + (torch.float32, "dense", 512), + ], + ids=["bf16", "fp32", "padded_slots", "strided_gates", "padded_row", "dcp8"], +) +def test_kda_prefill_checkpoints(state_dtype, layout, prefix_len): + torch.manual_seed(11 if prefix_len else 7) + lengths = [578] if prefix_len else [130, 128] + if layout == "padded_row": + lengths.append(0) + heads, dim = (2 if prefix_len else 12), 128 + total_tokens = sum(lengths) + + def tensor(values): + return torch.tensor(values, device="cuda", dtype=torch.int32) + + def randn(*shape, scale=1): + return torch.randn(*shape, device="cuda", dtype=torch.bfloat16) * scale + + q, k, v = [randn(1, total_tokens, heads, dim, scale=0.01) for _ in range(3)] + gate_dim = dim * 2 if layout == "strided_gates" else dim + beta_dim = heads * 12 if layout == "strided_gates" else heads + g = randn(1, total_tokens + 14, heads, gate_dim, scale=0.1)[..., :dim] + beta = randn(1, total_tokens + 14, beta_dim)[..., :heads] + initial = (torch.randn(4, heads, dim, dim, device="cuda") * 0.01).to(state_dtype) + slots = tensor([1] if prefix_len else [1, 2, 0][: len(lengths)]) + batch = SimpleNamespace( + extend_seq_lens_cpu=lengths, + mamba_track_seqlens_cpu=[1025] if prefix_len else lengths, + extend_prefix_lens_cpu=[prefix_len] * len(lengths), + mamba_prefill_track_mask_cpu=[n > 0 for n in lengths], + ) + host_plan = build_prefill_track_plan( + batch.mamba_prefill_track_mask_cpu, + batch.mamba_track_seqlens_cpu, + lengths, + batch.extend_prefix_lens_cpu, + 64, + mamba2=False, + ) + metadata = SimpleNamespace(track_ssm_h_batch_src=tensor(host_plan.unaligned_rows)) + build_flashinfer_kda_checkpoint_plan(batch, metadata, "cuda", 64) + assert metadata.state_checkpoint_cu_starts.tolist() == ( + [0, 9] if prefix_len else [0, 2, 4, 4][: len(lengths) + 1] + ) + assert metadata.num_state_checkpoints == 1 + assert metadata.state_checkpoint_indices.tolist() == ( + [-1] * 7 + [0, -1] if prefix_len else [-1, 0, -1, -1] + ) + fi, triton = FlashInferKDAPrefillKernel(), TritonKDAKernel() + gates = dict( + A_log=torch.zeros(heads, device="cuda"), + dt_bias=torch.zeros(heads, dim, device="cuda"), + lower_bound=-5.0, + beta_is_raw=True, + ) + + def run(kernel, *, track=False, prefix_tokens=None): + state = initial.clone() + if layout == "padded_slots" and kernel is fi: + state = torch.empty_strided( + initial.shape, + (heads * dim * dim + 16, dim * dim, dim, 1), + device="cuda", + dtype=state_dtype, + ) + state.copy_(initial) + run_lengths = lengths if prefix_tokens is None else [prefix_tokens] + offsets = tensor([0, *accumulate(run_lengths)]) + inputs = [q.clone(), k.clone(), v.clone(), g, beta] + if prefix_tokens is not None: + inputs = [x[:, :prefix_tokens] for x in inputs] + snapshots = torch.full( + (len(lengths), heads, dim, dim), + torch.nan, + device="cuda", + ) + kwargs = dict(gates, extend_seq_lens_cpu=run_lengths) + if track: + kwargs.update( + return_intermediate_states=True, + track_state=snapshots, + track_chunk_idx=tensor(host_plan.chunk_indices), + ) + if kernel is fi and track: + kwargs.update(vars(metadata)) + output = kernel.extend( + *inputs, + ssm_states=state, + cache_indices=slots[: len(run_lengths)], + query_start_loc=offsets, + **kwargs, + ) + assert torch.isfinite(output[0] if track else output).all() + assert torch.isfinite(state).all() + return state, snapshots + + fi_state, fi_track = run(fi, track=True) + assert torch.isfinite(fi_track[0]).all() and torch.isnan(fi_track[1:]).all() + if layout == "padded_row": + torch.testing.assert_close(fi_state[0], initial[0], atol=0, rtol=0) + if not prefix_len: + _, ref_track = run(triton, track=True) + assert ( + torch.linalg.vector_norm(fi_track[0] - ref_track[0]) + / (torch.linalg.vector_norm(ref_track[0]) + 1e-12) + < 5e-2 + ) + + boundary = 512 if prefix_len else 128 + prefix_state, _ = run(fi, prefix_tokens=boundary) + if prefix_len: + assert ( + torch.linalg.vector_norm(fi_track[0] - prefix_state[1]) + / (torch.linalg.vector_norm(prefix_state[1]) + 1e-12) + < 5e-2 + ) + triton_prefix, _ = run(triton, prefix_tokens=boundary) + torch.testing.assert_close(fi_track[0], triton_prefix[1], atol=1e-5, rtol=5e-2) + else: + torch.testing.assert_close(fi_track[0], prefix_state[1].float(), atol=0, rtol=0) + + +@pytest.mark.parametrize( + "extend_lens", [(130, 128), (1,)], ids=["tracked_prefill", "single_token"] +) +def test_kda_backend_prefill_dispatch_and_tracked_state(extend_lens): + """Raw beta works in FlashInfer and the single-token Triton fallback.""" + with get_parallel().override(attn_dcp_rank=0, attn_dcp_size=1): + case = KDAAttentionCase( + name="flashinfer_kda_tracked_extend", + backend="triton", + forward_mode=ForwardMode.EXTEND, + num_k_heads=2, + num_v_heads=2, + page_size=16, + prefix_lens=(0,) * len(extend_lens), + extend_lens=extend_lens, + ) + fixture = build_kda_attention_fixture( + CustomTestCase(), + case, + head_k_dim=128, + head_v_dim=128, + max_context_len=256, + runner_batch_size=6, + ) + batch = fixture.forward_batch + tracked = [n >= 64 for n in extend_lens] + batch.mamba_track_mask = torch.tensor(tracked, device="cuda") + batch.mamba_track_indices = torch.arange( + 4, 4 + len(extend_lens), device="cuda", dtype=torch.int32 + ) + batch.mamba_track_seqlens = torch.tensor( + extend_lens, device="cuda", dtype=torch.int32 + ) + batch.mamba_prefill_track_mask_cpu = tracked + batch.mamba_track_seqlens_cpu = list(extend_lens) + fixture.actual_module.attn.lower_bound = -5.0 + cache = fixture.runner.req_to_token_pool.mamba2_layer_cache(0) + initial_conv, initial_ssm = cache.conv[0].clone(), cache.temporal.clone() + + triton_output = run_kda_fixture_eager(fixture) + triton_state = cache.temporal.clone() + triton_conv = cache.conv[0].clone() + + cache.conv[0].copy_(initial_conv) + cache.temporal.copy_(initial_ssm) + dispatcher = flashinfer_dispatcher() + backend = fixture.backend.linear_attn_backend + backend.kernel_dispatcher = dispatcher + flashinfer_output = run_kda_fixture_eager(fixture) + + torch.testing.assert_close( + flashinfer_output.float(), triton_output.float(), atol=3e-2, rtol=3e-2 + ) + torch.testing.assert_close( + cache.temporal.float(), triton_state.float(), atol=3e-2, rtol=3e-2 + ) + torch.testing.assert_close(cache.conv[0], triton_conv, atol=0, rtol=0) + + +if __name__ == "__main__": + import sys + + sys.exit(pytest.main([__file__])) diff --git a/test/registered/attention/unittests/kda/README.md b/test/registered/attention/unittests/kda/README.md index d1917ec424..9c18d78983 100644 --- a/test/registered/attention/unittests/kda/README.md +++ b/test/registered/attention/unittests/kda/README.md @@ -10,8 +10,7 @@ Triton kernel. ## Coverage Matrix -Columns are runner modes; rows are the linear-attention kernel backend -(`triton` is the only one wired today). Cells use: +Columns are runner modes; rows are the linear-attention kernel backend. Cells use: - **✓ \** — exercised, with the config variants listed in the cell - **—** — not applicable / not exercised - **blocked: \** — production-unsupported, not a follow-up @@ -20,6 +19,7 @@ Columns are runner modes; rows are the linear-attention kernel backend | Linear-attn kernel | Eager Phase 2 | CG decode | PCG extend | BCG extend | Verify eager | Verify CG | DE eager | DE CG | DE-V2 CG | EAGLE-draft runner | EAGLE-DE runner | FKVMTP runner | |---|---|---|---|---|---|---|---|---|---|---|---|---| | `triton` | ✓ 10 input layouts (page 1/16/32, prefix/decode edges) | ✓ decode page-boundary (uses `KDA_GRAPH_ATOL=1e-1` to absorb Triton recurrent-kernel CG-replay drift; eager `KDA_ATOL=3e-2` kept for non-graph cases) | ✓ ragged page-boundary extend | ✓ ragged page-boundary extend | ✓ EAGLE chain (topk=1) + EAGLE tree (topk=2) (`atol=1e-1` because the verify reference's pure-Python per-token recurrence drifts ~0.07 vs the Triton kernel even before CG capture/replay) | ✓ EAGLE chain CG + EAGLE tree CG (same `1e-1` tolerance) | — | blocked: HybridLinearAttnBackend `_replay_metadata` rejects modes outside `DECODE_OR_IDLE` / `TARGET_VERIFY` (`hybrid_linear_attn_backend.py:509,572`) | blocked: same `_replay_metadata` reject | deferred | blocked: same `_replay_metadata` reject | — | +| `flashinfer` prefill (SM100/SM103, FlashInfer 0.7.0) | ✓ KDA backend extend with beta logits, 128-D heads, and aligned/unaligned radix-cache snapshots; kernel tests cover BF16/FP32 and indexed state pools | — | — | — | — | — | — | — | — | — | — | — | ## Input And Config Coverage @@ -28,6 +28,33 @@ Columns are runner modes; rows are the linear-attention kernel backend page-boundary, batch-size-1 decode. - `num_k_heads=2, num_v_heads=2` with head dims defaulted by `DEFAULT_HEAD_K_DIM = DEFAULT_HEAD_V_DIM = 32`. +- `test_kda_prefill_flashinfer.py` exercises the 128-D FlashInfer path against + Triton on B200/GB300, including the 130-token to 128-token cache boundary. + +## FlashInfer Prefill Contract + +FlashInfer prefill execution requires SM100/SM103, +BF16 model activations, BF16 or FP32 SSM state, equal Q/K/V head counts with +128-D heads, and a finite negative safe-gate lower bound. +FlashInfer validates this tensor contract when the kernel runs, including +during server warmup. Radix-cache checkpoints require a positive mamba chunk +size divisible by 32. Incompatible intervals are rejected at backend +initialization; FlashInfer rejects unbounded gates during warmup, so original +Kimi Linear should use Triton prefill. + +The backend maps radix-cache checkpoint destinations once per tracked batch, +before layer execution. The adapter consumes raw gate/beta projections and +normalizes projection strides before calling `recurrent_kda(backend="cute-dsl")` +directly. FlashInfer owns execution scheduling and workspace allocation; its +direct API may read packed sequence offsets on the host to size that workspace. +Packed batches may include single-token sequences and zero-length padding rows; +only the total packed token count must exceed one. + +KDA models pass raw beta logits in every phase; activation belongs to the +kernel adapters. A packed batch with only one token uses Triton with the same +raw beta format. Invalid checkpoint metadata raises an assertion instead of changing +kernels. TBO and full prefill CUDA graphs are rejected at initialization; BCG +keeps linear attention eager. Verify uses its separately resolved backend. ## Production-Unsupported @@ -40,6 +67,4 @@ Columns are runner modes; rows are the linear-attention kernel backend ## Next Work -- Consider additional KDA kernel backend variants when available. CG - decode, PCG/BCG split-op extend, and EAGLE chain/tree verify - (eager + CG) are all wired (see matrix above). +- Add model-level Kimi K3 coverage when weights are available in the runner. diff --git a/test/registered/unit/layers/attention/linear/kernels/test_kda_nvidia.py b/test/registered/unit/layers/attention/linear/kernels/test_kda_nvidia.py index ffa952a586..ecaa219707 100644 --- a/test/registered/unit/layers/attention/linear/kernels/test_kda_nvidia.py +++ b/test/registered/unit/layers/attention/linear/kernels/test_kda_nvidia.py @@ -95,7 +95,7 @@ class TestNvidiaKDAAllPrefillWrapper(CustomTestCase): def test_short_single_sequence_uses_triton(self): kernel, calls = self._make_kernel() - kernel._triton.extend = Mock(return_value="triton") + kernel._triton.extend = Mock(side_effect=lambda *args, **kwargs: args[4]) x = self._inputs([5]) states = torch.zeros(3, 1, 128, 128, dtype=torch.bfloat16) @@ -110,9 +110,10 @@ class TestNvidiaKDAAllPrefillWrapper(CustomTestCase): query_start_loc=x["query_start_loc"], extend_seq_lens_cpu=[5], A_log=torch.zeros(128, dtype=torch.float32), + beta_is_raw=True, ) - self.assertEqual(output, "triton") + torch.testing.assert_close(output, x["beta"].sigmoid(), atol=0, rtol=0) self.assertEqual(calls, []) kernel._triton.extend.assert_called_once() self.assertTrue(torch.count_nonzero(states).item() == 0) @@ -136,6 +137,7 @@ class TestNvidiaKDAAllPrefillWrapper(CustomTestCase): query_start_loc=x["query_start_loc"], extend_seq_lens_cpu=seq_lens, A_log=torch.zeros(128, dtype=torch.float32), + beta_is_raw=True, ) self.assertEqual(len(calls), 1) @@ -148,6 +150,12 @@ class TestNvidiaKDAAllPrefillWrapper(CustomTestCase): for row, length in enumerate(seq_lens): end = start + length self.assertTrue(torch.equal(call["v"][row, :length], x["v"][0, start:end])) + torch.testing.assert_close( + call["beta"][row, :length], + x["beta"][0, start:end].sigmoid().bfloat16(), + atol=0, + rtol=0, + ) self.assertTrue(torch.count_nonzero(call["q"][row, length:]).item() == 0) self.assertTrue(torch.count_nonzero(call["k"][row, length:]).item() == 0) self.assertTrue(torch.count_nonzero(call["v"][row, length:]).item() == 0) diff --git a/test/registered/unit/layers/attention/test_linear_attn_config.py b/test/registered/unit/layers/attention/test_linear_attn_config.py index 57eb734de9..c46aea918f 100644 --- a/test/registered/unit/layers/attention/test_linear_attn_config.py +++ b/test/registered/unit/layers/attention/test_linear_attn_config.py @@ -157,5 +157,56 @@ class TestLinearAttnBackends(CustomTestCase): ) +class TestFlashInferKDAPrefillPolicy(CustomTestCase): + def test_fixed_restrictions_are_rejected_before_forward(self): + from sglang.srt.layers.attention.linear.kda_backend import ( + _validate_flashinfer_kda_prefill, + ) + + for chunk, tbo, graph, error in [ + (0, False, "disabled", "checkpoint interval"), + (33, False, "disabled", "checkpoint interval"), + (64, True, "disabled", "two-batch overlap"), + (64, False, "full", "eager linear attention"), + (64, False, "breakable", None), + ]: + with self.subTest(chunk=chunk, tbo=tbo, graph=graph): + kwargs = dict( + chunk_size=chunk, + enable_two_batch_overlap=tbo, + prefill_cuda_graph_backend=graph, + ) + if error is not None: + with self.assertRaisesRegex(ValueError, error): + _validate_flashinfer_kda_prefill(**kwargs) + else: + _validate_flashinfer_kda_prefill(**kwargs) + + def test_invalid_checkpoint_metadata_fails_instead_of_falling_back(self): + import torch + + from sglang.srt.layers.attention.linear.kernels.kda_flashinfer_prefill import ( + build_flashinfer_kda_checkpoint_plan, + ) + + for extend_len, track_len, rows in ( + (128, 1, [0]), + (128, 193, [0]), + (130, 130, []), + ): + with self.subTest(extend_len=extend_len, track_len=track_len, rows=rows): + batch = SimpleNamespace( + extend_seq_lens_cpu=[extend_len], + extend_prefix_lens_cpu=[0], + mamba_track_seqlens_cpu=[track_len], + mamba_prefill_track_mask_cpu=[True], + ) + metadata = SimpleNamespace( + track_ssm_h_batch_src=torch.tensor(rows, dtype=torch.int32) + ) + with self.assertRaisesRegex(AssertionError, "KDA checkpoint"): + build_flashinfer_kda_checkpoint_plan(batch, metadata, "cpu", 64) + + if __name__ == "__main__": unittest.main()