[Perf] cuda-graph: Capture FlashInfer KDA prefill (#45473)

This commit is contained in:
Baizhou Zhang
2026-10-21 11:57:23 -07:00
committed by GitHub
parent 740d5f9f9e
commit f06d1bb1f4
5 changed files with 241 additions and 46 deletions
@@ -624,8 +624,15 @@ class KDAAttnBackend(MambaAttnBackendBase):
def supports_prefill_graph_extend(self) -> bool:
return (
not envs.SGLANG_DISABLE_KDA_PREFILL_GRAPH_EXTEND.get()
# Only the Triton chain guards padded rows (-1 state slots, chunks past T).
and isinstance(self.kernel_dispatcher.extend_kernel, TritonKDAKernel)
and (
isinstance(self.kernel_dispatcher.extend_kernel, TritonKDAKernel)
or getattr(
self.kernel_dispatcher.extend_kernel,
"prefill_graph_wrapper_cls",
None,
)
is not None
)
# Fused-accept staging (flashinfer verify) writes per extend.
and self.accept_lens_pool is None
)
@@ -638,13 +645,17 @@ class KDAAttnBackend(MambaAttnBackendBase):
) -> KDAPrefillGraphMetadata:
cache_indices = self._prefill_graph_cache_indices(forward_batch)
meta = KDAPrefillGraphMetadata.allocate(
num_tokens=forward_batch.positions.shape[0],
# KDA sees global sequence order, even when CP positions are local.
num_tokens=forward_batch.extend_num_tokens,
max_seqs_cap=self.prefill_graph_max_seqs,
# Any replay of this bucket may carry prefix-cache snapshots (the
# extra buffer is only on with the radix cache).
track=get_exec().mamba.enable_mamba_extra_buffer,
cache_index_dtype=cache_indices.dtype,
device=self.device,
flashinfer_wrapper_cls=getattr(
self.kernel_dispatcher.extend_kernel, "prefill_graph_wrapper_cls", None
),
)
self._fill_prefill_graph_metadata(
meta=meta, forward_batch=forward_batch, cache_indices=cache_indices
@@ -0,0 +1,88 @@
"""Fixed-capacity FlashInfer KDA plans for breakable prefill graphs."""
import numpy as np
import torch
import torch.nn.functional as F
class FlashInferKDAPrefillGraph:
def __init__(self, *, wrapper_cls, num_tokens, max_seqs, track, device):
self.wrapper_cls = wrapper_cls
# A workspace can be captured only once: each layer owns its plan.
self.wrappers = {}
self.cu_seqlens = torch.empty(max_seqs + 1, dtype=torch.int64, device=device)
self.cache_indices = torch.empty(max_seqs, dtype=torch.int32, device=device)
self.track = track
if track:
self.checkpoint_cu_starts = torch.empty_like(self.cu_seqlens)
self.checkpoint_indices = torch.full(
(num_tokens // 64,), -1, dtype=torch.int32, device=device
)
def fill(self, meta, extend_seq_lens, track_rows):
self.cu_seqlens.copy_(meta.cu_seqlens)
# Empty padded sequences may read their initial state, but do not write
# it. FlashInfer does not accept Triton's negative state-slot sentinel.
self.cache_indices.copy_(meta.cache_indices.clamp_min(0))
if self.track:
counts = np.asarray(extend_seq_lens, dtype=np.int64) // 64
starts = np.empty(self.cu_seqlens.numel(), dtype=np.int64)
starts[0] = 0
np.cumsum(counts, out=starts[1 : len(counts) + 1])
starts[len(counts) + 1 :] = starts[len(counts)]
indices = np.full(self.checkpoint_indices.numel(), -1, dtype=np.int32)
if track_rows is not None:
for row, chunk in track_rows.chunk_rows:
# Triton names the state before chunk; FlashInfer snapshots
# after each interval. Write that snapshot directly to row.
indices[starts[row] + chunk - 1] = row
self.checkpoint_cu_starts.copy_(
torch.from_numpy(starts).pin_memory(), non_blocking=True
)
self.checkpoint_indices.copy_(
torch.from_numpy(indices).pin_memory(), non_blocking=True
)
for wrapper in self.wrappers.values():
wrapper.plan(self.cu_seqlens)
def run(self, *, layer, q, k, v, g, beta, ssm_states, h_track):
num_tokens = q.shape[1]
if num_tokens <= self.cache_indices.numel():
# FlashInfer selects prefill only when T > number of sequence slots.
# Small buckets have one slot per token, including empty sequences.
# An unused trailing token keeps the prefill dispatch; cu_seqlens
# excludes it, so it cannot read or update any request state.
q, k, v, g, beta = (
F.pad(x, (0, 0) * (x.ndim - 2) + (0, 1)) for x in (q, k, v, g, beta)
)
wrapper = self.wrappers.get(layer.layer_id)
if wrapper is None:
wrapper = self.wrapper_cls(q.device)
wrapper.plan(self.cu_seqlens)
self.wrappers[layer.layer_id] = wrapper
checkpoints = {}
if self.track:
checkpoints = dict(
state_checkpoints=h_track,
checkpoint_cu_starts=self.checkpoint_cu_starts,
checkpoint_state_indices=self.checkpoint_indices,
checkpoint_every_n_tokens=64,
)
return wrapper.run(
q=q.contiguous(),
k=k.contiguous(),
v=v.contiguous(),
g=g.contiguous(),
beta=beta.contiguous(),
A_log=layer.A_log.reshape(-1).float().contiguous(),
dt_bias=layer.dt_bias.reshape(-1).float().contiguous(),
initial_state=ssm_states,
ssm_state_indices=self.cache_indices,
output_final_state=False,
use_qk_l2norm_in_kernel=True,
use_gate_in_kernel=True,
lower_bound=layer.lower_bound,
beta_is_logit=True,
output=torch.empty_like(q),
**checkpoints,
)[0][:, :num_tokens]
@@ -44,6 +44,9 @@ from sglang.kernels.ops.mamba.kda_prefill_track import (
kda_track_conv_window,
kda_track_ssm_state,
)
from sglang.srt.layers.attention.linear.kda_flashinfer_prefill_graph import (
FlashInferKDAPrefillGraph,
)
from sglang.srt.layers.attention.mamba.prefill_track_metadata import (
build_prefill_track_plan,
)
@@ -101,6 +104,7 @@ class KDAPrefillGraphMetadata(msgspec.Struct, frozen=True):
has_initial_state: torch.Tensor # [max_seqs] bool
track_dst: torch.Tensor # [max_seqs] int64
token_index: torch.Tensor # [num_tokens] int32 arange
flashinfer: Optional[FlashInferKDAPrefillGraph] = None
@classmethod
def allocate(
@@ -111,6 +115,7 @@ class KDAPrefillGraphMetadata(msgspec.Struct, frozen=True):
track: bool,
cache_index_dtype: torch.dtype,
device: torch.device,
flashinfer_wrapper_cls=None,
) -> KDAPrefillGraphMetadata:
max_seqs = max(1, min(num_tokens, max_seqs_cap))
max_chunks = triton.cdiv(num_tokens, KDA_CHUNK_SIZE) + max_seqs
@@ -149,6 +154,17 @@ class KDAPrefillGraphMetadata(msgspec.Struct, frozen=True):
has_initial_state=torch.zeros(max_seqs, dtype=torch.bool, device=device),
track_dst=torch.zeros(max_seqs, dtype=torch.int64, device=device),
token_index=torch.arange(num_tokens, dtype=torch.int32, device=device),
flashinfer=(
FlashInferKDAPrefillGraph(
wrapper_cls=flashinfer_wrapper_cls,
num_tokens=num_tokens,
max_seqs=max_seqs,
track=track,
device=device,
)
if flashinfer_wrapper_cls is not None
else None
),
)
def can_fit(self, extend_seq_lens: list[int]) -> bool:
@@ -185,6 +201,8 @@ class KDAPrefillGraphMetadata(msgspec.Struct, frozen=True):
self.has_initial_state[bs:].fill_(False)
if track_rows is not None:
self.track_dst[:bs].copy_(track_rows.dst[:bs])
if self.flashinfer is not None:
self.flashinfer.fill(self, extend_seq_lens, track_rows)
def _write_host_tables(
self, *, lens: np.ndarray, track_rows: Optional[KDAPrefillTrackRows]
@@ -350,27 +368,39 @@ def kda_extend_in_graph(
dtype=torch.float32,
device=ssm_states.device,
)
core_attn_out = kernel_dispatcher.extend(
q=q,
k=k,
v=v,
g=a,
beta=b,
ssm_states=ssm_states,
cache_indices=meta.cache_indices,
query_start_loc=meta.cu_seqlens,
A_log=layer.A_log,
dt_bias=layer.dt_bias,
lower_bound=layer.lower_bound,
beta_is_raw=gate_was_flat,
extend_seq_lens_cpu=None,
is_spec_decode=False,
return_intermediate_states=False,
track_state=h_track,
track_chunk_idx=meta.track_chunk_idx if meta.track else None,
chunk_indices=meta.chunk_indices,
chunk_offsets=meta.chunk_offsets,
)
if meta.flashinfer is not None:
core_attn_out = meta.flashinfer.run(
layer=layer,
q=q,
k=k,
v=v,
g=a,
beta=b,
ssm_states=ssm_states,
h_track=h_track,
)
else:
core_attn_out = kernel_dispatcher.extend(
q=q,
k=k,
v=v,
g=a,
beta=b,
ssm_states=ssm_states,
cache_indices=meta.cache_indices,
query_start_loc=meta.cu_seqlens,
A_log=layer.A_log,
dt_bias=layer.dt_bias,
lower_bound=layer.lower_bound,
beta_is_raw=gate_was_flat,
extend_seq_lens_cpu=None,
is_spec_decode=False,
return_intermediate_states=False,
track_state=h_track,
track_chunk_idx=meta.track_chunk_idx if meta.track else None,
chunk_indices=meta.chunk_indices,
chunk_offsets=meta.chunk_offsets,
)
if meta.track:
kda_track_ssm_state(
@@ -77,9 +77,13 @@ class FlashInferKDAPrefillKernel(LinearAttnKernelBase):
def __init__(self):
if torch.cuda.get_device_capability() not in ((10, 0), (10, 3)):
raise ValueError("FlashInfer KDA prefill requires SM100 or SM103")
from flashinfer.kda import recurrent_kda
from flashinfer import kda
self._recurrent_kda = recurrent_kda
self._recurrent_kda = kda.recurrent_kda
# Older FlashInfer installations retain the eager BCG break.
self.prefill_graph_wrapper_cls = getattr(
kda, "RecurrentKDAPrefillWrapper", None
)
def decode(self, *args, **kwargs):
raise NotImplementedError("FlashInferKDAPrefillKernel is prefill-only")