[Perf] cuda-graph: Capture FlashInfer KDA prefill (#45473)

This commit is contained in:
Baizhou Zhang
2026-10-21 11:57:23 -07:00
committed by GitHub
parent 740d5f9f9e
commit f06d1bb1f4
5 changed files with 241 additions and 46 deletions
@@ -624,8 +624,15 @@ class KDAAttnBackend(MambaAttnBackendBase):
def supports_prefill_graph_extend(self) -> bool:
return (
not envs.SGLANG_DISABLE_KDA_PREFILL_GRAPH_EXTEND.get()
# Only the Triton chain guards padded rows (-1 state slots, chunks past T).
and isinstance(self.kernel_dispatcher.extend_kernel, TritonKDAKernel)
and (
isinstance(self.kernel_dispatcher.extend_kernel, TritonKDAKernel)
or getattr(
self.kernel_dispatcher.extend_kernel,
"prefill_graph_wrapper_cls",
None,
)
is not None
)
# Fused-accept staging (flashinfer verify) writes per extend.
and self.accept_lens_pool is None
)
@@ -638,13 +645,17 @@ class KDAAttnBackend(MambaAttnBackendBase):
) -> KDAPrefillGraphMetadata:
cache_indices = self._prefill_graph_cache_indices(forward_batch)
meta = KDAPrefillGraphMetadata.allocate(
num_tokens=forward_batch.positions.shape[0],
# KDA sees global sequence order, even when CP positions are local.
num_tokens=forward_batch.extend_num_tokens,
max_seqs_cap=self.prefill_graph_max_seqs,
# Any replay of this bucket may carry prefix-cache snapshots (the
# extra buffer is only on with the radix cache).
track=get_exec().mamba.enable_mamba_extra_buffer,
cache_index_dtype=cache_indices.dtype,
device=self.device,
flashinfer_wrapper_cls=getattr(
self.kernel_dispatcher.extend_kernel, "prefill_graph_wrapper_cls", None
),
)
self._fill_prefill_graph_metadata(
meta=meta, forward_batch=forward_batch, cache_indices=cache_indices
@@ -0,0 +1,88 @@
"""Fixed-capacity FlashInfer KDA plans for breakable prefill graphs."""
import numpy as np
import torch
import torch.nn.functional as F
class FlashInferKDAPrefillGraph:
def __init__(self, *, wrapper_cls, num_tokens, max_seqs, track, device):
self.wrapper_cls = wrapper_cls
# A workspace can be captured only once: each layer owns its plan.
self.wrappers = {}
self.cu_seqlens = torch.empty(max_seqs + 1, dtype=torch.int64, device=device)
self.cache_indices = torch.empty(max_seqs, dtype=torch.int32, device=device)
self.track = track
if track:
self.checkpoint_cu_starts = torch.empty_like(self.cu_seqlens)
self.checkpoint_indices = torch.full(
(num_tokens // 64,), -1, dtype=torch.int32, device=device
)
def fill(self, meta, extend_seq_lens, track_rows):
self.cu_seqlens.copy_(meta.cu_seqlens)
# Empty padded sequences may read their initial state, but do not write
# it. FlashInfer does not accept Triton's negative state-slot sentinel.
self.cache_indices.copy_(meta.cache_indices.clamp_min(0))
if self.track:
counts = np.asarray(extend_seq_lens, dtype=np.int64) // 64
starts = np.empty(self.cu_seqlens.numel(), dtype=np.int64)
starts[0] = 0
np.cumsum(counts, out=starts[1 : len(counts) + 1])
starts[len(counts) + 1 :] = starts[len(counts)]
indices = np.full(self.checkpoint_indices.numel(), -1, dtype=np.int32)
if track_rows is not None:
for row, chunk in track_rows.chunk_rows:
# Triton names the state before chunk; FlashInfer snapshots
# after each interval. Write that snapshot directly to row.
indices[starts[row] + chunk - 1] = row
self.checkpoint_cu_starts.copy_(
torch.from_numpy(starts).pin_memory(), non_blocking=True
)
self.checkpoint_indices.copy_(
torch.from_numpy(indices).pin_memory(), non_blocking=True
)
for wrapper in self.wrappers.values():
wrapper.plan(self.cu_seqlens)
def run(self, *, layer, q, k, v, g, beta, ssm_states, h_track):
num_tokens = q.shape[1]
if num_tokens <= self.cache_indices.numel():
# FlashInfer selects prefill only when T > number of sequence slots.
# Small buckets have one slot per token, including empty sequences.
# An unused trailing token keeps the prefill dispatch; cu_seqlens
# excludes it, so it cannot read or update any request state.
q, k, v, g, beta = (
F.pad(x, (0, 0) * (x.ndim - 2) + (0, 1)) for x in (q, k, v, g, beta)
)
wrapper = self.wrappers.get(layer.layer_id)
if wrapper is None:
wrapper = self.wrapper_cls(q.device)
wrapper.plan(self.cu_seqlens)
self.wrappers[layer.layer_id] = wrapper
checkpoints = {}
if self.track:
checkpoints = dict(
state_checkpoints=h_track,
checkpoint_cu_starts=self.checkpoint_cu_starts,
checkpoint_state_indices=self.checkpoint_indices,
checkpoint_every_n_tokens=64,
)
return wrapper.run(
q=q.contiguous(),
k=k.contiguous(),
v=v.contiguous(),
g=g.contiguous(),
beta=beta.contiguous(),
A_log=layer.A_log.reshape(-1).float().contiguous(),
dt_bias=layer.dt_bias.reshape(-1).float().contiguous(),
initial_state=ssm_states,
ssm_state_indices=self.cache_indices,
output_final_state=False,
use_qk_l2norm_in_kernel=True,
use_gate_in_kernel=True,
lower_bound=layer.lower_bound,
beta_is_logit=True,
output=torch.empty_like(q),
**checkpoints,
)[0][:, :num_tokens]
@@ -44,6 +44,9 @@ from sglang.kernels.ops.mamba.kda_prefill_track import (
kda_track_conv_window,
kda_track_ssm_state,
)
from sglang.srt.layers.attention.linear.kda_flashinfer_prefill_graph import (
FlashInferKDAPrefillGraph,
)
from sglang.srt.layers.attention.mamba.prefill_track_metadata import (
build_prefill_track_plan,
)
@@ -101,6 +104,7 @@ class KDAPrefillGraphMetadata(msgspec.Struct, frozen=True):
has_initial_state: torch.Tensor # [max_seqs] bool
track_dst: torch.Tensor # [max_seqs] int64
token_index: torch.Tensor # [num_tokens] int32 arange
flashinfer: Optional[FlashInferKDAPrefillGraph] = None
@classmethod
def allocate(
@@ -111,6 +115,7 @@ class KDAPrefillGraphMetadata(msgspec.Struct, frozen=True):
track: bool,
cache_index_dtype: torch.dtype,
device: torch.device,
flashinfer_wrapper_cls=None,
) -> KDAPrefillGraphMetadata:
max_seqs = max(1, min(num_tokens, max_seqs_cap))
max_chunks = triton.cdiv(num_tokens, KDA_CHUNK_SIZE) + max_seqs
@@ -149,6 +154,17 @@ class KDAPrefillGraphMetadata(msgspec.Struct, frozen=True):
has_initial_state=torch.zeros(max_seqs, dtype=torch.bool, device=device),
track_dst=torch.zeros(max_seqs, dtype=torch.int64, device=device),
token_index=torch.arange(num_tokens, dtype=torch.int32, device=device),
flashinfer=(
FlashInferKDAPrefillGraph(
wrapper_cls=flashinfer_wrapper_cls,
num_tokens=num_tokens,
max_seqs=max_seqs,
track=track,
device=device,
)
if flashinfer_wrapper_cls is not None
else None
),
)
def can_fit(self, extend_seq_lens: list[int]) -> bool:
@@ -185,6 +201,8 @@ class KDAPrefillGraphMetadata(msgspec.Struct, frozen=True):
self.has_initial_state[bs:].fill_(False)
if track_rows is not None:
self.track_dst[:bs].copy_(track_rows.dst[:bs])
if self.flashinfer is not None:
self.flashinfer.fill(self, extend_seq_lens, track_rows)
def _write_host_tables(
self, *, lens: np.ndarray, track_rows: Optional[KDAPrefillTrackRows]
@@ -350,27 +368,39 @@ def kda_extend_in_graph(
dtype=torch.float32,
device=ssm_states.device,
)
core_attn_out = kernel_dispatcher.extend(
q=q,
k=k,
v=v,
g=a,
beta=b,
ssm_states=ssm_states,
cache_indices=meta.cache_indices,
query_start_loc=meta.cu_seqlens,
A_log=layer.A_log,
dt_bias=layer.dt_bias,
lower_bound=layer.lower_bound,
beta_is_raw=gate_was_flat,
extend_seq_lens_cpu=None,
is_spec_decode=False,
return_intermediate_states=False,
track_state=h_track,
track_chunk_idx=meta.track_chunk_idx if meta.track else None,
chunk_indices=meta.chunk_indices,
chunk_offsets=meta.chunk_offsets,
)
if meta.flashinfer is not None:
core_attn_out = meta.flashinfer.run(
layer=layer,
q=q,
k=k,
v=v,
g=a,
beta=b,
ssm_states=ssm_states,
h_track=h_track,
)
else:
core_attn_out = kernel_dispatcher.extend(
q=q,
k=k,
v=v,
g=a,
beta=b,
ssm_states=ssm_states,
cache_indices=meta.cache_indices,
query_start_loc=meta.cu_seqlens,
A_log=layer.A_log,
dt_bias=layer.dt_bias,
lower_bound=layer.lower_bound,
beta_is_raw=gate_was_flat,
extend_seq_lens_cpu=None,
is_spec_decode=False,
return_intermediate_states=False,
track_state=h_track,
track_chunk_idx=meta.track_chunk_idx if meta.track else None,
chunk_indices=meta.chunk_indices,
chunk_offsets=meta.chunk_offsets,
)
if meta.track:
kda_track_ssm_state(
@@ -77,9 +77,13 @@ class FlashInferKDAPrefillKernel(LinearAttnKernelBase):
def __init__(self):
if torch.cuda.get_device_capability() not in ((10, 0), (10, 3)):
raise ValueError("FlashInfer KDA prefill requires SM100 or SM103")
from flashinfer.kda import recurrent_kda
from flashinfer import kda
self._recurrent_kda = recurrent_kda
self._recurrent_kda = kda.recurrent_kda
# Older FlashInfer installations retain the eager BCG break.
self.prefill_graph_wrapper_cls = getattr(
kda, "RecurrentKDAPrefillWrapper", None
)
def decode(self, *args, **kwargs):
raise NotImplementedError("FlashInferKDAPrefillKernel is prefill-only")
@@ -4,8 +4,10 @@ eager KDAAttnBackend.forward_extend.
One bucket is captured once, then replayed for batches of different shapes:
several sequences, cached prefixes, padding past the batch, chunk-aligned and
unaligned prefix-cache snapshots, and a batch without tracking. Each replay must
match the eager path bit for bit on the output, the conv / SSM state pools and
the track slots, and must leave every other pool slot untouched.
match the eager path on the output, the conv / SSM state pools and the track
slots, and must leave every other pool slot untouched. Comparisons are bitwise
except FlashInfer's small padded plans and the eager Triton fallback for
one-token sequences, which can use different floating-point operation orders.
"""
import unittest
@@ -58,7 +60,7 @@ def _layer():
)
def _backend(conv_pool, ssm_pool):
def _backend(conv_pool, ssm_pool, prefill_backend=LinearAttnKernelBackend.TRITON):
"""The production backend on test pools: slot of request i is i + 1."""
backend = KDAAttnBackend.__new__(KDAAttnBackend)
cache = SimpleNamespace(conv=[conv_pool], temporal=ssm_pool)
@@ -76,7 +78,7 @@ def _backend(conv_pool, ssm_pool):
backend.prefill_graph_metadata = None
backend.kernel_dispatcher = KDAKernelDispatcher(
LinearAttnKernelBackend.TRITON,
LinearAttnKernelBackend.TRITON,
prefill_backend,
LinearAttnKernelBackend.TRITON,
)
return backend
@@ -127,6 +129,8 @@ def _batch(extend_lens, prefix_lens, track=None):
class TestKDAPrefillGraph(CustomTestCase):
prefill_backend = LinearAttnKernelBackend.TRITON
def setUp(self):
self.layer = _layer()
g = torch.Generator(device="cuda").manual_seed(11)
@@ -139,7 +143,7 @@ class TestKDAPrefillGraph(CustomTestCase):
)
* 0.05
)
self.backend = _backend(self.conv_pool, self.ssm_pool)
self.backend = _backend(self.conv_pool, self.ssm_pool, self.prefill_backend)
# Static inputs the captured graph reads.
self.mixed = torch.zeros(BUCKET, DIM, device="cuda", dtype=torch.bfloat16)
self.a = torch.zeros(1, BUCKET, HEADS * HEAD_DIM, device="cuda")
@@ -157,21 +161,26 @@ class TestKDAPrefillGraph(CustomTestCase):
cache_indices=self.backend._prefill_graph_cache_indices(batch),
)
def _capture(self):
meta = KDAPrefillGraphMetadata.allocate(
num_tokens=BUCKET,
max_seqs_cap=MAX_SEQS,
track=True,
cache_index_dtype=torch.int32,
device=torch.device("cuda"),
)
capture_batch = _batch([BUCKET], [0])
self._fill(meta, capture_batch)
def _capture(self, *, track=True, num_tokens=BUCKET, max_seqs=MAX_SEQS):
from sglang.srt.runtime_context import get_context
capture_batch = _batch([num_tokens], [0])
capture_batch.extend_num_tokens = num_tokens
capture_batch.positions = torch.arange(num_tokens, device="cuda")
self.backend.prefill_graph_max_seqs = max_seqs
with get_context().override_server_args(
disable_radix_cache=not track, mamba_radix_cache_strategy="extra_buffer"
):
meta = self.backend.init_prefill_graph_metadata(capture_batch)
snapshot = (self.conv_pool.clone(), self.ssm_pool.clone())
def run():
return self.backend.forward_extend(
self.layer, capture_batch, self.mixed, self.a, self.b
self.layer,
capture_batch,
self.mixed[:num_tokens],
self.a[:, :num_tokens],
self.b[:, :num_tokens],
)
stream = torch.cuda.Stream()
@@ -180,7 +189,7 @@ class TestKDAPrefillGraph(CustomTestCase):
run() # warmup: Triton compile / autotune outside the capture
torch.cuda.current_stream().wait_stream(stream)
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
with torch.cuda.graph(graph, stream=stream):
out = run()
# Warmup and capture ran on the pools; restore them.
self.conv_pool.copy_(snapshot[0])
@@ -205,7 +214,7 @@ class TestKDAPrefillGraph(CustomTestCase):
# Eager reference on cloned pools.
ref_conv, ref_ssm = self.conv_pool.clone(), self.ssm_pool.clone()
ref_backend = _backend(ref_conv, ref_ssm)
ref_backend = _backend(ref_conv, ref_ssm, self.prefill_backend)
ref_batch = _batch(lens, prefixes, track=track)
ref_backend.init_forward_metadata(ref_batch)
ref_out = ref_backend.forward_extend(
@@ -220,10 +229,34 @@ class TestKDAPrefillGraph(CustomTestCase):
graph.replay()
torch.cuda.synchronize()
self.assertTrue(torch.equal(out[:, :total], ref_out[:, :total]))
different_kernel_schedule = (
self.prefill_backend == LinearAttnKernelBackend.FLASHINFER
and (max(lens) == 1 or meta.num_tokens <= meta.max_seqs)
)
if different_kernel_schedule:
# Planned small FlashInfer buckets and the eager Triton fallback
# need numerical comparison. Unused slots must remain bitwise equal.
torch.testing.assert_close(
out[:, :total], ref_out[:, :total], rtol=1e-2, atol=1e-4
)
touched = list(range(1, len(lens) + 1))
touched += [NUM_SLOTS // 2 + row for row in (track or {})]
untouched = [slot for slot in range(NUM_SLOTS) if slot not in touched]
torch.testing.assert_close(
self.ssm_pool[touched], ref_ssm[touched], rtol=1e-2, atol=1e-3
)
self.assertTrue(torch.equal(self.ssm_pool[untouched], ref_ssm[untouched]))
else:
self.assertTrue(torch.equal(out[:, :total], ref_out[:, :total]))
self.assertTrue(torch.equal(self.ssm_pool, ref_ssm), "ssm pool differs")
self.assertTrue(bool((out[:, total:] == 0).all()), "padded rows not zeroed")
self.assertTrue(torch.equal(self.conv_pool, ref_conv), "conv pool differs")
self.assertTrue(torch.equal(self.ssm_pool, ref_ssm), "ssm pool differs")
def test_without_radix_tracking(self):
meta, graph, out = self._capture(track=False)
self._check_replay(
meta, graph, out, lens=[129, 7], prefixes=[64, 0], track=None, seed=300
)
def test_replays_match_eager(self):
meta, graph, out = self._capture()
@@ -240,6 +273,7 @@ class TestKDAPrefillGraph(CustomTestCase):
),
([200, 11], [0, 0], None),
([1], [900], None),
([1] * MAX_SEQS, [64] * MAX_SEQS, None),
]
for i, (lens, prefixes, track) in enumerate(cases):
with self.subTest(case=i):
@@ -270,5 +304,33 @@ class TestKDAPrefillGraph(CustomTestCase):
self.assertLessEqual(chunks, meta.max_chunks)
class TestFlashInferKDAPrefillGraph(TestKDAPrefillGraph):
prefill_backend = LinearAttnKernelBackend.FLASHINFER
def test_small_buckets_with_padded_sequences(self):
for num_tokens in (1, 8, 16, 128):
with self.subTest(num_tokens=num_tokens):
meta, graph, out = self._capture(num_tokens=num_tokens, max_seqs=128)
for lens in ([num_tokens], [1] * min(num_tokens, MAX_SEQS)):
self._check_replay(
meta,
graph,
out,
lens=lens,
prefixes=[64] * len(lens),
track={0: 70} if lens == [128] else None,
seed=400 + num_tokens,
)
@classmethod
def setUpClass(cls):
if torch.cuda.get_device_capability() not in ((10, 0), (10, 3)):
raise unittest.SkipTest("FlashInfer KDA prefill requires SM100/SM103")
from flashinfer import kda
if not hasattr(kda, "RecurrentKDAPrefillWrapper"):
raise unittest.SkipTest("FlashInfer lacks the KDA prefill graph wrapper")
if __name__ == "__main__":
unittest.main()