[Perf] cuda-graph: Capture FlashInfer KDA prefill (#45473)
This commit is contained in:
@@ -624,8 +624,15 @@ class KDAAttnBackend(MambaAttnBackendBase):
|
||||
def supports_prefill_graph_extend(self) -> bool:
|
||||
return (
|
||||
not envs.SGLANG_DISABLE_KDA_PREFILL_GRAPH_EXTEND.get()
|
||||
# Only the Triton chain guards padded rows (-1 state slots, chunks past T).
|
||||
and isinstance(self.kernel_dispatcher.extend_kernel, TritonKDAKernel)
|
||||
and (
|
||||
isinstance(self.kernel_dispatcher.extend_kernel, TritonKDAKernel)
|
||||
or getattr(
|
||||
self.kernel_dispatcher.extend_kernel,
|
||||
"prefill_graph_wrapper_cls",
|
||||
None,
|
||||
)
|
||||
is not None
|
||||
)
|
||||
# Fused-accept staging (flashinfer verify) writes per extend.
|
||||
and self.accept_lens_pool is None
|
||||
)
|
||||
@@ -638,13 +645,17 @@ class KDAAttnBackend(MambaAttnBackendBase):
|
||||
) -> KDAPrefillGraphMetadata:
|
||||
cache_indices = self._prefill_graph_cache_indices(forward_batch)
|
||||
meta = KDAPrefillGraphMetadata.allocate(
|
||||
num_tokens=forward_batch.positions.shape[0],
|
||||
# KDA sees global sequence order, even when CP positions are local.
|
||||
num_tokens=forward_batch.extend_num_tokens,
|
||||
max_seqs_cap=self.prefill_graph_max_seqs,
|
||||
# Any replay of this bucket may carry prefix-cache snapshots (the
|
||||
# extra buffer is only on with the radix cache).
|
||||
track=get_exec().mamba.enable_mamba_extra_buffer,
|
||||
cache_index_dtype=cache_indices.dtype,
|
||||
device=self.device,
|
||||
flashinfer_wrapper_cls=getattr(
|
||||
self.kernel_dispatcher.extend_kernel, "prefill_graph_wrapper_cls", None
|
||||
),
|
||||
)
|
||||
self._fill_prefill_graph_metadata(
|
||||
meta=meta, forward_batch=forward_batch, cache_indices=cache_indices
|
||||
|
||||
@@ -0,0 +1,88 @@
|
||||
"""Fixed-capacity FlashInfer KDA plans for breakable prefill graphs."""
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
|
||||
class FlashInferKDAPrefillGraph:
|
||||
def __init__(self, *, wrapper_cls, num_tokens, max_seqs, track, device):
|
||||
self.wrapper_cls = wrapper_cls
|
||||
# A workspace can be captured only once: each layer owns its plan.
|
||||
self.wrappers = {}
|
||||
self.cu_seqlens = torch.empty(max_seqs + 1, dtype=torch.int64, device=device)
|
||||
self.cache_indices = torch.empty(max_seqs, dtype=torch.int32, device=device)
|
||||
self.track = track
|
||||
if track:
|
||||
self.checkpoint_cu_starts = torch.empty_like(self.cu_seqlens)
|
||||
self.checkpoint_indices = torch.full(
|
||||
(num_tokens // 64,), -1, dtype=torch.int32, device=device
|
||||
)
|
||||
|
||||
def fill(self, meta, extend_seq_lens, track_rows):
|
||||
self.cu_seqlens.copy_(meta.cu_seqlens)
|
||||
# Empty padded sequences may read their initial state, but do not write
|
||||
# it. FlashInfer does not accept Triton's negative state-slot sentinel.
|
||||
self.cache_indices.copy_(meta.cache_indices.clamp_min(0))
|
||||
if self.track:
|
||||
counts = np.asarray(extend_seq_lens, dtype=np.int64) // 64
|
||||
starts = np.empty(self.cu_seqlens.numel(), dtype=np.int64)
|
||||
starts[0] = 0
|
||||
np.cumsum(counts, out=starts[1 : len(counts) + 1])
|
||||
starts[len(counts) + 1 :] = starts[len(counts)]
|
||||
indices = np.full(self.checkpoint_indices.numel(), -1, dtype=np.int32)
|
||||
if track_rows is not None:
|
||||
for row, chunk in track_rows.chunk_rows:
|
||||
# Triton names the state before chunk; FlashInfer snapshots
|
||||
# after each interval. Write that snapshot directly to row.
|
||||
indices[starts[row] + chunk - 1] = row
|
||||
self.checkpoint_cu_starts.copy_(
|
||||
torch.from_numpy(starts).pin_memory(), non_blocking=True
|
||||
)
|
||||
self.checkpoint_indices.copy_(
|
||||
torch.from_numpy(indices).pin_memory(), non_blocking=True
|
||||
)
|
||||
for wrapper in self.wrappers.values():
|
||||
wrapper.plan(self.cu_seqlens)
|
||||
|
||||
def run(self, *, layer, q, k, v, g, beta, ssm_states, h_track):
|
||||
num_tokens = q.shape[1]
|
||||
if num_tokens <= self.cache_indices.numel():
|
||||
# FlashInfer selects prefill only when T > number of sequence slots.
|
||||
# Small buckets have one slot per token, including empty sequences.
|
||||
# An unused trailing token keeps the prefill dispatch; cu_seqlens
|
||||
# excludes it, so it cannot read or update any request state.
|
||||
q, k, v, g, beta = (
|
||||
F.pad(x, (0, 0) * (x.ndim - 2) + (0, 1)) for x in (q, k, v, g, beta)
|
||||
)
|
||||
wrapper = self.wrappers.get(layer.layer_id)
|
||||
if wrapper is None:
|
||||
wrapper = self.wrapper_cls(q.device)
|
||||
wrapper.plan(self.cu_seqlens)
|
||||
self.wrappers[layer.layer_id] = wrapper
|
||||
checkpoints = {}
|
||||
if self.track:
|
||||
checkpoints = dict(
|
||||
state_checkpoints=h_track,
|
||||
checkpoint_cu_starts=self.checkpoint_cu_starts,
|
||||
checkpoint_state_indices=self.checkpoint_indices,
|
||||
checkpoint_every_n_tokens=64,
|
||||
)
|
||||
return wrapper.run(
|
||||
q=q.contiguous(),
|
||||
k=k.contiguous(),
|
||||
v=v.contiguous(),
|
||||
g=g.contiguous(),
|
||||
beta=beta.contiguous(),
|
||||
A_log=layer.A_log.reshape(-1).float().contiguous(),
|
||||
dt_bias=layer.dt_bias.reshape(-1).float().contiguous(),
|
||||
initial_state=ssm_states,
|
||||
ssm_state_indices=self.cache_indices,
|
||||
output_final_state=False,
|
||||
use_qk_l2norm_in_kernel=True,
|
||||
use_gate_in_kernel=True,
|
||||
lower_bound=layer.lower_bound,
|
||||
beta_is_logit=True,
|
||||
output=torch.empty_like(q),
|
||||
**checkpoints,
|
||||
)[0][:, :num_tokens]
|
||||
@@ -44,6 +44,9 @@ from sglang.kernels.ops.mamba.kda_prefill_track import (
|
||||
kda_track_conv_window,
|
||||
kda_track_ssm_state,
|
||||
)
|
||||
from sglang.srt.layers.attention.linear.kda_flashinfer_prefill_graph import (
|
||||
FlashInferKDAPrefillGraph,
|
||||
)
|
||||
from sglang.srt.layers.attention.mamba.prefill_track_metadata import (
|
||||
build_prefill_track_plan,
|
||||
)
|
||||
@@ -101,6 +104,7 @@ class KDAPrefillGraphMetadata(msgspec.Struct, frozen=True):
|
||||
has_initial_state: torch.Tensor # [max_seqs] bool
|
||||
track_dst: torch.Tensor # [max_seqs] int64
|
||||
token_index: torch.Tensor # [num_tokens] int32 arange
|
||||
flashinfer: Optional[FlashInferKDAPrefillGraph] = None
|
||||
|
||||
@classmethod
|
||||
def allocate(
|
||||
@@ -111,6 +115,7 @@ class KDAPrefillGraphMetadata(msgspec.Struct, frozen=True):
|
||||
track: bool,
|
||||
cache_index_dtype: torch.dtype,
|
||||
device: torch.device,
|
||||
flashinfer_wrapper_cls=None,
|
||||
) -> KDAPrefillGraphMetadata:
|
||||
max_seqs = max(1, min(num_tokens, max_seqs_cap))
|
||||
max_chunks = triton.cdiv(num_tokens, KDA_CHUNK_SIZE) + max_seqs
|
||||
@@ -149,6 +154,17 @@ class KDAPrefillGraphMetadata(msgspec.Struct, frozen=True):
|
||||
has_initial_state=torch.zeros(max_seqs, dtype=torch.bool, device=device),
|
||||
track_dst=torch.zeros(max_seqs, dtype=torch.int64, device=device),
|
||||
token_index=torch.arange(num_tokens, dtype=torch.int32, device=device),
|
||||
flashinfer=(
|
||||
FlashInferKDAPrefillGraph(
|
||||
wrapper_cls=flashinfer_wrapper_cls,
|
||||
num_tokens=num_tokens,
|
||||
max_seqs=max_seqs,
|
||||
track=track,
|
||||
device=device,
|
||||
)
|
||||
if flashinfer_wrapper_cls is not None
|
||||
else None
|
||||
),
|
||||
)
|
||||
|
||||
def can_fit(self, extend_seq_lens: list[int]) -> bool:
|
||||
@@ -185,6 +201,8 @@ class KDAPrefillGraphMetadata(msgspec.Struct, frozen=True):
|
||||
self.has_initial_state[bs:].fill_(False)
|
||||
if track_rows is not None:
|
||||
self.track_dst[:bs].copy_(track_rows.dst[:bs])
|
||||
if self.flashinfer is not None:
|
||||
self.flashinfer.fill(self, extend_seq_lens, track_rows)
|
||||
|
||||
def _write_host_tables(
|
||||
self, *, lens: np.ndarray, track_rows: Optional[KDAPrefillTrackRows]
|
||||
@@ -350,27 +368,39 @@ def kda_extend_in_graph(
|
||||
dtype=torch.float32,
|
||||
device=ssm_states.device,
|
||||
)
|
||||
core_attn_out = kernel_dispatcher.extend(
|
||||
q=q,
|
||||
k=k,
|
||||
v=v,
|
||||
g=a,
|
||||
beta=b,
|
||||
ssm_states=ssm_states,
|
||||
cache_indices=meta.cache_indices,
|
||||
query_start_loc=meta.cu_seqlens,
|
||||
A_log=layer.A_log,
|
||||
dt_bias=layer.dt_bias,
|
||||
lower_bound=layer.lower_bound,
|
||||
beta_is_raw=gate_was_flat,
|
||||
extend_seq_lens_cpu=None,
|
||||
is_spec_decode=False,
|
||||
return_intermediate_states=False,
|
||||
track_state=h_track,
|
||||
track_chunk_idx=meta.track_chunk_idx if meta.track else None,
|
||||
chunk_indices=meta.chunk_indices,
|
||||
chunk_offsets=meta.chunk_offsets,
|
||||
)
|
||||
if meta.flashinfer is not None:
|
||||
core_attn_out = meta.flashinfer.run(
|
||||
layer=layer,
|
||||
q=q,
|
||||
k=k,
|
||||
v=v,
|
||||
g=a,
|
||||
beta=b,
|
||||
ssm_states=ssm_states,
|
||||
h_track=h_track,
|
||||
)
|
||||
else:
|
||||
core_attn_out = kernel_dispatcher.extend(
|
||||
q=q,
|
||||
k=k,
|
||||
v=v,
|
||||
g=a,
|
||||
beta=b,
|
||||
ssm_states=ssm_states,
|
||||
cache_indices=meta.cache_indices,
|
||||
query_start_loc=meta.cu_seqlens,
|
||||
A_log=layer.A_log,
|
||||
dt_bias=layer.dt_bias,
|
||||
lower_bound=layer.lower_bound,
|
||||
beta_is_raw=gate_was_flat,
|
||||
extend_seq_lens_cpu=None,
|
||||
is_spec_decode=False,
|
||||
return_intermediate_states=False,
|
||||
track_state=h_track,
|
||||
track_chunk_idx=meta.track_chunk_idx if meta.track else None,
|
||||
chunk_indices=meta.chunk_indices,
|
||||
chunk_offsets=meta.chunk_offsets,
|
||||
)
|
||||
|
||||
if meta.track:
|
||||
kda_track_ssm_state(
|
||||
|
||||
@@ -77,9 +77,13 @@ class FlashInferKDAPrefillKernel(LinearAttnKernelBase):
|
||||
def __init__(self):
|
||||
if torch.cuda.get_device_capability() not in ((10, 0), (10, 3)):
|
||||
raise ValueError("FlashInfer KDA prefill requires SM100 or SM103")
|
||||
from flashinfer.kda import recurrent_kda
|
||||
from flashinfer import kda
|
||||
|
||||
self._recurrent_kda = recurrent_kda
|
||||
self._recurrent_kda = kda.recurrent_kda
|
||||
# Older FlashInfer installations retain the eager BCG break.
|
||||
self.prefill_graph_wrapper_cls = getattr(
|
||||
kda, "RecurrentKDAPrefillWrapper", None
|
||||
)
|
||||
|
||||
def decode(self, *args, **kwargs):
|
||||
raise NotImplementedError("FlashInferKDAPrefillKernel is prefill-only")
|
||||
|
||||
Reference in New Issue
Block a user