[GLM-5.3-Flash] Capture the KDA extend in breakable prefill CUDA graphs (#41105)

Co-authored-by: luoyuan.luo <luoyuan.luo@antgroup.com>
This commit is contained in:
Yuan Luo
2026-10-09 13:37:20 +08:00
committed by GitHub
co-authored by luoyuan.luo
parent 33e7b4cc4b
commit 9a1aeacc3b
15 changed files with 1177 additions and 22 deletions
@@ -363,6 +363,7 @@ def chunk_gated_delta_rule_fwd_h(
inplace_update: bool = True,
track_state: Optional[torch.Tensor] = None,
track_chunk_idx: Optional[torch.Tensor] = None,
chunk_offsets: Optional[torch.Tensor] = None,
) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
assert not (use_exp2 and g is not None), (
"use_exp2 covers only the per-channel gk path; scalar g stays natural-exp"
@@ -386,11 +387,11 @@ def chunk_gated_delta_rule_fwd_h(
if cu_seqlens is None:
N, NT, chunk_offsets = B, triton.cdiv(T, BT), None
else:
N, NT, chunk_offsets = (
len(cu_seqlens) - 1,
len(chunk_indices),
prepare_chunk_offsets(cu_seqlens, BT),
)
# A caller-built chunk_offsets (CUDA-graph capture) must not go through
# prepare_chunk_offsets: its tensor_cache would replay capture-time values.
N, NT = len(cu_seqlens) - 1, len(chunk_indices)
if chunk_offsets is None:
chunk_offsets = prepare_chunk_offsets(cu_seqlens, BT)
assert K <= 256, "current kernel does not support head dimension larger than 256."
h = k.new_empty(B, NT, H, V, K)
+28 -6
View File
@@ -554,6 +554,12 @@ def _recompute_w_u_fwd_kernel(
T = eos - bos
else:
bos, eos = i_b * T, i_b * T + T
# Padded chunk_indices rows (kda_prefill_graph.py) start past T, where every
# store below is masked; this early exit only skips their wasted work.
if i_t * BT >= T:
return
p_b = tl.make_block_ptr(beta + bos * H + i_h, (T,), (H,), (i_t * BT,), (BT,), (0,))
b_b = tl.load(p_b, boundary_check=(0,))
@@ -797,6 +803,11 @@ def chunk_gla_fwd_kernel_o(
i_tg = i_b * NT + i_t
bos, eos = i_b * T, i_b * T + T
# Padded chunk_indices rows (kda_prefill_graph.py) start past T, where every
# store below is masked; this early exit only skips their wasted work.
if i_t * BT >= T:
return
m_s = tl.arange(0, BT)[:, None] >= tl.arange(0, BT)[None, :]
b_o = tl.zeros([BT, BV], dtype=tl.float32)
@@ -973,6 +984,11 @@ def kda_gate_chunk_cumsum_vector_kernel(
else:
bos, eos = i_b * T, i_b * T + T
# Padded chunk_indices rows (kda_prefill_graph.py) start past T, where every
# store below is masked; this early exit only skips their wasted work.
if i_t * BT >= T:
return
p_s = tl.make_block_ptr(
s + (bos * H + i_h) * S,
(T, S),
@@ -1145,15 +1161,16 @@ def chunk_kda_fwd(
track_state: Optional[torch.Tensor] = None,
track_chunk_idx: Optional[torch.Tensor] = None,
beta_is_raw: bool = False,
chunk_indices: Optional[torch.Tensor] = None,
chunk_offsets: Optional[torch.Tensor] = None,
):
chunk_size = 64
# Pre-compute chunk indices once and thread through all downstream kernels.
# Without this, each of the 4 callees would recompute independently.
chunk_indices = (
prepare_chunk_indices(cu_seqlens, chunk_size)
if cu_seqlens is not None
else None
)
# Without this, each of the 4 callees would recompute independently. A
# CUDA-graph caller passes padded chunk_indices / chunk_offsets it rebuilds
# before every replay (kda_prefill_graph.py).
if chunk_indices is None and cu_seqlens is not None:
chunk_indices = prepare_chunk_indices(cu_seqlens, chunk_size)
if A_log is not None:
# Fused: gate activation + chunk-local cumsum in one kernel.
@@ -1227,6 +1244,7 @@ def chunk_kda_fwd(
use_exp2=True,
track_state=track_state,
track_chunk_idx=track_chunk_idx,
chunk_offsets=chunk_offsets,
)
del w, u, kg
@@ -1271,6 +1289,8 @@ def chunk_kda(
track_state: Optional[torch.Tensor] = None,
track_chunk_idx: Optional[torch.Tensor] = None,
beta_is_raw: bool = False,
chunk_indices: Optional[torch.Tensor] = None,
chunk_offsets: Optional[torch.Tensor] = None,
**kwargs,
):
if scale is None:
@@ -1298,4 +1318,6 @@ def chunk_kda(
track_state=track_state,
track_chunk_idx=track_chunk_idx,
beta_is_raw=beta_is_raw,
chunk_indices=chunk_indices,
chunk_offsets=chunk_offsets,
)
@@ -0,0 +1,148 @@
"""Prefix-cache track copies for the CUDA-graph captured KDA extend.
The eager extend selects tracked rows on the host (``nonzero`` / masked index
lists) and scatters with data-dependent shapes. Under a breakable prefill CUDA
graph the batch changes on every replay, so these kernels walk a fixed number of
rows (the bucket's sequence bound) and read a per-row mode instead:
* ``track_mode[r] == 0``: row ``r`` is not tracked (or is padding); no-op.
* ``track_mode[r] == 1``: the tracked length is chunk aligned; the snapshot is
the post-extend state of the row's own cache slot.
* ``track_mode[r] == 2``: the snapshot is the chunk-boundary state the chunked
delta-rule kernel wrote to ``h_track[r]`` (fp32).
Every tracked row (mode 1 or 2) also snapshots its conv window: the
``STATE_LEN`` raw pre-conv input rows ending at the tracked boundary.
"""
import torch
import triton
import triton.language as tl
TRACK_NONE = 0
TRACK_FINAL_STATE = 1
TRACK_CHUNK_STATE = 2
@triton.jit
def _kda_track_conv_window_kernel(
conv_pool, # [slots, STATE_LEN, dim]
x, # [tokens, dim] raw pre-conv input
track_mode, # [rows] int32
track_conv_start, # [rows] int32, first token row of the window (unclamped)
track_dst, # [rows] int64 physical track slot
num_tokens, # [1] int32 logical tokens of the batch
stride_pool_slot,
stride_pool_row,
stride_x_token,
dim,
STATE_LEN: tl.constexpr,
BLOCK_D: tl.constexpr,
):
r = tl.program_id(0)
mode = tl.load(track_mode + r)
if mode == 0:
return
dst = tl.load(track_dst + r).to(tl.int64)
start = tl.load(track_conv_start + r)
total = tl.load(num_tokens)
offs = tl.program_id(1) * BLOCK_D + tl.arange(0, BLOCK_D)
mask = offs < dim
for j in tl.static_range(STATE_LEN):
# Same clamp as the eager index list: clamp(0, total - 1).
t = tl.minimum(tl.maximum(start + j, 0), total - 1).to(tl.int64)
val = tl.load(x + t * stride_x_token + offs, mask=mask)
tl.store(
conv_pool + dst * stride_pool_slot + j * stride_pool_row + offs,
val.to(conv_pool.dtype.element_ty),
mask=mask,
)
@triton.jit
def _kda_track_ssm_kernel(
ssm_pool, # [slots, ...] per-slot contiguous
h_track, # [rows, ...] fp32 chunk-boundary snapshots
track_mode, # [rows] int32
track_dst, # [rows] int64 physical track slot
cache_indices, # [rows] physical cache slot of the row
stride_ssm_slot,
stride_h_row,
numel,
BLOCK: tl.constexpr,
):
r = tl.program_id(0)
mode = tl.load(track_mode + r)
if mode == 0:
return
dst = tl.load(track_dst + r).to(tl.int64)
offs = tl.program_id(1) * BLOCK + tl.arange(0, BLOCK)
mask = offs < numel
if mode == 1:
src = tl.load(cache_indices + r).to(tl.int64)
val = tl.load(ssm_pool + src * stride_ssm_slot + offs, mask=mask)
else:
val = tl.load(h_track + r.to(tl.int64) * stride_h_row + offs, mask=mask)
tl.store(
ssm_pool + dst * stride_ssm_slot + offs,
val.to(ssm_pool.dtype.element_ty),
mask=mask,
)
def kda_track_conv_window(
conv_pool: torch.Tensor,
x: torch.Tensor,
track_mode: torch.Tensor,
track_conv_start: torch.Tensor,
track_dst: torch.Tensor,
num_tokens: torch.Tensor,
) -> None:
"""``conv_pool[track_dst[r]] = x[clamp(start[r] + arange(STATE_LEN))]`` for
every tracked row ``r``; the grid depends only on the row bound."""
rows = track_mode.shape[0]
state_len, dim = conv_pool.shape[-2], conv_pool.shape[-1]
assert x.shape[-1] == dim and x.stride(-1) == 1 and conv_pool.stride(-1) == 1
block_d = 1024
_kda_track_conv_window_kernel[(rows, triton.cdiv(dim, block_d))](
conv_pool,
x,
track_mode,
track_conv_start,
track_dst,
num_tokens,
conv_pool.stride(0),
conv_pool.stride(1),
x.stride(0),
dim,
STATE_LEN=state_len,
BLOCK_D=block_d,
)
def kda_track_ssm_state(
ssm_pool: torch.Tensor,
h_track: torch.Tensor,
track_mode: torch.Tensor,
track_dst: torch.Tensor,
cache_indices: torch.Tensor,
) -> None:
"""Snapshot the tracked rows' SSM state into their track slots (mode 1:
the row's final state, mode 2: its fp32 chunk-boundary state)."""
rows = track_mode.shape[0]
numel = ssm_pool[0].numel()
assert ssm_pool[0].is_contiguous(), "per-slot SSM state must be contiguous"
assert h_track.shape[0] >= rows and h_track[0].is_contiguous()
assert h_track[0].numel() == numel
block = 1024
_kda_track_ssm_kernel[(rows, triton.cdiv(numel, block))](
ssm_pool,
h_track,
track_mode,
track_dst,
cache_indices,
ssm_pool.stride(0),
h_track.stride(0),
numel,
BLOCK=block,
)
@@ -1389,6 +1389,8 @@ def chunk_kda(
track_state: torch.Tensor | None = None,
track_chunk_idx: torch.Tensor | None = None,
beta_is_raw: bool = False,
chunk_indices: torch.Tensor | None = None,
chunk_offsets: torch.Tensor | None = None,
**kwargs: object,
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
"""Match the public forward contract of SGLang's Triton ``chunk_kda``."""
@@ -1432,6 +1434,8 @@ def chunk_kda(
output_intermediate_states=output_intermediate_states,
track_state=track_state,
track_chunk_idx=track_chunk_idx,
chunk_indices=chunk_indices,
chunk_offsets=chunk_offsets,
)
q = q.contiguous()
@@ -1441,11 +1445,8 @@ def chunk_kda(
v = v.contiguous()
g = g.contiguous()
beta = beta.contiguous()
chunk_indices = (
prepare_chunk_indices(cu_seqlens, CHUNK_SIZE)
if cu_seqlens is not None
else None
)
if chunk_indices is None and cu_seqlens is not None:
chunk_indices = prepare_chunk_indices(cu_seqlens, CHUNK_SIZE)
g, qg, wk, kg, chunk_decay = gate_chunk_cumsum_operands(
g,
q,
@@ -1473,7 +1474,8 @@ def chunk_kda(
is_varlen = cu_seqlens is not None
if is_varlen:
chunk_offsets = prepare_chunk_offsets(cu_seqlens, CHUNK_SIZE)
if chunk_offsets is None:
chunk_offsets = prepare_chunk_offsets(cu_seqlens, CHUNK_SIZE)
metadata = cu_seqlens
else:
metadata = torch.empty(0, device=q.device, dtype=torch.int32)
@@ -15,6 +15,11 @@ from sglang.kernels.jit.utils import is_arch_support_pdl
PAD_SLOT_ID = -1
# Tokens per program of _causal_conv1d_fwd_kernel; a block_table enumerates
# (sequence, chunk) pairs in these units.
CAUSAL_CONV1D_FWD_BLOCK_M = 8
@triton.jit()
def _causal_conv1d_fwd_kernel( # continuous batching
# Pointers to matrices
@@ -26,6 +31,7 @@ def _causal_conv1d_fwd_kernel( # continuous batching
has_initial_states_ptr,
query_start_loc_ptr,
o_ptr, # (dim, seqlen) - actually pointing to x_ptr
block_table_ptr, # (num_blocks, 2) int32 (seq, chunk) rows; USE_BLOCK_TABLE only
# Matrix dimensions
dim: tl.constexpr,
seqlen: tl.int32, # cu_seqlen
@@ -55,6 +61,7 @@ def _causal_conv1d_fwd_kernel( # continuous batching
NP2_STATELEN: tl.constexpr,
BLOCK_M: tl.constexpr,
BLOCK_N: tl.constexpr,
USE_BLOCK_TABLE: tl.constexpr = False,
):
conv_states_ptr = initial_states_ptr
conv_state_indices_ptr = cache_indices_ptr
@@ -68,9 +75,17 @@ def _causal_conv1d_fwd_kernel( # continuous batching
# one program handles one chunk in a single sequence
# rather than mixing sequences - to make updating initial_states across sequences efficiently
# single-sequence id
idx_seq = tl.program_id(0)
chunk_offset = tl.program_id(1)
if USE_BLOCK_TABLE:
# CUDA-graph safe launch: the grid is a fixed upper bound on the
# (sequence, BLOCK_M-chunk) pairs of a token bucket and each program
# reads its pair from the table; padded rows carry pad_slot_id.
idx_block = tl.program_id(0)
idx_seq = tl.load(block_table_ptr + idx_block * 2)
chunk_offset = tl.load(block_table_ptr + idx_block * 2 + 1)
else:
# single-sequence id
idx_seq = tl.program_id(0)
chunk_offset = tl.program_id(1)
# BLOCK_N elements along the feature-dimension (channel)
idx_feats = tl.program_id(2) * BLOCK_N + tl.arange(0, BLOCK_N)
@@ -402,6 +417,7 @@ def causal_conv1d_fn(
activation: Optional[str] = "silu",
pad_slot_id: int = PAD_SLOT_ID,
validate_data=False,
block_table: Optional[torch.Tensor] = None,
**kwargs,
):
"""support varlen + continuous batching when x is 2D tensor
@@ -515,6 +531,9 @@ def causal_conv1d_fn(
assert is_channel_last, "Need to run in channel-last layout"
def grid(META):
if block_table is not None:
# (num_blocks, 1, feature blocks): no host-side sequence lengths.
return (block_table.shape[0], 1, triton.cdiv(dim, META["BLOCK_N"]))
max_seq_len = max(seq_lens_cpu)
return (
len(seq_lens_cpu), # batch_size
@@ -532,6 +551,7 @@ def causal_conv1d_fn(
has_initial_state,
query_start_loc,
out,
block_table,
# Matrix dimensions
dim,
cu_seqlen,
@@ -560,8 +580,9 @@ def causal_conv1d_fn(
USE_PAD_SLOT=pad_slot_id is not None,
NP2_STATELEN=np2_statelen,
# launch_cooperative_grid=True
BLOCK_M=8,
BLOCK_M=CAUSAL_CONV1D_FWD_BLOCK_M,
BLOCK_N=256,
USE_BLOCK_TABLE=block_table is not None,
num_stages=2,
)
return out
+7
View File
@@ -1499,6 +1499,13 @@ class Envs:
# Eager forward wraps the ForwardBatch's own tensors instead of copying them
# into the CUDA graph buffer registry (no per-iter device-to-device copy).
SGLANG_EAGER_INPUT_NO_COPY = EnvBool(False)
# Breakable prefill CUDA graphs capture the Triton KDA extend instead of
# breaking the graph at every KDA layer (layers/attention/linear/
# kda_prefill_graph.py); this kill-switch restores the eager break.
SGLANG_DISABLE_KDA_PREFILL_GRAPH_EXTEND = EnvBool(False)
# Sequences a captured KDA extend bucket accepts; a larger prefill batch on
# any dp rank runs the step eagerly. Bounds the padded grids and scratch.
SGLANG_KDA_PREFILL_GRAPH_MAX_SEQS = EnvInt(128)
# ===================================================================
# Tokenizer, request state, embeddings, and reasoning controls
@@ -152,6 +152,9 @@ class AttentionBackend(ABC):
# those tensor addresses. Such backends opt in here, create the metadata
# object during capture, and refresh its dynamic fields before each replay.
use_captured_forward_metadata_for_breakable_cuda_graph: bool = False
# Most requests one replay of that captured metadata can carry (None: no
# bound). Fixed at init: the dp-attention replay vote checks it too.
prefill_cuda_graph_max_batch_size: Optional[int] = None
# Backends may keep MIXED prefill eager under DP attention when replaying
# the EXTEND graph is a known serving-performance regression.
@@ -35,6 +35,11 @@ from sglang.srt.layers.attention.mamba.replay_state_indices_validator import (
)
from sglang.srt.layers.radix_attention import RadixAttention
from sglang.srt.mem_cache.memory_pool import HybridReqToTokenPool
from sglang.srt.model_executor.cuda_graph_config import (
Backend,
Phase,
check_cuda_graph_backend,
)
from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode
from sglang.srt.model_executor.model_runner import ModelRunner
from sglang.srt.runtime_context import get_exec, get_memory, get_spec
@@ -68,6 +73,10 @@ class MambaAttnBackendBase(AttentionBackend):
# else — update_mamba_state_after_mtp_verify keys the fused branch on it.
accept_lens_pool: Optional[torch.Tensor] = None
# Most sequences the prefill graph tables hold (larger batches replay no
# graph); set at init by backends whose supports_prefill_graph_extend holds.
prefill_graph_max_seqs: int
def __init__(self, model_runner: ModelRunner):
self.validate_mis_support(model_runner.server_args)
super().__init__()
@@ -109,6 +118,21 @@ class MambaAttnBackendBase(AttentionBackend):
# skips the per-cycle method dispatch.
self._graph_seq_len_fill_value = self.get_cuda_graph_seq_len_fill_value()
def supports_prefill_graph_extend(self) -> bool:
"""Whether breakable prefill CUDA graphs can capture this backend's
extend on static per-bucket tables instead of breaking at each layer."""
return False
def prefill_graph_extend_active(self) -> bool:
"""Whether the extend being captured or replayed runs on those tables."""
return False
def init_prefill_graph_metadata(self, forward_batch: ForwardBatch):
raise NotImplementedError()
def refresh_prefill_graph_metadata(self, meta, forward_batch: ForwardBatch):
raise NotImplementedError()
@property
def mamba_chunk_size(self) -> int:
if self._mamba_chunk_size is None:
@@ -1178,6 +1202,22 @@ class HybridLinearAttnBackend(AttentionBackend):
self.extend_dummy_seqs_capped_by_req_pool = getattr(
full_attn_backend, "extend_dummy_seqs_capped_by_req_pool", False
) or getattr(linear_attn_backend, "extend_dummy_seqs_capped_by_req_pool", False)
# Breakable prefill CUDA graphs capture the linear extend when the linear
# backend can run it on static tables (linear/kda_prefill_graph.py);
# those tables follow the runner's captured-metadata contract.
self.use_captured_forward_metadata_for_breakable_cuda_graph = (
check_cuda_graph_backend(Phase.PREFILL, Backend.BREAKABLE)
and linear_attn_backend.supports_prefill_graph_extend()
and not full_attn_backend.use_captured_forward_metadata_for_breakable_cuda_graph
)
if self.use_captured_forward_metadata_for_breakable_cuda_graph:
self.prefill_cuda_graph_max_batch_size = (
linear_attn_backend.prefill_graph_max_seqs
)
logger.info(
"Breakable prefill CUDA graphs capture the %s extend.",
type(linear_attn_backend).__name__,
)
@property
def data_type(self):
@@ -1230,6 +1270,49 @@ class HybridLinearAttnBackend(AttentionBackend):
for attn_backend in self.attn_backend_list:
attn_backend.init_forward_metadata_in_graph(forward_batch)
@staticmethod
def _is_plain_extend(forward_batch: ForwardBatch) -> bool:
mode = forward_batch.forward_mode
return (
mode.is_extend()
and not mode.is_target_verify()
and not mode.is_draft_extend_v2()
)
def linear_extend_in_graph(self) -> bool:
"""Whether the linear layers of the prefill being captured run inside
the graph (no eager break)."""
return self.linear_attn_backend.prefill_graph_extend_active()
def init_forward_metadata_for_breakable_cuda_graph_capture(
self, forward_batch: ForwardBatch
):
self.full_attn_backend.init_forward_metadata(forward_batch)
if not self._is_plain_extend(forward_batch):
if not forward_batch.forward_mode.is_draft_extend_v2():
self.linear_attn_backend.init_forward_metadata(forward_batch)
return None
return self.linear_attn_backend.init_prefill_graph_metadata(forward_batch)
def prepare_forward_metadata_for_breakable_cuda_graph_replay(
self,
capture_metadata,
forward_batch: ForwardBatch,
*,
static_forward_batch: Optional[ForwardBatch] = None,
) -> None:
self.full_attn_backend.init_forward_metadata(forward_batch)
if capture_metadata is not None:
self.linear_attn_backend.refresh_prefill_graph_metadata(
capture_metadata, forward_batch
)
elif not forward_batch.forward_mode.is_draft_extend_v2():
self.linear_attn_backend.init_forward_metadata(forward_batch)
if static_forward_batch is not None:
self.prepare_prefill_shared_read_snapshot(
forward_batch, num_qo_tokens=static_forward_batch.positions.shape[0]
)
def get_indexer_metadata(self, layer_id: int, forward_batch: ForwardBatch):
if layer_id in self.full_attn_layers:
return self.full_attn_backend.get_indexer_metadata(layer_id, forward_batch)
@@ -10,6 +10,12 @@ from sglang.kernels.ops.mamba.causal_conv1d_triton import (
)
from sglang.srt.environ import envs
from sglang.srt.layers.attention.hybrid_linear_attn_backend import MambaAttnBackendBase
from sglang.srt.layers.attention.linear.kda_prefill_graph import (
KDAPrefillGraphMetadata,
KDAPrefillTrackRows,
build_track_rows,
kda_extend_in_graph,
)
from sglang.srt.layers.attention.linear.kernels.kda_flashinfer import (
build_fused_accept_indices,
)
@@ -43,6 +49,7 @@ from sglang.srt.runtime_context import (
get_memory,
get_platform,
get_spec,
mamba_cache_chunk_size,
)
@@ -454,8 +461,13 @@ class KDAAttnBackend(MambaAttnBackendBase):
# force-flush path.
needs_cpu_seq_lens: bool = False
# Set while a breakable prefill CUDA graph captures or replays a bucket:
# forward_extend then runs the graph-safe chain on these static tables.
prefill_graph_metadata: Optional[KDAPrefillGraphMetadata] = None
def __init__(self, model_runner: ModelRunner):
super().__init__(model_runner)
self.prefill_graph_max_seqs = envs.SGLANG_KDA_PREFILL_GRAPH_MAX_SEQS.get()
# Needed by the extra_buffer track path: _init_track_conv_indices reads
# conv_states_shape[-1] as the conv window length (kernel_size - 1).
# The KDA pool stores conv states as [kernel-1, dim] — transposed vs
@@ -590,7 +602,110 @@ class KDAAttnBackend(MambaAttnBackendBase):
metadata.fused_accept_num_accepted,
)
def supports_prefill_graph_extend(self) -> bool:
return (
not envs.SGLANG_DISABLE_KDA_PREFILL_GRAPH_EXTEND.get()
# Only the Triton chain guards padded rows (-1 state slots, chunks past T).
and isinstance(self.kernel_dispatcher.extend_kernel, TritonKDAKernel)
# Fused-accept staging (flashinfer verify) writes per extend.
and self.accept_lens_pool is None
)
def prefill_graph_extend_active(self) -> bool:
return self.prefill_graph_metadata is not None
def init_prefill_graph_metadata(
self, forward_batch: ForwardBatch
) -> KDAPrefillGraphMetadata:
cache_indices = self._prefill_graph_cache_indices(forward_batch)
meta = KDAPrefillGraphMetadata.allocate(
num_tokens=forward_batch.positions.shape[0],
max_seqs_cap=self.prefill_graph_max_seqs,
# Any replay of this bucket may carry prefix-cache snapshots (the
# extra buffer is only on with the radix cache).
track=get_exec().mamba.enable_mamba_extra_buffer,
cache_index_dtype=cache_indices.dtype,
device=self.device,
)
self._fill_prefill_graph_metadata(
meta=meta, forward_batch=forward_batch, cache_indices=cache_indices
)
return meta
def refresh_prefill_graph_metadata(
self, meta: KDAPrefillGraphMetadata, forward_batch: ForwardBatch
) -> None:
self._fill_prefill_graph_metadata(
meta=meta,
forward_batch=forward_batch,
cache_indices=self._prefill_graph_cache_indices(forward_batch),
)
def _prefill_graph_cache_indices(self, forward_batch: ForwardBatch):
# Same slot resolution as MambaAttnBackendBase._forward_metadata.
cache_indices = self._translate_mamba_indices(
self.req_to_token_pool.get_mamba_indices(forward_batch.req_pool_indices)
)
real_bs = forward_batch._original_batch_size
if real_bs is not None and real_bs < cache_indices.shape[0]:
cache_indices = cache_indices.clone()
cache_indices[real_bs:] = -1
return cache_indices
def _fill_prefill_graph_metadata(
self,
*,
meta: KDAPrefillGraphMetadata,
forward_batch: ForwardBatch,
cache_indices: torch.Tensor,
) -> None:
if forward_batch.mamba_track_indices is not None:
# Same one-time in-place translation as the eager metadata path.
forward_batch.mamba_track_indices = self._translate_mamba_indices(
forward_batch.mamba_track_indices
)
track_rows = None
if forward_batch.mamba_track_mask is not None:
assert meta.track, (
"prefix-cache track batch replayed on a bucket captured without "
"track tables"
)
track_rows = self._prefill_graph_track_rows(forward_batch)
meta.fill(
extend_seq_lens=list(forward_batch.extend_seq_lens_cpu),
cache_indices=cache_indices,
extend_prefix_lens=forward_batch.extend_prefix_lens,
track_rows=track_rows,
)
self.prefill_graph_metadata = meta
def _prefill_graph_track_rows(
self, forward_batch: ForwardBatch
) -> Optional[KDAPrefillTrackRows]:
bs = forward_batch.batch_size
if self._has_cpu_prefill_track_metadata(forward_batch):
track_mask = list(forward_batch.mamba_prefill_track_mask_cpu)
track_seq_lens = list(forward_batch.mamba_track_seqlens_cpu)
extend_prefix_lens = list(forward_batch.extend_prefix_lens_cpu)
else:
# The same host copies the eager path takes without CPU metadata.
track_mask = forward_batch.mamba_track_mask[:bs].tolist()
track_seq_lens = forward_batch.mamba_track_seqlens[:bs].tolist()
extend_prefix_lens = forward_batch.extend_prefix_lens[:bs].tolist()
return build_track_rows(
track_mask=track_mask,
track_seq_lens=track_seq_lens,
extend_seq_lens=list(forward_batch.extend_seq_lens_cpu),
extend_prefix_lens=extend_prefix_lens,
state_chunk_size=self.mamba_chunk_size,
cache_chunk_size=mamba_cache_chunk_size(),
conv_len=self.conv_states_shape[-1],
dst=forward_batch.mamba_track_indices,
)
def init_forward_metadata(self, forward_batch: ForwardBatch):
# An eager forward never runs on a graph bucket's tables.
self.prefill_graph_metadata = None
super().init_forward_metadata(forward_batch)
if self.forward_metadata.has_mamba_track_mask:
if self.forward_metadata.mamba_track_mask_indices is None:
@@ -908,6 +1023,21 @@ class KDAAttnBackend(MambaAttnBackendBase):
if forward_batch.forward_mode.is_target_verify():
return self._forward_target_verify(layer, forward_batch, mixed_qkv, a, b)
if self.prefill_graph_metadata is not None:
# Breakable prefill CUDA graph bucket: host-free chain on static
# tables refreshed before each replay.
layer_cache = self.req_to_token_pool.mamba2_layer_cache(layer.layer_id)
return kda_extend_in_graph(
layer=layer,
meta=self.prefill_graph_metadata,
conv_pool=layer_cache.conv[0],
ssm_states=layer_cache.temporal,
kernel_dispatcher=self.kernel_dispatcher,
mixed_qkv=mixed_qkv,
a=a,
b=b,
)
query_start_loc = self.forward_metadata.query_start_loc
cache_indices = self.forward_metadata.mamba_cache_indices
@@ -0,0 +1,389 @@
"""KDA extend captured inside breakable prefill CUDA graphs.
A breakable prefill CUDA graph (BCG) is captured once per token bucket and
replayed for any batch that pads into that bucket. The eager KDA extend derives
its launch grids and index tables from host-side sequence lengths
(``prepare_chunk_indices``, ``causal_conv1d_fn``'s ``seq_lens_cpu``, the
prefix-cache track plan), so without this module BCG breaks the graph at every
KDA layer and runs the whole chain -- about sixteen kernels per layer plus their
Python -- eagerly.
For each bucket this module allocates static tables sized by the bucket's
bounds:
* ``max_seqs`` sequences (``min(bucket, cap)``; batches with more sequences run
eagerly, through the runner's replay vote on
``AttentionBackend.prefill_cuda_graph_max_batch_size``),
* ``max_chunks = cdiv(bucket, 64) + max_seqs`` delta-rule chunks,
* ``max_conv_blocks = cdiv(bucket, 8) + max_seqs`` conv blocks.
Before every replay ``fill`` rebuilds the tables from the batch's CPU lengths
(one host-to-device copy plus a few device gathers, no synchronization). The
captured kernels launch fixed grids over the bounds; padded rows point past
their sequence (chunk tables) or carry ``PAD_SLOT_ID`` (conv blocks, state
slots) and return immediately.
"""
from __future__ import annotations
from typing import TYPE_CHECKING, Optional
import msgspec
import numpy as np
import torch
import triton
from sglang.kernels.ops.attention.fla.kda_prefill_graph_track import (
TRACK_CHUNK_STATE,
TRACK_FINAL_STATE,
kda_track_conv_window,
kda_track_ssm_state,
)
from sglang.kernels.ops.mamba.causal_conv1d_triton import (
CAUSAL_CONV1D_FWD_BLOCK_M,
PAD_SLOT_ID,
causal_conv1d_fn,
)
from sglang.srt.layers.attention.mamba.prefill_track_metadata import (
build_prefill_track_plan,
)
if TYPE_CHECKING:
from sglang.srt.layers.attention.linear.kda_backend import KDAKernelDispatcher
from sglang.srt.layers.radix_linear_attention import RadixLinearAttention
# Delta-rule chunk size of chunk_kda (fla/kda.py, fla/chunk_delta_h.py).
KDA_CHUNK_SIZE = 64
# Chunk index of padded chunk_indices rows: past the end of any sequence, so
# every varlen chunk kernel returns on its `i_t * BT >= T` guard.
_PAD_CHUNK = 1 << 20
class KDAPrefillTrackRows(msgspec.Struct, frozen=True):
"""One batch's prefix-cache track plan, in batch rows."""
# (row, chunk): the snapshot is the state at that chunk boundary.
chunk_rows: list[tuple[int, int]]
# Rows whose snapshot is the post-extend state (chunk-aligned length).
final_rows: list[int]
# (row, aligned_len) of every tracked row: its conv window ends there.
conv_rows: list[tuple[int, int]]
# Conv window length (kernel_size - 1).
conv_len: int
# [bs] physical track slots, on the device.
dst: torch.Tensor
class KDAPrefillGraphMetadata(msgspec.Struct, frozen=True):
"""Static per-bucket tables read by the captured KDA extend."""
num_tokens: int
max_seqs: int
max_chunks: int
max_conv_blocks: int
# Allocates and records the prefix-cache snapshot copies.
track: bool
# One int32 device buffer, refreshed by a single copy, and its views.
packed: torch.Tensor
cu_seqlens: torch.Tensor # [max_seqs + 1]
chunk_indices: torch.Tensor # [max_chunks, 2] (seq, chunk)
chunk_offsets: torch.Tensor # [max_seqs + 1]
conv_block_table: torch.Tensor # [max_conv_blocks, 2] (seq, block)
track_chunk_idx: torch.Tensor # [max_seqs]
track_mode: torch.Tensor # [max_seqs]
track_conv_start: torch.Tensor # [max_seqs]
num_logical_tokens: torch.Tensor # [1]
host: np.ndarray # int32 staging for `packed`
# Filled on the device from the batch.
cache_indices: torch.Tensor # [max_seqs], PAD_SLOT_ID past the batch
has_initial_state: torch.Tensor # [max_seqs] bool
track_dst: torch.Tensor # [max_seqs] int64
token_index: torch.Tensor # [num_tokens] int32 arange
@classmethod
def allocate(
cls,
*,
num_tokens: int,
max_seqs_cap: int,
track: bool,
cache_index_dtype: torch.dtype,
device: torch.device,
) -> KDAPrefillGraphMetadata:
max_seqs = max(1, min(num_tokens, max_seqs_cap))
max_chunks = triton.cdiv(num_tokens, KDA_CHUNK_SIZE) + max_seqs
max_conv_blocks = triton.cdiv(num_tokens, CAUSAL_CONV1D_FWD_BLOCK_M) + max_seqs
sizes = [
max_seqs + 1, # cu_seqlens
2 * max_chunks, # chunk_indices
max_seqs + 1, # chunk_offsets
2 * max_conv_blocks, # conv_block_table
max_seqs, # track_chunk_idx
max_seqs, # track_mode
max_seqs, # track_conv_start
1, # num_logical_tokens
]
packed = torch.zeros(sum(sizes), dtype=torch.int32, device=device)
views = torch.split(packed, sizes)
return cls(
num_tokens=num_tokens,
max_seqs=max_seqs,
max_chunks=max_chunks,
max_conv_blocks=max_conv_blocks,
track=track,
packed=packed,
cu_seqlens=views[0],
chunk_indices=views[1].view(max_chunks, 2),
chunk_offsets=views[2],
conv_block_table=views[3].view(max_conv_blocks, 2),
track_chunk_idx=views[4],
track_mode=views[5],
track_conv_start=views[6],
num_logical_tokens=views[7],
host=np.zeros(sum(sizes), dtype=np.int32),
cache_indices=torch.full(
(max_seqs,), PAD_SLOT_ID, dtype=cache_index_dtype, device=device
),
has_initial_state=torch.zeros(max_seqs, dtype=torch.bool, device=device),
track_dst=torch.zeros(max_seqs, dtype=torch.int64, device=device),
token_index=torch.arange(num_tokens, dtype=torch.int32, device=device),
)
def can_fit(self, extend_seq_lens: list[int]) -> bool:
return (
len(extend_seq_lens) <= self.max_seqs
and sum(extend_seq_lens) <= self.num_tokens
)
def fill(
self,
*,
extend_seq_lens: list[int],
cache_indices: torch.Tensor,
extend_prefix_lens: torch.Tensor,
track_rows: Optional[KDAPrefillTrackRows],
) -> None:
"""Rebuild the tables for this batch. ``cache_indices`` and
``extend_prefix_lens`` are the batch's device tensors."""
bs = len(extend_seq_lens)
assert self.can_fit(extend_seq_lens), (
f"batch of {bs} sequences / {sum(extend_seq_lens)} tokens exceeds the "
f"captured bounds ({self.max_seqs} / {self.num_tokens})"
)
self._write_host_tables(
lens=np.asarray(extend_seq_lens, dtype=np.int64), track_rows=track_rows
)
# A pinned block from the caching host allocator keeps the copy
# asynchronous and is not reused before the copy completes.
self.packed.copy_(torch.from_numpy(self.host).pin_memory(), non_blocking=True)
self.cache_indices[:bs].copy_(cache_indices[:bs])
self.cache_indices[bs:].fill_(PAD_SLOT_ID)
torch.gt(extend_prefix_lens[:bs], 0, out=self.has_initial_state[:bs])
self.has_initial_state[bs:].fill_(False)
if track_rows is not None:
self.track_dst[:bs].copy_(track_rows.dst[:bs])
def _write_host_tables(
self, *, lens: np.ndarray, track_rows: Optional[KDAPrefillTrackRows]
) -> None:
"""Write every host-built table into ``self.host`` in ``packed`` order."""
bs = lens.shape[0]
total = int(lens.sum())
host = self.host
# Padded sequences are empty and sit at the end.
cu = np.empty(self.max_seqs + 1, dtype=np.int64)
cu[0] = 0
np.cumsum(lens, out=cu[1 : bs + 1])
cu[bs + 1 :] = total
o = self._write(o=0, values=cu)
# Sequence-major (seq, chunk) pairs: chunk_gla_fwd_o reads the per-chunk
# states by row, chunk_gated_delta_rule_fwd_h writes them at
# chunk_offsets[seq] + chunk.
n_chunks = (lens + KDA_CHUNK_SIZE - 1) // KDA_CHUNK_SIZE
o = self._write_pairs(
o=o, counts=n_chunks, rows=self.max_chunks, pad=(0, _PAD_CHUNK)
)
offsets = np.empty(self.max_seqs + 1, dtype=np.int64)
offsets[0] = 0
np.cumsum(n_chunks, out=offsets[1 : bs + 1])
offsets[bs + 1 :] = offsets[bs]
o = self._write(o=o, values=offsets)
n_blocks = (lens + CAUSAL_CONV1D_FWD_BLOCK_M - 1) // CAUSAL_CONV1D_FWD_BLOCK_M
o = self._write_pairs(
o=o, counts=n_blocks, rows=self.max_conv_blocks, pad=(PAD_SLOT_ID, 0)
)
track_chunk_idx = np.full(self.max_seqs, -1, dtype=np.int64)
track_mode = np.zeros(self.max_seqs, dtype=np.int64)
track_conv_start = np.zeros(self.max_seqs, dtype=np.int64)
if track_rows is not None:
for row, chunk in track_rows.chunk_rows:
track_chunk_idx[row] = chunk
track_mode[row] = TRACK_CHUNK_STATE
for row in track_rows.final_rows:
track_mode[row] = TRACK_FINAL_STATE
for row, aligned_len in track_rows.conv_rows:
track_conv_start[row] = cu[row] + aligned_len - track_rows.conv_len
o = self._write(o=o, values=track_chunk_idx)
o = self._write(o=o, values=track_mode)
o = self._write(o=o, values=track_conv_start)
o = self._write(o=o, values=np.array([total]))
assert o == host.shape[0]
def _write(self, *, o: int, values: np.ndarray) -> int:
self.host[o : o + values.shape[0]] = values
return o + values.shape[0]
def _write_pairs(
self, *, o: int, counts: np.ndarray, rows: int, pad: tuple[int, int]
) -> int:
"""Write (seq, index) rows, `counts[seq]` indices per sequence, into a
[rows, 2] table at `o`, padding the tail with `pad`."""
n = int(counts.sum())
table = self.host[o : o + 2 * rows].reshape(rows, 2)
if n:
table[:n, 0] = np.repeat(np.arange(counts.shape[0]), counts)
table[:n, 1] = np.arange(n) - np.repeat(np.cumsum(counts) - counts, counts)
table[n:, 0] = pad[0]
table[n:, 1] = pad[1]
return o + 2 * rows
def build_track_rows(
*,
track_mask: list[bool],
track_seq_lens: list[int],
extend_seq_lens: list[int],
extend_prefix_lens: list[int],
state_chunk_size: int,
cache_chunk_size: int,
conv_len: int,
dst: torch.Tensor,
) -> Optional[KDAPrefillTrackRows]:
"""The eager track selection (_init_track_ssm_indices_from_cpu and
_init_track_conv_indices) as per-row lists; None when nothing is tracked."""
if not any(track_mask):
return None
plan = build_prefill_track_plan(
track_mask,
track_seq_lens,
extend_seq_lens,
extend_prefix_lens,
state_chunk_size,
mamba2=False,
)
conv_rows = [
(
row,
(track_seq_lens[row] - extend_prefix_lens[row])
// cache_chunk_size
* cache_chunk_size,
)
for row in plan.tracked_rows
]
return KDAPrefillTrackRows(
chunk_rows=[(row, plan.chunk_indices[row]) for row in plan.unaligned_rows],
final_rows=list(plan.final_rows),
conv_rows=conv_rows,
conv_len=conv_len,
dst=dst,
)
def kda_extend_in_graph(
*,
layer: RadixLinearAttention,
meta: KDAPrefillGraphMetadata,
conv_pool: torch.Tensor,
ssm_states: torch.Tensor,
kernel_dispatcher: KDAKernelDispatcher,
mixed_qkv: torch.Tensor,
a: torch.Tensor,
b: torch.Tensor,
) -> torch.Tensor:
"""KDAAttnBackend.forward_extend with launch grids that depend only on the
bucket. ``conv_pool`` is the layer's [slots, kernel-1, dim] conv cache.
Returns [1, num_tokens, H, V]; rows past the batch's tokens are zero."""
if meta.track:
# Snapshot the conv window before the conv kernel updates the cache;
# the source is the raw input, the destinations are track slots.
kda_track_conv_window(
conv_pool=conv_pool,
x=mixed_qkv,
track_mode=meta.track_mode,
track_conv_start=meta.track_conv_start,
track_dst=meta.track_dst,
num_tokens=meta.num_logical_tokens,
)
qkv = causal_conv1d_fn(
mixed_qkv.transpose(0, 1),
layer.conv_weights,
layer.bias,
activation="silu",
conv_states=conv_pool.transpose(-1, -2),
has_initial_state=meta.has_initial_state,
cache_indices=meta.cache_indices,
query_start_loc=meta.cu_seqlens,
seq_lens_cpu=None,
block_table=meta.conv_block_table,
).transpose(0, 1)
q, k, v = qkv.split([layer.q_dim, layer.k_dim, layer.v_dim], dim=-1)
q = q.unflatten(-1, (-1, layer.head_q_dim)).unsqueeze(0)
k = k.unflatten(-1, (-1, layer.head_k_dim)).unsqueeze(0)
v = v.unflatten(-1, (-1, layer.head_v_dim)).unsqueeze(0)
gate_was_flat = a.ndim == 3
if gate_was_flat:
a = a.unflatten(-1, (-1, layer.head_k_dim))
h_track = None
if meta.track:
h_track = torch.empty(
(meta.max_seqs, *ssm_states.shape[1:]),
dtype=torch.float32,
device=ssm_states.device,
)
core_attn_out = kernel_dispatcher.extend(
q=q,
k=k,
v=v,
g=a,
beta=b,
ssm_states=ssm_states,
cache_indices=meta.cache_indices,
query_start_loc=meta.cu_seqlens,
A_log=layer.A_log,
dt_bias=layer.dt_bias,
lower_bound=layer.lower_bound,
beta_is_raw=gate_was_flat,
extend_seq_lens_cpu=None,
is_spec_decode=False,
return_intermediate_states=False,
track_state=h_track,
track_chunk_idx=meta.track_chunk_idx if meta.track else None,
chunk_indices=meta.chunk_indices,
chunk_offsets=meta.chunk_offsets,
)
if meta.track:
kda_track_ssm_state(
ssm_pool=ssm_states,
h_track=h_track,
track_mode=meta.track_mode,
track_dst=meta.track_dst,
cache_indices=meta.cache_indices,
)
# Rows past the batch belong to no sequence and were never written; keep the
# layer's padded output finite for the MoE and the collectives.
core_attn_out.masked_fill_(
(meta.token_index >= meta.num_logical_tokens).view(1, -1, 1, 1), 0
)
return core_attn_out
@@ -251,4 +251,6 @@ class TritonKDAKernel(LinearAttnKernelBase):
output_intermediate_states=return_intermediate_states,
track_state=kwargs.get("track_state"),
track_chunk_idx=kwargs.get("track_chunk_idx"),
chunk_indices=kwargs.get("chunk_indices"),
chunk_offsets=kwargs.get("chunk_offsets"),
)
@@ -93,7 +93,7 @@ class RadixLinearAttention(nn.Module):
dtype=mixed_qkv.dtype,
device=mixed_qkv.device,
)
if is_in_breakable_cuda_graph():
if is_in_breakable_cuda_graph() and not _linear_extend_in_graph():
bcg_unified_linear_attention_with_output(
mixed_qkv,
a,
@@ -148,6 +148,20 @@ class RadixLinearAttention(nn.Module):
)
def _linear_extend_in_graph() -> bool:
"""Whether the backend runs this breakable prefill capture's linear extend
on static tables inside the graph (no eager break at the layer)."""
from sglang.srt.layers.attention.hybrid_linear_attn_backend import (
HybridLinearAttnBackend,
)
backend = get_attn_backend()
return (
isinstance(backend, HybridLinearAttnBackend)
and backend.linear_extend_in_graph()
)
def _linear_attention_with_output_impl(
mixed_qkv: torch.Tensor,
a: torch.Tensor,
@@ -289,6 +289,7 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
_use_draft_input_embeds = False
_backend_can_run_prefill_cuda_graph = None
_captured_attn_metadata_max_bs: Optional[int] = None
dllm_attention = None
def __init__(self, model_runner: ModelRunner):
@@ -612,6 +613,11 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
self.use_captured_attn_metadata = model_runner.attn_backend.use_captured_forward_metadata_for_breakable_cuda_graph
else:
self.use_captured_attn_metadata = False
self._captured_attn_metadata_max_bs = (
model_runner.attn_backend.prefill_cuda_graph_max_batch_size
if self.use_captured_attn_metadata
else None
)
self.attn_metadata_buffers: Optional[Dict[ShapeKey, object]] = (
{} if self.use_captured_attn_metadata else None
)
@@ -1274,6 +1280,11 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
return False
if self._is_full_backend and batch_size > self._capture_req_slots:
return False
if (
self._captured_attn_metadata_max_bs is not None
and batch_size > self._captured_attn_metadata_max_bs
):
return False
# LoRA replays need prepare_lora_batch's static metadata. lora_manager
# keeps LoRA prefill eager on every rank under dp attention, so the
# schedule-time vote derives this from enable_lora alone.