[GLM-5.3-Flash] Capture the KDA extend in breakable prefill CUDA graphs (#41105)
Co-authored-by: luoyuan.luo <luoyuan.luo@antgroup.com>
This commit is contained in:
@@ -363,6 +363,7 @@ def chunk_gated_delta_rule_fwd_h(
|
||||
inplace_update: bool = True,
|
||||
track_state: Optional[torch.Tensor] = None,
|
||||
track_chunk_idx: Optional[torch.Tensor] = None,
|
||||
chunk_offsets: Optional[torch.Tensor] = None,
|
||||
) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
assert not (use_exp2 and g is not None), (
|
||||
"use_exp2 covers only the per-channel gk path; scalar g stays natural-exp"
|
||||
@@ -386,11 +387,11 @@ def chunk_gated_delta_rule_fwd_h(
|
||||
if cu_seqlens is None:
|
||||
N, NT, chunk_offsets = B, triton.cdiv(T, BT), None
|
||||
else:
|
||||
N, NT, chunk_offsets = (
|
||||
len(cu_seqlens) - 1,
|
||||
len(chunk_indices),
|
||||
prepare_chunk_offsets(cu_seqlens, BT),
|
||||
)
|
||||
# A caller-built chunk_offsets (CUDA-graph capture) must not go through
|
||||
# prepare_chunk_offsets: its tensor_cache would replay capture-time values.
|
||||
N, NT = len(cu_seqlens) - 1, len(chunk_indices)
|
||||
if chunk_offsets is None:
|
||||
chunk_offsets = prepare_chunk_offsets(cu_seqlens, BT)
|
||||
assert K <= 256, "current kernel does not support head dimension larger than 256."
|
||||
|
||||
h = k.new_empty(B, NT, H, V, K)
|
||||
|
||||
@@ -554,6 +554,12 @@ def _recompute_w_u_fwd_kernel(
|
||||
T = eos - bos
|
||||
else:
|
||||
bos, eos = i_b * T, i_b * T + T
|
||||
|
||||
# Padded chunk_indices rows (kda_prefill_graph.py) start past T, where every
|
||||
# store below is masked; this early exit only skips their wasted work.
|
||||
if i_t * BT >= T:
|
||||
return
|
||||
|
||||
p_b = tl.make_block_ptr(beta + bos * H + i_h, (T,), (H,), (i_t * BT,), (BT,), (0,))
|
||||
b_b = tl.load(p_b, boundary_check=(0,))
|
||||
|
||||
@@ -797,6 +803,11 @@ def chunk_gla_fwd_kernel_o(
|
||||
i_tg = i_b * NT + i_t
|
||||
bos, eos = i_b * T, i_b * T + T
|
||||
|
||||
# Padded chunk_indices rows (kda_prefill_graph.py) start past T, where every
|
||||
# store below is masked; this early exit only skips their wasted work.
|
||||
if i_t * BT >= T:
|
||||
return
|
||||
|
||||
m_s = tl.arange(0, BT)[:, None] >= tl.arange(0, BT)[None, :]
|
||||
|
||||
b_o = tl.zeros([BT, BV], dtype=tl.float32)
|
||||
@@ -973,6 +984,11 @@ def kda_gate_chunk_cumsum_vector_kernel(
|
||||
else:
|
||||
bos, eos = i_b * T, i_b * T + T
|
||||
|
||||
# Padded chunk_indices rows (kda_prefill_graph.py) start past T, where every
|
||||
# store below is masked; this early exit only skips their wasted work.
|
||||
if i_t * BT >= T:
|
||||
return
|
||||
|
||||
p_s = tl.make_block_ptr(
|
||||
s + (bos * H + i_h) * S,
|
||||
(T, S),
|
||||
@@ -1145,15 +1161,16 @@ def chunk_kda_fwd(
|
||||
track_state: Optional[torch.Tensor] = None,
|
||||
track_chunk_idx: Optional[torch.Tensor] = None,
|
||||
beta_is_raw: bool = False,
|
||||
chunk_indices: Optional[torch.Tensor] = None,
|
||||
chunk_offsets: Optional[torch.Tensor] = None,
|
||||
):
|
||||
chunk_size = 64
|
||||
# Pre-compute chunk indices once and thread through all downstream kernels.
|
||||
# Without this, each of the 4 callees would recompute independently.
|
||||
chunk_indices = (
|
||||
prepare_chunk_indices(cu_seqlens, chunk_size)
|
||||
if cu_seqlens is not None
|
||||
else None
|
||||
)
|
||||
# Without this, each of the 4 callees would recompute independently. A
|
||||
# CUDA-graph caller passes padded chunk_indices / chunk_offsets it rebuilds
|
||||
# before every replay (kda_prefill_graph.py).
|
||||
if chunk_indices is None and cu_seqlens is not None:
|
||||
chunk_indices = prepare_chunk_indices(cu_seqlens, chunk_size)
|
||||
|
||||
if A_log is not None:
|
||||
# Fused: gate activation + chunk-local cumsum in one kernel.
|
||||
@@ -1227,6 +1244,7 @@ def chunk_kda_fwd(
|
||||
use_exp2=True,
|
||||
track_state=track_state,
|
||||
track_chunk_idx=track_chunk_idx,
|
||||
chunk_offsets=chunk_offsets,
|
||||
)
|
||||
del w, u, kg
|
||||
|
||||
@@ -1271,6 +1289,8 @@ def chunk_kda(
|
||||
track_state: Optional[torch.Tensor] = None,
|
||||
track_chunk_idx: Optional[torch.Tensor] = None,
|
||||
beta_is_raw: bool = False,
|
||||
chunk_indices: Optional[torch.Tensor] = None,
|
||||
chunk_offsets: Optional[torch.Tensor] = None,
|
||||
**kwargs,
|
||||
):
|
||||
if scale is None:
|
||||
@@ -1298,4 +1318,6 @@ def chunk_kda(
|
||||
track_state=track_state,
|
||||
track_chunk_idx=track_chunk_idx,
|
||||
beta_is_raw=beta_is_raw,
|
||||
chunk_indices=chunk_indices,
|
||||
chunk_offsets=chunk_offsets,
|
||||
)
|
||||
|
||||
@@ -0,0 +1,148 @@
|
||||
"""Prefix-cache track copies for the CUDA-graph captured KDA extend.
|
||||
|
||||
The eager extend selects tracked rows on the host (``nonzero`` / masked index
|
||||
lists) and scatters with data-dependent shapes. Under a breakable prefill CUDA
|
||||
graph the batch changes on every replay, so these kernels walk a fixed number of
|
||||
rows (the bucket's sequence bound) and read a per-row mode instead:
|
||||
|
||||
* ``track_mode[r] == 0``: row ``r`` is not tracked (or is padding); no-op.
|
||||
* ``track_mode[r] == 1``: the tracked length is chunk aligned; the snapshot is
|
||||
the post-extend state of the row's own cache slot.
|
||||
* ``track_mode[r] == 2``: the snapshot is the chunk-boundary state the chunked
|
||||
delta-rule kernel wrote to ``h_track[r]`` (fp32).
|
||||
|
||||
Every tracked row (mode 1 or 2) also snapshots its conv window: the
|
||||
``STATE_LEN`` raw pre-conv input rows ending at the tracked boundary.
|
||||
"""
|
||||
|
||||
import torch
|
||||
import triton
|
||||
import triton.language as tl
|
||||
|
||||
TRACK_NONE = 0
|
||||
TRACK_FINAL_STATE = 1
|
||||
TRACK_CHUNK_STATE = 2
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _kda_track_conv_window_kernel(
|
||||
conv_pool, # [slots, STATE_LEN, dim]
|
||||
x, # [tokens, dim] raw pre-conv input
|
||||
track_mode, # [rows] int32
|
||||
track_conv_start, # [rows] int32, first token row of the window (unclamped)
|
||||
track_dst, # [rows] int64 physical track slot
|
||||
num_tokens, # [1] int32 logical tokens of the batch
|
||||
stride_pool_slot,
|
||||
stride_pool_row,
|
||||
stride_x_token,
|
||||
dim,
|
||||
STATE_LEN: tl.constexpr,
|
||||
BLOCK_D: tl.constexpr,
|
||||
):
|
||||
r = tl.program_id(0)
|
||||
mode = tl.load(track_mode + r)
|
||||
if mode == 0:
|
||||
return
|
||||
dst = tl.load(track_dst + r).to(tl.int64)
|
||||
start = tl.load(track_conv_start + r)
|
||||
total = tl.load(num_tokens)
|
||||
offs = tl.program_id(1) * BLOCK_D + tl.arange(0, BLOCK_D)
|
||||
mask = offs < dim
|
||||
for j in tl.static_range(STATE_LEN):
|
||||
# Same clamp as the eager index list: clamp(0, total - 1).
|
||||
t = tl.minimum(tl.maximum(start + j, 0), total - 1).to(tl.int64)
|
||||
val = tl.load(x + t * stride_x_token + offs, mask=mask)
|
||||
tl.store(
|
||||
conv_pool + dst * stride_pool_slot + j * stride_pool_row + offs,
|
||||
val.to(conv_pool.dtype.element_ty),
|
||||
mask=mask,
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _kda_track_ssm_kernel(
|
||||
ssm_pool, # [slots, ...] per-slot contiguous
|
||||
h_track, # [rows, ...] fp32 chunk-boundary snapshots
|
||||
track_mode, # [rows] int32
|
||||
track_dst, # [rows] int64 physical track slot
|
||||
cache_indices, # [rows] physical cache slot of the row
|
||||
stride_ssm_slot,
|
||||
stride_h_row,
|
||||
numel,
|
||||
BLOCK: tl.constexpr,
|
||||
):
|
||||
r = tl.program_id(0)
|
||||
mode = tl.load(track_mode + r)
|
||||
if mode == 0:
|
||||
return
|
||||
dst = tl.load(track_dst + r).to(tl.int64)
|
||||
offs = tl.program_id(1) * BLOCK + tl.arange(0, BLOCK)
|
||||
mask = offs < numel
|
||||
if mode == 1:
|
||||
src = tl.load(cache_indices + r).to(tl.int64)
|
||||
val = tl.load(ssm_pool + src * stride_ssm_slot + offs, mask=mask)
|
||||
else:
|
||||
val = tl.load(h_track + r.to(tl.int64) * stride_h_row + offs, mask=mask)
|
||||
tl.store(
|
||||
ssm_pool + dst * stride_ssm_slot + offs,
|
||||
val.to(ssm_pool.dtype.element_ty),
|
||||
mask=mask,
|
||||
)
|
||||
|
||||
|
||||
def kda_track_conv_window(
|
||||
conv_pool: torch.Tensor,
|
||||
x: torch.Tensor,
|
||||
track_mode: torch.Tensor,
|
||||
track_conv_start: torch.Tensor,
|
||||
track_dst: torch.Tensor,
|
||||
num_tokens: torch.Tensor,
|
||||
) -> None:
|
||||
"""``conv_pool[track_dst[r]] = x[clamp(start[r] + arange(STATE_LEN))]`` for
|
||||
every tracked row ``r``; the grid depends only on the row bound."""
|
||||
rows = track_mode.shape[0]
|
||||
state_len, dim = conv_pool.shape[-2], conv_pool.shape[-1]
|
||||
assert x.shape[-1] == dim and x.stride(-1) == 1 and conv_pool.stride(-1) == 1
|
||||
block_d = 1024
|
||||
_kda_track_conv_window_kernel[(rows, triton.cdiv(dim, block_d))](
|
||||
conv_pool,
|
||||
x,
|
||||
track_mode,
|
||||
track_conv_start,
|
||||
track_dst,
|
||||
num_tokens,
|
||||
conv_pool.stride(0),
|
||||
conv_pool.stride(1),
|
||||
x.stride(0),
|
||||
dim,
|
||||
STATE_LEN=state_len,
|
||||
BLOCK_D=block_d,
|
||||
)
|
||||
|
||||
|
||||
def kda_track_ssm_state(
|
||||
ssm_pool: torch.Tensor,
|
||||
h_track: torch.Tensor,
|
||||
track_mode: torch.Tensor,
|
||||
track_dst: torch.Tensor,
|
||||
cache_indices: torch.Tensor,
|
||||
) -> None:
|
||||
"""Snapshot the tracked rows' SSM state into their track slots (mode 1:
|
||||
the row's final state, mode 2: its fp32 chunk-boundary state)."""
|
||||
rows = track_mode.shape[0]
|
||||
numel = ssm_pool[0].numel()
|
||||
assert ssm_pool[0].is_contiguous(), "per-slot SSM state must be contiguous"
|
||||
assert h_track.shape[0] >= rows and h_track[0].is_contiguous()
|
||||
assert h_track[0].numel() == numel
|
||||
block = 1024
|
||||
_kda_track_ssm_kernel[(rows, triton.cdiv(numel, block))](
|
||||
ssm_pool,
|
||||
h_track,
|
||||
track_mode,
|
||||
track_dst,
|
||||
cache_indices,
|
||||
ssm_pool.stride(0),
|
||||
h_track.stride(0),
|
||||
numel,
|
||||
BLOCK=block,
|
||||
)
|
||||
@@ -1389,6 +1389,8 @@ def chunk_kda(
|
||||
track_state: torch.Tensor | None = None,
|
||||
track_chunk_idx: torch.Tensor | None = None,
|
||||
beta_is_raw: bool = False,
|
||||
chunk_indices: torch.Tensor | None = None,
|
||||
chunk_offsets: torch.Tensor | None = None,
|
||||
**kwargs: object,
|
||||
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Match the public forward contract of SGLang's Triton ``chunk_kda``."""
|
||||
@@ -1432,6 +1434,8 @@ def chunk_kda(
|
||||
output_intermediate_states=output_intermediate_states,
|
||||
track_state=track_state,
|
||||
track_chunk_idx=track_chunk_idx,
|
||||
chunk_indices=chunk_indices,
|
||||
chunk_offsets=chunk_offsets,
|
||||
)
|
||||
|
||||
q = q.contiguous()
|
||||
@@ -1441,11 +1445,8 @@ def chunk_kda(
|
||||
v = v.contiguous()
|
||||
g = g.contiguous()
|
||||
beta = beta.contiguous()
|
||||
chunk_indices = (
|
||||
prepare_chunk_indices(cu_seqlens, CHUNK_SIZE)
|
||||
if cu_seqlens is not None
|
||||
else None
|
||||
)
|
||||
if chunk_indices is None and cu_seqlens is not None:
|
||||
chunk_indices = prepare_chunk_indices(cu_seqlens, CHUNK_SIZE)
|
||||
g, qg, wk, kg, chunk_decay = gate_chunk_cumsum_operands(
|
||||
g,
|
||||
q,
|
||||
@@ -1473,7 +1474,8 @@ def chunk_kda(
|
||||
|
||||
is_varlen = cu_seqlens is not None
|
||||
if is_varlen:
|
||||
chunk_offsets = prepare_chunk_offsets(cu_seqlens, CHUNK_SIZE)
|
||||
if chunk_offsets is None:
|
||||
chunk_offsets = prepare_chunk_offsets(cu_seqlens, CHUNK_SIZE)
|
||||
metadata = cu_seqlens
|
||||
else:
|
||||
metadata = torch.empty(0, device=q.device, dtype=torch.int32)
|
||||
|
||||
@@ -15,6 +15,11 @@ from sglang.kernels.jit.utils import is_arch_support_pdl
|
||||
PAD_SLOT_ID = -1
|
||||
|
||||
|
||||
# Tokens per program of _causal_conv1d_fwd_kernel; a block_table enumerates
|
||||
# (sequence, chunk) pairs in these units.
|
||||
CAUSAL_CONV1D_FWD_BLOCK_M = 8
|
||||
|
||||
|
||||
@triton.jit()
|
||||
def _causal_conv1d_fwd_kernel( # continuous batching
|
||||
# Pointers to matrices
|
||||
@@ -26,6 +31,7 @@ def _causal_conv1d_fwd_kernel( # continuous batching
|
||||
has_initial_states_ptr,
|
||||
query_start_loc_ptr,
|
||||
o_ptr, # (dim, seqlen) - actually pointing to x_ptr
|
||||
block_table_ptr, # (num_blocks, 2) int32 (seq, chunk) rows; USE_BLOCK_TABLE only
|
||||
# Matrix dimensions
|
||||
dim: tl.constexpr,
|
||||
seqlen: tl.int32, # cu_seqlen
|
||||
@@ -55,6 +61,7 @@ def _causal_conv1d_fwd_kernel( # continuous batching
|
||||
NP2_STATELEN: tl.constexpr,
|
||||
BLOCK_M: tl.constexpr,
|
||||
BLOCK_N: tl.constexpr,
|
||||
USE_BLOCK_TABLE: tl.constexpr = False,
|
||||
):
|
||||
conv_states_ptr = initial_states_ptr
|
||||
conv_state_indices_ptr = cache_indices_ptr
|
||||
@@ -68,9 +75,17 @@ def _causal_conv1d_fwd_kernel( # continuous batching
|
||||
# one program handles one chunk in a single sequence
|
||||
# rather than mixing sequences - to make updating initial_states across sequences efficiently
|
||||
|
||||
# single-sequence id
|
||||
idx_seq = tl.program_id(0)
|
||||
chunk_offset = tl.program_id(1)
|
||||
if USE_BLOCK_TABLE:
|
||||
# CUDA-graph safe launch: the grid is a fixed upper bound on the
|
||||
# (sequence, BLOCK_M-chunk) pairs of a token bucket and each program
|
||||
# reads its pair from the table; padded rows carry pad_slot_id.
|
||||
idx_block = tl.program_id(0)
|
||||
idx_seq = tl.load(block_table_ptr + idx_block * 2)
|
||||
chunk_offset = tl.load(block_table_ptr + idx_block * 2 + 1)
|
||||
else:
|
||||
# single-sequence id
|
||||
idx_seq = tl.program_id(0)
|
||||
chunk_offset = tl.program_id(1)
|
||||
|
||||
# BLOCK_N elements along the feature-dimension (channel)
|
||||
idx_feats = tl.program_id(2) * BLOCK_N + tl.arange(0, BLOCK_N)
|
||||
@@ -402,6 +417,7 @@ def causal_conv1d_fn(
|
||||
activation: Optional[str] = "silu",
|
||||
pad_slot_id: int = PAD_SLOT_ID,
|
||||
validate_data=False,
|
||||
block_table: Optional[torch.Tensor] = None,
|
||||
**kwargs,
|
||||
):
|
||||
"""support varlen + continuous batching when x is 2D tensor
|
||||
@@ -515,6 +531,9 @@ def causal_conv1d_fn(
|
||||
assert is_channel_last, "Need to run in channel-last layout"
|
||||
|
||||
def grid(META):
|
||||
if block_table is not None:
|
||||
# (num_blocks, 1, feature blocks): no host-side sequence lengths.
|
||||
return (block_table.shape[0], 1, triton.cdiv(dim, META["BLOCK_N"]))
|
||||
max_seq_len = max(seq_lens_cpu)
|
||||
return (
|
||||
len(seq_lens_cpu), # batch_size
|
||||
@@ -532,6 +551,7 @@ def causal_conv1d_fn(
|
||||
has_initial_state,
|
||||
query_start_loc,
|
||||
out,
|
||||
block_table,
|
||||
# Matrix dimensions
|
||||
dim,
|
||||
cu_seqlen,
|
||||
@@ -560,8 +580,9 @@ def causal_conv1d_fn(
|
||||
USE_PAD_SLOT=pad_slot_id is not None,
|
||||
NP2_STATELEN=np2_statelen,
|
||||
# launch_cooperative_grid=True
|
||||
BLOCK_M=8,
|
||||
BLOCK_M=CAUSAL_CONV1D_FWD_BLOCK_M,
|
||||
BLOCK_N=256,
|
||||
USE_BLOCK_TABLE=block_table is not None,
|
||||
num_stages=2,
|
||||
)
|
||||
return out
|
||||
|
||||
@@ -1499,6 +1499,13 @@ class Envs:
|
||||
# Eager forward wraps the ForwardBatch's own tensors instead of copying them
|
||||
# into the CUDA graph buffer registry (no per-iter device-to-device copy).
|
||||
SGLANG_EAGER_INPUT_NO_COPY = EnvBool(False)
|
||||
# Breakable prefill CUDA graphs capture the Triton KDA extend instead of
|
||||
# breaking the graph at every KDA layer (layers/attention/linear/
|
||||
# kda_prefill_graph.py); this kill-switch restores the eager break.
|
||||
SGLANG_DISABLE_KDA_PREFILL_GRAPH_EXTEND = EnvBool(False)
|
||||
# Sequences a captured KDA extend bucket accepts; a larger prefill batch on
|
||||
# any dp rank runs the step eagerly. Bounds the padded grids and scratch.
|
||||
SGLANG_KDA_PREFILL_GRAPH_MAX_SEQS = EnvInt(128)
|
||||
|
||||
# ===================================================================
|
||||
# Tokenizer, request state, embeddings, and reasoning controls
|
||||
|
||||
@@ -152,6 +152,9 @@ class AttentionBackend(ABC):
|
||||
# those tensor addresses. Such backends opt in here, create the metadata
|
||||
# object during capture, and refresh its dynamic fields before each replay.
|
||||
use_captured_forward_metadata_for_breakable_cuda_graph: bool = False
|
||||
# Most requests one replay of that captured metadata can carry (None: no
|
||||
# bound). Fixed at init: the dp-attention replay vote checks it too.
|
||||
prefill_cuda_graph_max_batch_size: Optional[int] = None
|
||||
|
||||
# Backends may keep MIXED prefill eager under DP attention when replaying
|
||||
# the EXTEND graph is a known serving-performance regression.
|
||||
|
||||
@@ -35,6 +35,11 @@ from sglang.srt.layers.attention.mamba.replay_state_indices_validator import (
|
||||
)
|
||||
from sglang.srt.layers.radix_attention import RadixAttention
|
||||
from sglang.srt.mem_cache.memory_pool import HybridReqToTokenPool
|
||||
from sglang.srt.model_executor.cuda_graph_config import (
|
||||
Backend,
|
||||
Phase,
|
||||
check_cuda_graph_backend,
|
||||
)
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode
|
||||
from sglang.srt.model_executor.model_runner import ModelRunner
|
||||
from sglang.srt.runtime_context import get_exec, get_memory, get_spec
|
||||
@@ -68,6 +73,10 @@ class MambaAttnBackendBase(AttentionBackend):
|
||||
# else — update_mamba_state_after_mtp_verify keys the fused branch on it.
|
||||
accept_lens_pool: Optional[torch.Tensor] = None
|
||||
|
||||
# Most sequences the prefill graph tables hold (larger batches replay no
|
||||
# graph); set at init by backends whose supports_prefill_graph_extend holds.
|
||||
prefill_graph_max_seqs: int
|
||||
|
||||
def __init__(self, model_runner: ModelRunner):
|
||||
self.validate_mis_support(model_runner.server_args)
|
||||
super().__init__()
|
||||
@@ -109,6 +118,21 @@ class MambaAttnBackendBase(AttentionBackend):
|
||||
# skips the per-cycle method dispatch.
|
||||
self._graph_seq_len_fill_value = self.get_cuda_graph_seq_len_fill_value()
|
||||
|
||||
def supports_prefill_graph_extend(self) -> bool:
|
||||
"""Whether breakable prefill CUDA graphs can capture this backend's
|
||||
extend on static per-bucket tables instead of breaking at each layer."""
|
||||
return False
|
||||
|
||||
def prefill_graph_extend_active(self) -> bool:
|
||||
"""Whether the extend being captured or replayed runs on those tables."""
|
||||
return False
|
||||
|
||||
def init_prefill_graph_metadata(self, forward_batch: ForwardBatch):
|
||||
raise NotImplementedError()
|
||||
|
||||
def refresh_prefill_graph_metadata(self, meta, forward_batch: ForwardBatch):
|
||||
raise NotImplementedError()
|
||||
|
||||
@property
|
||||
def mamba_chunk_size(self) -> int:
|
||||
if self._mamba_chunk_size is None:
|
||||
@@ -1178,6 +1202,22 @@ class HybridLinearAttnBackend(AttentionBackend):
|
||||
self.extend_dummy_seqs_capped_by_req_pool = getattr(
|
||||
full_attn_backend, "extend_dummy_seqs_capped_by_req_pool", False
|
||||
) or getattr(linear_attn_backend, "extend_dummy_seqs_capped_by_req_pool", False)
|
||||
# Breakable prefill CUDA graphs capture the linear extend when the linear
|
||||
# backend can run it on static tables (linear/kda_prefill_graph.py);
|
||||
# those tables follow the runner's captured-metadata contract.
|
||||
self.use_captured_forward_metadata_for_breakable_cuda_graph = (
|
||||
check_cuda_graph_backend(Phase.PREFILL, Backend.BREAKABLE)
|
||||
and linear_attn_backend.supports_prefill_graph_extend()
|
||||
and not full_attn_backend.use_captured_forward_metadata_for_breakable_cuda_graph
|
||||
)
|
||||
if self.use_captured_forward_metadata_for_breakable_cuda_graph:
|
||||
self.prefill_cuda_graph_max_batch_size = (
|
||||
linear_attn_backend.prefill_graph_max_seqs
|
||||
)
|
||||
logger.info(
|
||||
"Breakable prefill CUDA graphs capture the %s extend.",
|
||||
type(linear_attn_backend).__name__,
|
||||
)
|
||||
|
||||
@property
|
||||
def data_type(self):
|
||||
@@ -1230,6 +1270,49 @@ class HybridLinearAttnBackend(AttentionBackend):
|
||||
for attn_backend in self.attn_backend_list:
|
||||
attn_backend.init_forward_metadata_in_graph(forward_batch)
|
||||
|
||||
@staticmethod
|
||||
def _is_plain_extend(forward_batch: ForwardBatch) -> bool:
|
||||
mode = forward_batch.forward_mode
|
||||
return (
|
||||
mode.is_extend()
|
||||
and not mode.is_target_verify()
|
||||
and not mode.is_draft_extend_v2()
|
||||
)
|
||||
|
||||
def linear_extend_in_graph(self) -> bool:
|
||||
"""Whether the linear layers of the prefill being captured run inside
|
||||
the graph (no eager break)."""
|
||||
return self.linear_attn_backend.prefill_graph_extend_active()
|
||||
|
||||
def init_forward_metadata_for_breakable_cuda_graph_capture(
|
||||
self, forward_batch: ForwardBatch
|
||||
):
|
||||
self.full_attn_backend.init_forward_metadata(forward_batch)
|
||||
if not self._is_plain_extend(forward_batch):
|
||||
if not forward_batch.forward_mode.is_draft_extend_v2():
|
||||
self.linear_attn_backend.init_forward_metadata(forward_batch)
|
||||
return None
|
||||
return self.linear_attn_backend.init_prefill_graph_metadata(forward_batch)
|
||||
|
||||
def prepare_forward_metadata_for_breakable_cuda_graph_replay(
|
||||
self,
|
||||
capture_metadata,
|
||||
forward_batch: ForwardBatch,
|
||||
*,
|
||||
static_forward_batch: Optional[ForwardBatch] = None,
|
||||
) -> None:
|
||||
self.full_attn_backend.init_forward_metadata(forward_batch)
|
||||
if capture_metadata is not None:
|
||||
self.linear_attn_backend.refresh_prefill_graph_metadata(
|
||||
capture_metadata, forward_batch
|
||||
)
|
||||
elif not forward_batch.forward_mode.is_draft_extend_v2():
|
||||
self.linear_attn_backend.init_forward_metadata(forward_batch)
|
||||
if static_forward_batch is not None:
|
||||
self.prepare_prefill_shared_read_snapshot(
|
||||
forward_batch, num_qo_tokens=static_forward_batch.positions.shape[0]
|
||||
)
|
||||
|
||||
def get_indexer_metadata(self, layer_id: int, forward_batch: ForwardBatch):
|
||||
if layer_id in self.full_attn_layers:
|
||||
return self.full_attn_backend.get_indexer_metadata(layer_id, forward_batch)
|
||||
|
||||
@@ -10,6 +10,12 @@ from sglang.kernels.ops.mamba.causal_conv1d_triton import (
|
||||
)
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.layers.attention.hybrid_linear_attn_backend import MambaAttnBackendBase
|
||||
from sglang.srt.layers.attention.linear.kda_prefill_graph import (
|
||||
KDAPrefillGraphMetadata,
|
||||
KDAPrefillTrackRows,
|
||||
build_track_rows,
|
||||
kda_extend_in_graph,
|
||||
)
|
||||
from sglang.srt.layers.attention.linear.kernels.kda_flashinfer import (
|
||||
build_fused_accept_indices,
|
||||
)
|
||||
@@ -43,6 +49,7 @@ from sglang.srt.runtime_context import (
|
||||
get_memory,
|
||||
get_platform,
|
||||
get_spec,
|
||||
mamba_cache_chunk_size,
|
||||
)
|
||||
|
||||
|
||||
@@ -454,8 +461,13 @@ class KDAAttnBackend(MambaAttnBackendBase):
|
||||
# force-flush path.
|
||||
needs_cpu_seq_lens: bool = False
|
||||
|
||||
# Set while a breakable prefill CUDA graph captures or replays a bucket:
|
||||
# forward_extend then runs the graph-safe chain on these static tables.
|
||||
prefill_graph_metadata: Optional[KDAPrefillGraphMetadata] = None
|
||||
|
||||
def __init__(self, model_runner: ModelRunner):
|
||||
super().__init__(model_runner)
|
||||
self.prefill_graph_max_seqs = envs.SGLANG_KDA_PREFILL_GRAPH_MAX_SEQS.get()
|
||||
# Needed by the extra_buffer track path: _init_track_conv_indices reads
|
||||
# conv_states_shape[-1] as the conv window length (kernel_size - 1).
|
||||
# The KDA pool stores conv states as [kernel-1, dim] — transposed vs
|
||||
@@ -590,7 +602,110 @@ class KDAAttnBackend(MambaAttnBackendBase):
|
||||
metadata.fused_accept_num_accepted,
|
||||
)
|
||||
|
||||
def supports_prefill_graph_extend(self) -> bool:
|
||||
return (
|
||||
not envs.SGLANG_DISABLE_KDA_PREFILL_GRAPH_EXTEND.get()
|
||||
# Only the Triton chain guards padded rows (-1 state slots, chunks past T).
|
||||
and isinstance(self.kernel_dispatcher.extend_kernel, TritonKDAKernel)
|
||||
# Fused-accept staging (flashinfer verify) writes per extend.
|
||||
and self.accept_lens_pool is None
|
||||
)
|
||||
|
||||
def prefill_graph_extend_active(self) -> bool:
|
||||
return self.prefill_graph_metadata is not None
|
||||
|
||||
def init_prefill_graph_metadata(
|
||||
self, forward_batch: ForwardBatch
|
||||
) -> KDAPrefillGraphMetadata:
|
||||
cache_indices = self._prefill_graph_cache_indices(forward_batch)
|
||||
meta = KDAPrefillGraphMetadata.allocate(
|
||||
num_tokens=forward_batch.positions.shape[0],
|
||||
max_seqs_cap=self.prefill_graph_max_seqs,
|
||||
# Any replay of this bucket may carry prefix-cache snapshots (the
|
||||
# extra buffer is only on with the radix cache).
|
||||
track=get_exec().mamba.enable_mamba_extra_buffer,
|
||||
cache_index_dtype=cache_indices.dtype,
|
||||
device=self.device,
|
||||
)
|
||||
self._fill_prefill_graph_metadata(
|
||||
meta=meta, forward_batch=forward_batch, cache_indices=cache_indices
|
||||
)
|
||||
return meta
|
||||
|
||||
def refresh_prefill_graph_metadata(
|
||||
self, meta: KDAPrefillGraphMetadata, forward_batch: ForwardBatch
|
||||
) -> None:
|
||||
self._fill_prefill_graph_metadata(
|
||||
meta=meta,
|
||||
forward_batch=forward_batch,
|
||||
cache_indices=self._prefill_graph_cache_indices(forward_batch),
|
||||
)
|
||||
|
||||
def _prefill_graph_cache_indices(self, forward_batch: ForwardBatch):
|
||||
# Same slot resolution as MambaAttnBackendBase._forward_metadata.
|
||||
cache_indices = self._translate_mamba_indices(
|
||||
self.req_to_token_pool.get_mamba_indices(forward_batch.req_pool_indices)
|
||||
)
|
||||
real_bs = forward_batch._original_batch_size
|
||||
if real_bs is not None and real_bs < cache_indices.shape[0]:
|
||||
cache_indices = cache_indices.clone()
|
||||
cache_indices[real_bs:] = -1
|
||||
return cache_indices
|
||||
|
||||
def _fill_prefill_graph_metadata(
|
||||
self,
|
||||
*,
|
||||
meta: KDAPrefillGraphMetadata,
|
||||
forward_batch: ForwardBatch,
|
||||
cache_indices: torch.Tensor,
|
||||
) -> None:
|
||||
if forward_batch.mamba_track_indices is not None:
|
||||
# Same one-time in-place translation as the eager metadata path.
|
||||
forward_batch.mamba_track_indices = self._translate_mamba_indices(
|
||||
forward_batch.mamba_track_indices
|
||||
)
|
||||
track_rows = None
|
||||
if forward_batch.mamba_track_mask is not None:
|
||||
assert meta.track, (
|
||||
"prefix-cache track batch replayed on a bucket captured without "
|
||||
"track tables"
|
||||
)
|
||||
track_rows = self._prefill_graph_track_rows(forward_batch)
|
||||
meta.fill(
|
||||
extend_seq_lens=list(forward_batch.extend_seq_lens_cpu),
|
||||
cache_indices=cache_indices,
|
||||
extend_prefix_lens=forward_batch.extend_prefix_lens,
|
||||
track_rows=track_rows,
|
||||
)
|
||||
self.prefill_graph_metadata = meta
|
||||
|
||||
def _prefill_graph_track_rows(
|
||||
self, forward_batch: ForwardBatch
|
||||
) -> Optional[KDAPrefillTrackRows]:
|
||||
bs = forward_batch.batch_size
|
||||
if self._has_cpu_prefill_track_metadata(forward_batch):
|
||||
track_mask = list(forward_batch.mamba_prefill_track_mask_cpu)
|
||||
track_seq_lens = list(forward_batch.mamba_track_seqlens_cpu)
|
||||
extend_prefix_lens = list(forward_batch.extend_prefix_lens_cpu)
|
||||
else:
|
||||
# The same host copies the eager path takes without CPU metadata.
|
||||
track_mask = forward_batch.mamba_track_mask[:bs].tolist()
|
||||
track_seq_lens = forward_batch.mamba_track_seqlens[:bs].tolist()
|
||||
extend_prefix_lens = forward_batch.extend_prefix_lens[:bs].tolist()
|
||||
return build_track_rows(
|
||||
track_mask=track_mask,
|
||||
track_seq_lens=track_seq_lens,
|
||||
extend_seq_lens=list(forward_batch.extend_seq_lens_cpu),
|
||||
extend_prefix_lens=extend_prefix_lens,
|
||||
state_chunk_size=self.mamba_chunk_size,
|
||||
cache_chunk_size=mamba_cache_chunk_size(),
|
||||
conv_len=self.conv_states_shape[-1],
|
||||
dst=forward_batch.mamba_track_indices,
|
||||
)
|
||||
|
||||
def init_forward_metadata(self, forward_batch: ForwardBatch):
|
||||
# An eager forward never runs on a graph bucket's tables.
|
||||
self.prefill_graph_metadata = None
|
||||
super().init_forward_metadata(forward_batch)
|
||||
if self.forward_metadata.has_mamba_track_mask:
|
||||
if self.forward_metadata.mamba_track_mask_indices is None:
|
||||
@@ -908,6 +1023,21 @@ class KDAAttnBackend(MambaAttnBackendBase):
|
||||
if forward_batch.forward_mode.is_target_verify():
|
||||
return self._forward_target_verify(layer, forward_batch, mixed_qkv, a, b)
|
||||
|
||||
if self.prefill_graph_metadata is not None:
|
||||
# Breakable prefill CUDA graph bucket: host-free chain on static
|
||||
# tables refreshed before each replay.
|
||||
layer_cache = self.req_to_token_pool.mamba2_layer_cache(layer.layer_id)
|
||||
return kda_extend_in_graph(
|
||||
layer=layer,
|
||||
meta=self.prefill_graph_metadata,
|
||||
conv_pool=layer_cache.conv[0],
|
||||
ssm_states=layer_cache.temporal,
|
||||
kernel_dispatcher=self.kernel_dispatcher,
|
||||
mixed_qkv=mixed_qkv,
|
||||
a=a,
|
||||
b=b,
|
||||
)
|
||||
|
||||
query_start_loc = self.forward_metadata.query_start_loc
|
||||
cache_indices = self.forward_metadata.mamba_cache_indices
|
||||
|
||||
|
||||
@@ -0,0 +1,389 @@
|
||||
"""KDA extend captured inside breakable prefill CUDA graphs.
|
||||
|
||||
A breakable prefill CUDA graph (BCG) is captured once per token bucket and
|
||||
replayed for any batch that pads into that bucket. The eager KDA extend derives
|
||||
its launch grids and index tables from host-side sequence lengths
|
||||
(``prepare_chunk_indices``, ``causal_conv1d_fn``'s ``seq_lens_cpu``, the
|
||||
prefix-cache track plan), so without this module BCG breaks the graph at every
|
||||
KDA layer and runs the whole chain -- about sixteen kernels per layer plus their
|
||||
Python -- eagerly.
|
||||
|
||||
For each bucket this module allocates static tables sized by the bucket's
|
||||
bounds:
|
||||
|
||||
* ``max_seqs`` sequences (``min(bucket, cap)``; batches with more sequences run
|
||||
eagerly, through the runner's replay vote on
|
||||
``AttentionBackend.prefill_cuda_graph_max_batch_size``),
|
||||
* ``max_chunks = cdiv(bucket, 64) + max_seqs`` delta-rule chunks,
|
||||
* ``max_conv_blocks = cdiv(bucket, 8) + max_seqs`` conv blocks.
|
||||
|
||||
Before every replay ``fill`` rebuilds the tables from the batch's CPU lengths
|
||||
(one host-to-device copy plus a few device gathers, no synchronization). The
|
||||
captured kernels launch fixed grids over the bounds; padded rows point past
|
||||
their sequence (chunk tables) or carry ``PAD_SLOT_ID`` (conv blocks, state
|
||||
slots) and return immediately.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING, Optional
|
||||
|
||||
import msgspec
|
||||
import numpy as np
|
||||
import torch
|
||||
import triton
|
||||
|
||||
from sglang.kernels.ops.attention.fla.kda_prefill_graph_track import (
|
||||
TRACK_CHUNK_STATE,
|
||||
TRACK_FINAL_STATE,
|
||||
kda_track_conv_window,
|
||||
kda_track_ssm_state,
|
||||
)
|
||||
from sglang.kernels.ops.mamba.causal_conv1d_triton import (
|
||||
CAUSAL_CONV1D_FWD_BLOCK_M,
|
||||
PAD_SLOT_ID,
|
||||
causal_conv1d_fn,
|
||||
)
|
||||
from sglang.srt.layers.attention.mamba.prefill_track_metadata import (
|
||||
build_prefill_track_plan,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.layers.attention.linear.kda_backend import KDAKernelDispatcher
|
||||
from sglang.srt.layers.radix_linear_attention import RadixLinearAttention
|
||||
|
||||
# Delta-rule chunk size of chunk_kda (fla/kda.py, fla/chunk_delta_h.py).
|
||||
KDA_CHUNK_SIZE = 64
|
||||
# Chunk index of padded chunk_indices rows: past the end of any sequence, so
|
||||
# every varlen chunk kernel returns on its `i_t * BT >= T` guard.
|
||||
_PAD_CHUNK = 1 << 20
|
||||
|
||||
|
||||
class KDAPrefillTrackRows(msgspec.Struct, frozen=True):
|
||||
"""One batch's prefix-cache track plan, in batch rows."""
|
||||
|
||||
# (row, chunk): the snapshot is the state at that chunk boundary.
|
||||
chunk_rows: list[tuple[int, int]]
|
||||
# Rows whose snapshot is the post-extend state (chunk-aligned length).
|
||||
final_rows: list[int]
|
||||
# (row, aligned_len) of every tracked row: its conv window ends there.
|
||||
conv_rows: list[tuple[int, int]]
|
||||
# Conv window length (kernel_size - 1).
|
||||
conv_len: int
|
||||
# [bs] physical track slots, on the device.
|
||||
dst: torch.Tensor
|
||||
|
||||
|
||||
class KDAPrefillGraphMetadata(msgspec.Struct, frozen=True):
|
||||
"""Static per-bucket tables read by the captured KDA extend."""
|
||||
|
||||
num_tokens: int
|
||||
max_seqs: int
|
||||
max_chunks: int
|
||||
max_conv_blocks: int
|
||||
# Allocates and records the prefix-cache snapshot copies.
|
||||
track: bool
|
||||
|
||||
# One int32 device buffer, refreshed by a single copy, and its views.
|
||||
packed: torch.Tensor
|
||||
cu_seqlens: torch.Tensor # [max_seqs + 1]
|
||||
chunk_indices: torch.Tensor # [max_chunks, 2] (seq, chunk)
|
||||
chunk_offsets: torch.Tensor # [max_seqs + 1]
|
||||
conv_block_table: torch.Tensor # [max_conv_blocks, 2] (seq, block)
|
||||
track_chunk_idx: torch.Tensor # [max_seqs]
|
||||
track_mode: torch.Tensor # [max_seqs]
|
||||
track_conv_start: torch.Tensor # [max_seqs]
|
||||
num_logical_tokens: torch.Tensor # [1]
|
||||
host: np.ndarray # int32 staging for `packed`
|
||||
|
||||
# Filled on the device from the batch.
|
||||
cache_indices: torch.Tensor # [max_seqs], PAD_SLOT_ID past the batch
|
||||
has_initial_state: torch.Tensor # [max_seqs] bool
|
||||
track_dst: torch.Tensor # [max_seqs] int64
|
||||
token_index: torch.Tensor # [num_tokens] int32 arange
|
||||
|
||||
@classmethod
|
||||
def allocate(
|
||||
cls,
|
||||
*,
|
||||
num_tokens: int,
|
||||
max_seqs_cap: int,
|
||||
track: bool,
|
||||
cache_index_dtype: torch.dtype,
|
||||
device: torch.device,
|
||||
) -> KDAPrefillGraphMetadata:
|
||||
max_seqs = max(1, min(num_tokens, max_seqs_cap))
|
||||
max_chunks = triton.cdiv(num_tokens, KDA_CHUNK_SIZE) + max_seqs
|
||||
max_conv_blocks = triton.cdiv(num_tokens, CAUSAL_CONV1D_FWD_BLOCK_M) + max_seqs
|
||||
sizes = [
|
||||
max_seqs + 1, # cu_seqlens
|
||||
2 * max_chunks, # chunk_indices
|
||||
max_seqs + 1, # chunk_offsets
|
||||
2 * max_conv_blocks, # conv_block_table
|
||||
max_seqs, # track_chunk_idx
|
||||
max_seqs, # track_mode
|
||||
max_seqs, # track_conv_start
|
||||
1, # num_logical_tokens
|
||||
]
|
||||
packed = torch.zeros(sum(sizes), dtype=torch.int32, device=device)
|
||||
views = torch.split(packed, sizes)
|
||||
return cls(
|
||||
num_tokens=num_tokens,
|
||||
max_seqs=max_seqs,
|
||||
max_chunks=max_chunks,
|
||||
max_conv_blocks=max_conv_blocks,
|
||||
track=track,
|
||||
packed=packed,
|
||||
cu_seqlens=views[0],
|
||||
chunk_indices=views[1].view(max_chunks, 2),
|
||||
chunk_offsets=views[2],
|
||||
conv_block_table=views[3].view(max_conv_blocks, 2),
|
||||
track_chunk_idx=views[4],
|
||||
track_mode=views[5],
|
||||
track_conv_start=views[6],
|
||||
num_logical_tokens=views[7],
|
||||
host=np.zeros(sum(sizes), dtype=np.int32),
|
||||
cache_indices=torch.full(
|
||||
(max_seqs,), PAD_SLOT_ID, dtype=cache_index_dtype, device=device
|
||||
),
|
||||
has_initial_state=torch.zeros(max_seqs, dtype=torch.bool, device=device),
|
||||
track_dst=torch.zeros(max_seqs, dtype=torch.int64, device=device),
|
||||
token_index=torch.arange(num_tokens, dtype=torch.int32, device=device),
|
||||
)
|
||||
|
||||
def can_fit(self, extend_seq_lens: list[int]) -> bool:
|
||||
return (
|
||||
len(extend_seq_lens) <= self.max_seqs
|
||||
and sum(extend_seq_lens) <= self.num_tokens
|
||||
)
|
||||
|
||||
def fill(
|
||||
self,
|
||||
*,
|
||||
extend_seq_lens: list[int],
|
||||
cache_indices: torch.Tensor,
|
||||
extend_prefix_lens: torch.Tensor,
|
||||
track_rows: Optional[KDAPrefillTrackRows],
|
||||
) -> None:
|
||||
"""Rebuild the tables for this batch. ``cache_indices`` and
|
||||
``extend_prefix_lens`` are the batch's device tensors."""
|
||||
bs = len(extend_seq_lens)
|
||||
assert self.can_fit(extend_seq_lens), (
|
||||
f"batch of {bs} sequences / {sum(extend_seq_lens)} tokens exceeds the "
|
||||
f"captured bounds ({self.max_seqs} / {self.num_tokens})"
|
||||
)
|
||||
self._write_host_tables(
|
||||
lens=np.asarray(extend_seq_lens, dtype=np.int64), track_rows=track_rows
|
||||
)
|
||||
|
||||
# A pinned block from the caching host allocator keeps the copy
|
||||
# asynchronous and is not reused before the copy completes.
|
||||
self.packed.copy_(torch.from_numpy(self.host).pin_memory(), non_blocking=True)
|
||||
self.cache_indices[:bs].copy_(cache_indices[:bs])
|
||||
self.cache_indices[bs:].fill_(PAD_SLOT_ID)
|
||||
torch.gt(extend_prefix_lens[:bs], 0, out=self.has_initial_state[:bs])
|
||||
self.has_initial_state[bs:].fill_(False)
|
||||
if track_rows is not None:
|
||||
self.track_dst[:bs].copy_(track_rows.dst[:bs])
|
||||
|
||||
def _write_host_tables(
|
||||
self, *, lens: np.ndarray, track_rows: Optional[KDAPrefillTrackRows]
|
||||
) -> None:
|
||||
"""Write every host-built table into ``self.host`` in ``packed`` order."""
|
||||
bs = lens.shape[0]
|
||||
total = int(lens.sum())
|
||||
host = self.host
|
||||
|
||||
# Padded sequences are empty and sit at the end.
|
||||
cu = np.empty(self.max_seqs + 1, dtype=np.int64)
|
||||
cu[0] = 0
|
||||
np.cumsum(lens, out=cu[1 : bs + 1])
|
||||
cu[bs + 1 :] = total
|
||||
o = self._write(o=0, values=cu)
|
||||
|
||||
# Sequence-major (seq, chunk) pairs: chunk_gla_fwd_o reads the per-chunk
|
||||
# states by row, chunk_gated_delta_rule_fwd_h writes them at
|
||||
# chunk_offsets[seq] + chunk.
|
||||
n_chunks = (lens + KDA_CHUNK_SIZE - 1) // KDA_CHUNK_SIZE
|
||||
o = self._write_pairs(
|
||||
o=o, counts=n_chunks, rows=self.max_chunks, pad=(0, _PAD_CHUNK)
|
||||
)
|
||||
offsets = np.empty(self.max_seqs + 1, dtype=np.int64)
|
||||
offsets[0] = 0
|
||||
np.cumsum(n_chunks, out=offsets[1 : bs + 1])
|
||||
offsets[bs + 1 :] = offsets[bs]
|
||||
o = self._write(o=o, values=offsets)
|
||||
|
||||
n_blocks = (lens + CAUSAL_CONV1D_FWD_BLOCK_M - 1) // CAUSAL_CONV1D_FWD_BLOCK_M
|
||||
o = self._write_pairs(
|
||||
o=o, counts=n_blocks, rows=self.max_conv_blocks, pad=(PAD_SLOT_ID, 0)
|
||||
)
|
||||
|
||||
track_chunk_idx = np.full(self.max_seqs, -1, dtype=np.int64)
|
||||
track_mode = np.zeros(self.max_seqs, dtype=np.int64)
|
||||
track_conv_start = np.zeros(self.max_seqs, dtype=np.int64)
|
||||
if track_rows is not None:
|
||||
for row, chunk in track_rows.chunk_rows:
|
||||
track_chunk_idx[row] = chunk
|
||||
track_mode[row] = TRACK_CHUNK_STATE
|
||||
for row in track_rows.final_rows:
|
||||
track_mode[row] = TRACK_FINAL_STATE
|
||||
for row, aligned_len in track_rows.conv_rows:
|
||||
track_conv_start[row] = cu[row] + aligned_len - track_rows.conv_len
|
||||
o = self._write(o=o, values=track_chunk_idx)
|
||||
o = self._write(o=o, values=track_mode)
|
||||
o = self._write(o=o, values=track_conv_start)
|
||||
o = self._write(o=o, values=np.array([total]))
|
||||
assert o == host.shape[0]
|
||||
|
||||
def _write(self, *, o: int, values: np.ndarray) -> int:
|
||||
self.host[o : o + values.shape[0]] = values
|
||||
return o + values.shape[0]
|
||||
|
||||
def _write_pairs(
|
||||
self, *, o: int, counts: np.ndarray, rows: int, pad: tuple[int, int]
|
||||
) -> int:
|
||||
"""Write (seq, index) rows, `counts[seq]` indices per sequence, into a
|
||||
[rows, 2] table at `o`, padding the tail with `pad`."""
|
||||
n = int(counts.sum())
|
||||
table = self.host[o : o + 2 * rows].reshape(rows, 2)
|
||||
if n:
|
||||
table[:n, 0] = np.repeat(np.arange(counts.shape[0]), counts)
|
||||
table[:n, 1] = np.arange(n) - np.repeat(np.cumsum(counts) - counts, counts)
|
||||
table[n:, 0] = pad[0]
|
||||
table[n:, 1] = pad[1]
|
||||
return o + 2 * rows
|
||||
|
||||
|
||||
def build_track_rows(
|
||||
*,
|
||||
track_mask: list[bool],
|
||||
track_seq_lens: list[int],
|
||||
extend_seq_lens: list[int],
|
||||
extend_prefix_lens: list[int],
|
||||
state_chunk_size: int,
|
||||
cache_chunk_size: int,
|
||||
conv_len: int,
|
||||
dst: torch.Tensor,
|
||||
) -> Optional[KDAPrefillTrackRows]:
|
||||
"""The eager track selection (_init_track_ssm_indices_from_cpu and
|
||||
_init_track_conv_indices) as per-row lists; None when nothing is tracked."""
|
||||
if not any(track_mask):
|
||||
return None
|
||||
plan = build_prefill_track_plan(
|
||||
track_mask,
|
||||
track_seq_lens,
|
||||
extend_seq_lens,
|
||||
extend_prefix_lens,
|
||||
state_chunk_size,
|
||||
mamba2=False,
|
||||
)
|
||||
conv_rows = [
|
||||
(
|
||||
row,
|
||||
(track_seq_lens[row] - extend_prefix_lens[row])
|
||||
// cache_chunk_size
|
||||
* cache_chunk_size,
|
||||
)
|
||||
for row in plan.tracked_rows
|
||||
]
|
||||
return KDAPrefillTrackRows(
|
||||
chunk_rows=[(row, plan.chunk_indices[row]) for row in plan.unaligned_rows],
|
||||
final_rows=list(plan.final_rows),
|
||||
conv_rows=conv_rows,
|
||||
conv_len=conv_len,
|
||||
dst=dst,
|
||||
)
|
||||
|
||||
|
||||
def kda_extend_in_graph(
|
||||
*,
|
||||
layer: RadixLinearAttention,
|
||||
meta: KDAPrefillGraphMetadata,
|
||||
conv_pool: torch.Tensor,
|
||||
ssm_states: torch.Tensor,
|
||||
kernel_dispatcher: KDAKernelDispatcher,
|
||||
mixed_qkv: torch.Tensor,
|
||||
a: torch.Tensor,
|
||||
b: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""KDAAttnBackend.forward_extend with launch grids that depend only on the
|
||||
bucket. ``conv_pool`` is the layer's [slots, kernel-1, dim] conv cache.
|
||||
Returns [1, num_tokens, H, V]; rows past the batch's tokens are zero."""
|
||||
if meta.track:
|
||||
# Snapshot the conv window before the conv kernel updates the cache;
|
||||
# the source is the raw input, the destinations are track slots.
|
||||
kda_track_conv_window(
|
||||
conv_pool=conv_pool,
|
||||
x=mixed_qkv,
|
||||
track_mode=meta.track_mode,
|
||||
track_conv_start=meta.track_conv_start,
|
||||
track_dst=meta.track_dst,
|
||||
num_tokens=meta.num_logical_tokens,
|
||||
)
|
||||
|
||||
qkv = causal_conv1d_fn(
|
||||
mixed_qkv.transpose(0, 1),
|
||||
layer.conv_weights,
|
||||
layer.bias,
|
||||
activation="silu",
|
||||
conv_states=conv_pool.transpose(-1, -2),
|
||||
has_initial_state=meta.has_initial_state,
|
||||
cache_indices=meta.cache_indices,
|
||||
query_start_loc=meta.cu_seqlens,
|
||||
seq_lens_cpu=None,
|
||||
block_table=meta.conv_block_table,
|
||||
).transpose(0, 1)
|
||||
q, k, v = qkv.split([layer.q_dim, layer.k_dim, layer.v_dim], dim=-1)
|
||||
q = q.unflatten(-1, (-1, layer.head_q_dim)).unsqueeze(0)
|
||||
k = k.unflatten(-1, (-1, layer.head_k_dim)).unsqueeze(0)
|
||||
v = v.unflatten(-1, (-1, layer.head_v_dim)).unsqueeze(0)
|
||||
|
||||
gate_was_flat = a.ndim == 3
|
||||
if gate_was_flat:
|
||||
a = a.unflatten(-1, (-1, layer.head_k_dim))
|
||||
|
||||
h_track = None
|
||||
if meta.track:
|
||||
h_track = torch.empty(
|
||||
(meta.max_seqs, *ssm_states.shape[1:]),
|
||||
dtype=torch.float32,
|
||||
device=ssm_states.device,
|
||||
)
|
||||
core_attn_out = kernel_dispatcher.extend(
|
||||
q=q,
|
||||
k=k,
|
||||
v=v,
|
||||
g=a,
|
||||
beta=b,
|
||||
ssm_states=ssm_states,
|
||||
cache_indices=meta.cache_indices,
|
||||
query_start_loc=meta.cu_seqlens,
|
||||
A_log=layer.A_log,
|
||||
dt_bias=layer.dt_bias,
|
||||
lower_bound=layer.lower_bound,
|
||||
beta_is_raw=gate_was_flat,
|
||||
extend_seq_lens_cpu=None,
|
||||
is_spec_decode=False,
|
||||
return_intermediate_states=False,
|
||||
track_state=h_track,
|
||||
track_chunk_idx=meta.track_chunk_idx if meta.track else None,
|
||||
chunk_indices=meta.chunk_indices,
|
||||
chunk_offsets=meta.chunk_offsets,
|
||||
)
|
||||
|
||||
if meta.track:
|
||||
kda_track_ssm_state(
|
||||
ssm_pool=ssm_states,
|
||||
h_track=h_track,
|
||||
track_mode=meta.track_mode,
|
||||
track_dst=meta.track_dst,
|
||||
cache_indices=meta.cache_indices,
|
||||
)
|
||||
|
||||
# Rows past the batch belong to no sequence and were never written; keep the
|
||||
# layer's padded output finite for the MoE and the collectives.
|
||||
core_attn_out.masked_fill_(
|
||||
(meta.token_index >= meta.num_logical_tokens).view(1, -1, 1, 1), 0
|
||||
)
|
||||
return core_attn_out
|
||||
@@ -251,4 +251,6 @@ class TritonKDAKernel(LinearAttnKernelBase):
|
||||
output_intermediate_states=return_intermediate_states,
|
||||
track_state=kwargs.get("track_state"),
|
||||
track_chunk_idx=kwargs.get("track_chunk_idx"),
|
||||
chunk_indices=kwargs.get("chunk_indices"),
|
||||
chunk_offsets=kwargs.get("chunk_offsets"),
|
||||
)
|
||||
|
||||
@@ -93,7 +93,7 @@ class RadixLinearAttention(nn.Module):
|
||||
dtype=mixed_qkv.dtype,
|
||||
device=mixed_qkv.device,
|
||||
)
|
||||
if is_in_breakable_cuda_graph():
|
||||
if is_in_breakable_cuda_graph() and not _linear_extend_in_graph():
|
||||
bcg_unified_linear_attention_with_output(
|
||||
mixed_qkv,
|
||||
a,
|
||||
@@ -148,6 +148,20 @@ class RadixLinearAttention(nn.Module):
|
||||
)
|
||||
|
||||
|
||||
def _linear_extend_in_graph() -> bool:
|
||||
"""Whether the backend runs this breakable prefill capture's linear extend
|
||||
on static tables inside the graph (no eager break at the layer)."""
|
||||
from sglang.srt.layers.attention.hybrid_linear_attn_backend import (
|
||||
HybridLinearAttnBackend,
|
||||
)
|
||||
|
||||
backend = get_attn_backend()
|
||||
return (
|
||||
isinstance(backend, HybridLinearAttnBackend)
|
||||
and backend.linear_extend_in_graph()
|
||||
)
|
||||
|
||||
|
||||
def _linear_attention_with_output_impl(
|
||||
mixed_qkv: torch.Tensor,
|
||||
a: torch.Tensor,
|
||||
|
||||
@@ -289,6 +289,7 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
|
||||
|
||||
_use_draft_input_embeds = False
|
||||
_backend_can_run_prefill_cuda_graph = None
|
||||
_captured_attn_metadata_max_bs: Optional[int] = None
|
||||
dllm_attention = None
|
||||
|
||||
def __init__(self, model_runner: ModelRunner):
|
||||
@@ -612,6 +613,11 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
|
||||
self.use_captured_attn_metadata = model_runner.attn_backend.use_captured_forward_metadata_for_breakable_cuda_graph
|
||||
else:
|
||||
self.use_captured_attn_metadata = False
|
||||
self._captured_attn_metadata_max_bs = (
|
||||
model_runner.attn_backend.prefill_cuda_graph_max_batch_size
|
||||
if self.use_captured_attn_metadata
|
||||
else None
|
||||
)
|
||||
self.attn_metadata_buffers: Optional[Dict[ShapeKey, object]] = (
|
||||
{} if self.use_captured_attn_metadata else None
|
||||
)
|
||||
@@ -1274,6 +1280,11 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
|
||||
return False
|
||||
if self._is_full_backend and batch_size > self._capture_req_slots:
|
||||
return False
|
||||
if (
|
||||
self._captured_attn_metadata_max_bs is not None
|
||||
and batch_size > self._captured_attn_metadata_max_bs
|
||||
):
|
||||
return False
|
||||
# LoRA replays need prepare_lora_batch's static metadata. lora_manager
|
||||
# keeps LoRA prefill eager on every rank under dp attention, so the
|
||||
# schedule-time vote derives this from enable_lora alone.
|
||||
|
||||
Reference in New Issue
Block a user