From 9a1aeacc3bcd3529445a31db8f71394e22e390ca Mon Sep 17 00:00:00 2001 From: Yuan Luo Date: Fri, 9 Oct 2026 13:37:20 +0800 Subject: [PATCH] [GLM-5.3-Flash] Capture the KDA extend in breakable prefill CUDA graphs (#41105) Co-authored-by: luoyuan.luo --- .../ops/attention/fla/chunk_delta_h.py | 11 +- .../sglang/kernels/ops/attention/fla/kda.py | 34 +- .../attention/fla/kda_prefill_graph_track.py | 148 +++++++ .../ops/attention/helion/kda_prefill.py | 14 +- .../kernels/ops/mamba/causal_conv1d_triton.py | 29 +- python/sglang/srt/environ.py | 7 + .../srt/layers/attention/base_attn_backend.py | 3 + .../attention/hybrid_linear_attn_backend.py | 83 ++++ .../layers/attention/linear/kda_backend.py | 130 ++++++ .../attention/linear/kda_prefill_graph.py | 389 ++++++++++++++++++ .../attention/linear/kernels/kda_triton.py | 2 + .../srt/layers/radix_linear_attention.py | 16 +- .../runner/prefill_cuda_graph_runner.py | 11 + .../ops/attention/test_kda_prefill_graph.py | 273 ++++++++++++ .../scheduler_components/test_dp_attn.py | 49 +++ 15 files changed, 1177 insertions(+), 22 deletions(-) create mode 100644 python/sglang/kernels/ops/attention/fla/kda_prefill_graph_track.py create mode 100644 python/sglang/srt/layers/attention/linear/kda_prefill_graph.py create mode 100644 test/registered/kernels/ops/attention/test_kda_prefill_graph.py diff --git a/python/sglang/kernels/ops/attention/fla/chunk_delta_h.py b/python/sglang/kernels/ops/attention/fla/chunk_delta_h.py index 84ae081f3a..b782e82778 100644 --- a/python/sglang/kernels/ops/attention/fla/chunk_delta_h.py +++ b/python/sglang/kernels/ops/attention/fla/chunk_delta_h.py @@ -363,6 +363,7 @@ def chunk_gated_delta_rule_fwd_h( inplace_update: bool = True, track_state: Optional[torch.Tensor] = None, track_chunk_idx: Optional[torch.Tensor] = None, + chunk_offsets: Optional[torch.Tensor] = None, ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: assert not (use_exp2 and g is not None), ( "use_exp2 covers only the per-channel gk path; scalar g stays natural-exp" @@ -386,11 +387,11 @@ def chunk_gated_delta_rule_fwd_h( if cu_seqlens is None: N, NT, chunk_offsets = B, triton.cdiv(T, BT), None else: - N, NT, chunk_offsets = ( - len(cu_seqlens) - 1, - len(chunk_indices), - prepare_chunk_offsets(cu_seqlens, BT), - ) + # A caller-built chunk_offsets (CUDA-graph capture) must not go through + # prepare_chunk_offsets: its tensor_cache would replay capture-time values. + N, NT = len(cu_seqlens) - 1, len(chunk_indices) + if chunk_offsets is None: + chunk_offsets = prepare_chunk_offsets(cu_seqlens, BT) assert K <= 256, "current kernel does not support head dimension larger than 256." h = k.new_empty(B, NT, H, V, K) diff --git a/python/sglang/kernels/ops/attention/fla/kda.py b/python/sglang/kernels/ops/attention/fla/kda.py index 05dee2f2ab..d4ec1ee88c 100644 --- a/python/sglang/kernels/ops/attention/fla/kda.py +++ b/python/sglang/kernels/ops/attention/fla/kda.py @@ -554,6 +554,12 @@ def _recompute_w_u_fwd_kernel( T = eos - bos else: bos, eos = i_b * T, i_b * T + T + + # Padded chunk_indices rows (kda_prefill_graph.py) start past T, where every + # store below is masked; this early exit only skips their wasted work. + if i_t * BT >= T: + return + p_b = tl.make_block_ptr(beta + bos * H + i_h, (T,), (H,), (i_t * BT,), (BT,), (0,)) b_b = tl.load(p_b, boundary_check=(0,)) @@ -797,6 +803,11 @@ def chunk_gla_fwd_kernel_o( i_tg = i_b * NT + i_t bos, eos = i_b * T, i_b * T + T + # Padded chunk_indices rows (kda_prefill_graph.py) start past T, where every + # store below is masked; this early exit only skips their wasted work. + if i_t * BT >= T: + return + m_s = tl.arange(0, BT)[:, None] >= tl.arange(0, BT)[None, :] b_o = tl.zeros([BT, BV], dtype=tl.float32) @@ -973,6 +984,11 @@ def kda_gate_chunk_cumsum_vector_kernel( else: bos, eos = i_b * T, i_b * T + T + # Padded chunk_indices rows (kda_prefill_graph.py) start past T, where every + # store below is masked; this early exit only skips their wasted work. + if i_t * BT >= T: + return + p_s = tl.make_block_ptr( s + (bos * H + i_h) * S, (T, S), @@ -1145,15 +1161,16 @@ def chunk_kda_fwd( track_state: Optional[torch.Tensor] = None, track_chunk_idx: Optional[torch.Tensor] = None, beta_is_raw: bool = False, + chunk_indices: Optional[torch.Tensor] = None, + chunk_offsets: Optional[torch.Tensor] = None, ): chunk_size = 64 # Pre-compute chunk indices once and thread through all downstream kernels. - # Without this, each of the 4 callees would recompute independently. - chunk_indices = ( - prepare_chunk_indices(cu_seqlens, chunk_size) - if cu_seqlens is not None - else None - ) + # Without this, each of the 4 callees would recompute independently. A + # CUDA-graph caller passes padded chunk_indices / chunk_offsets it rebuilds + # before every replay (kda_prefill_graph.py). + if chunk_indices is None and cu_seqlens is not None: + chunk_indices = prepare_chunk_indices(cu_seqlens, chunk_size) if A_log is not None: # Fused: gate activation + chunk-local cumsum in one kernel. @@ -1227,6 +1244,7 @@ def chunk_kda_fwd( use_exp2=True, track_state=track_state, track_chunk_idx=track_chunk_idx, + chunk_offsets=chunk_offsets, ) del w, u, kg @@ -1271,6 +1289,8 @@ def chunk_kda( track_state: Optional[torch.Tensor] = None, track_chunk_idx: Optional[torch.Tensor] = None, beta_is_raw: bool = False, + chunk_indices: Optional[torch.Tensor] = None, + chunk_offsets: Optional[torch.Tensor] = None, **kwargs, ): if scale is None: @@ -1298,4 +1318,6 @@ def chunk_kda( track_state=track_state, track_chunk_idx=track_chunk_idx, beta_is_raw=beta_is_raw, + chunk_indices=chunk_indices, + chunk_offsets=chunk_offsets, ) diff --git a/python/sglang/kernels/ops/attention/fla/kda_prefill_graph_track.py b/python/sglang/kernels/ops/attention/fla/kda_prefill_graph_track.py new file mode 100644 index 0000000000..c606cff781 --- /dev/null +++ b/python/sglang/kernels/ops/attention/fla/kda_prefill_graph_track.py @@ -0,0 +1,148 @@ +"""Prefix-cache track copies for the CUDA-graph captured KDA extend. + +The eager extend selects tracked rows on the host (``nonzero`` / masked index +lists) and scatters with data-dependent shapes. Under a breakable prefill CUDA +graph the batch changes on every replay, so these kernels walk a fixed number of +rows (the bucket's sequence bound) and read a per-row mode instead: + +* ``track_mode[r] == 0``: row ``r`` is not tracked (or is padding); no-op. +* ``track_mode[r] == 1``: the tracked length is chunk aligned; the snapshot is + the post-extend state of the row's own cache slot. +* ``track_mode[r] == 2``: the snapshot is the chunk-boundary state the chunked + delta-rule kernel wrote to ``h_track[r]`` (fp32). + +Every tracked row (mode 1 or 2) also snapshots its conv window: the +``STATE_LEN`` raw pre-conv input rows ending at the tracked boundary. +""" + +import torch +import triton +import triton.language as tl + +TRACK_NONE = 0 +TRACK_FINAL_STATE = 1 +TRACK_CHUNK_STATE = 2 + + +@triton.jit +def _kda_track_conv_window_kernel( + conv_pool, # [slots, STATE_LEN, dim] + x, # [tokens, dim] raw pre-conv input + track_mode, # [rows] int32 + track_conv_start, # [rows] int32, first token row of the window (unclamped) + track_dst, # [rows] int64 physical track slot + num_tokens, # [1] int32 logical tokens of the batch + stride_pool_slot, + stride_pool_row, + stride_x_token, + dim, + STATE_LEN: tl.constexpr, + BLOCK_D: tl.constexpr, +): + r = tl.program_id(0) + mode = tl.load(track_mode + r) + if mode == 0: + return + dst = tl.load(track_dst + r).to(tl.int64) + start = tl.load(track_conv_start + r) + total = tl.load(num_tokens) + offs = tl.program_id(1) * BLOCK_D + tl.arange(0, BLOCK_D) + mask = offs < dim + for j in tl.static_range(STATE_LEN): + # Same clamp as the eager index list: clamp(0, total - 1). + t = tl.minimum(tl.maximum(start + j, 0), total - 1).to(tl.int64) + val = tl.load(x + t * stride_x_token + offs, mask=mask) + tl.store( + conv_pool + dst * stride_pool_slot + j * stride_pool_row + offs, + val.to(conv_pool.dtype.element_ty), + mask=mask, + ) + + +@triton.jit +def _kda_track_ssm_kernel( + ssm_pool, # [slots, ...] per-slot contiguous + h_track, # [rows, ...] fp32 chunk-boundary snapshots + track_mode, # [rows] int32 + track_dst, # [rows] int64 physical track slot + cache_indices, # [rows] physical cache slot of the row + stride_ssm_slot, + stride_h_row, + numel, + BLOCK: tl.constexpr, +): + r = tl.program_id(0) + mode = tl.load(track_mode + r) + if mode == 0: + return + dst = tl.load(track_dst + r).to(tl.int64) + offs = tl.program_id(1) * BLOCK + tl.arange(0, BLOCK) + mask = offs < numel + if mode == 1: + src = tl.load(cache_indices + r).to(tl.int64) + val = tl.load(ssm_pool + src * stride_ssm_slot + offs, mask=mask) + else: + val = tl.load(h_track + r.to(tl.int64) * stride_h_row + offs, mask=mask) + tl.store( + ssm_pool + dst * stride_ssm_slot + offs, + val.to(ssm_pool.dtype.element_ty), + mask=mask, + ) + + +def kda_track_conv_window( + conv_pool: torch.Tensor, + x: torch.Tensor, + track_mode: torch.Tensor, + track_conv_start: torch.Tensor, + track_dst: torch.Tensor, + num_tokens: torch.Tensor, +) -> None: + """``conv_pool[track_dst[r]] = x[clamp(start[r] + arange(STATE_LEN))]`` for + every tracked row ``r``; the grid depends only on the row bound.""" + rows = track_mode.shape[0] + state_len, dim = conv_pool.shape[-2], conv_pool.shape[-1] + assert x.shape[-1] == dim and x.stride(-1) == 1 and conv_pool.stride(-1) == 1 + block_d = 1024 + _kda_track_conv_window_kernel[(rows, triton.cdiv(dim, block_d))]( + conv_pool, + x, + track_mode, + track_conv_start, + track_dst, + num_tokens, + conv_pool.stride(0), + conv_pool.stride(1), + x.stride(0), + dim, + STATE_LEN=state_len, + BLOCK_D=block_d, + ) + + +def kda_track_ssm_state( + ssm_pool: torch.Tensor, + h_track: torch.Tensor, + track_mode: torch.Tensor, + track_dst: torch.Tensor, + cache_indices: torch.Tensor, +) -> None: + """Snapshot the tracked rows' SSM state into their track slots (mode 1: + the row's final state, mode 2: its fp32 chunk-boundary state).""" + rows = track_mode.shape[0] + numel = ssm_pool[0].numel() + assert ssm_pool[0].is_contiguous(), "per-slot SSM state must be contiguous" + assert h_track.shape[0] >= rows and h_track[0].is_contiguous() + assert h_track[0].numel() == numel + block = 1024 + _kda_track_ssm_kernel[(rows, triton.cdiv(numel, block))]( + ssm_pool, + h_track, + track_mode, + track_dst, + cache_indices, + ssm_pool.stride(0), + h_track.stride(0), + numel, + BLOCK=block, + ) diff --git a/python/sglang/kernels/ops/attention/helion/kda_prefill.py b/python/sglang/kernels/ops/attention/helion/kda_prefill.py index 5b2f1c8275..c9798205cb 100644 --- a/python/sglang/kernels/ops/attention/helion/kda_prefill.py +++ b/python/sglang/kernels/ops/attention/helion/kda_prefill.py @@ -1389,6 +1389,8 @@ def chunk_kda( track_state: torch.Tensor | None = None, track_chunk_idx: torch.Tensor | None = None, beta_is_raw: bool = False, + chunk_indices: torch.Tensor | None = None, + chunk_offsets: torch.Tensor | None = None, **kwargs: object, ) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]: """Match the public forward contract of SGLang's Triton ``chunk_kda``.""" @@ -1432,6 +1434,8 @@ def chunk_kda( output_intermediate_states=output_intermediate_states, track_state=track_state, track_chunk_idx=track_chunk_idx, + chunk_indices=chunk_indices, + chunk_offsets=chunk_offsets, ) q = q.contiguous() @@ -1441,11 +1445,8 @@ def chunk_kda( v = v.contiguous() g = g.contiguous() beta = beta.contiguous() - chunk_indices = ( - prepare_chunk_indices(cu_seqlens, CHUNK_SIZE) - if cu_seqlens is not None - else None - ) + if chunk_indices is None and cu_seqlens is not None: + chunk_indices = prepare_chunk_indices(cu_seqlens, CHUNK_SIZE) g, qg, wk, kg, chunk_decay = gate_chunk_cumsum_operands( g, q, @@ -1473,7 +1474,8 @@ def chunk_kda( is_varlen = cu_seqlens is not None if is_varlen: - chunk_offsets = prepare_chunk_offsets(cu_seqlens, CHUNK_SIZE) + if chunk_offsets is None: + chunk_offsets = prepare_chunk_offsets(cu_seqlens, CHUNK_SIZE) metadata = cu_seqlens else: metadata = torch.empty(0, device=q.device, dtype=torch.int32) diff --git a/python/sglang/kernels/ops/mamba/causal_conv1d_triton.py b/python/sglang/kernels/ops/mamba/causal_conv1d_triton.py index 9449a3a432..ce8c4f1bc9 100644 --- a/python/sglang/kernels/ops/mamba/causal_conv1d_triton.py +++ b/python/sglang/kernels/ops/mamba/causal_conv1d_triton.py @@ -15,6 +15,11 @@ from sglang.kernels.jit.utils import is_arch_support_pdl PAD_SLOT_ID = -1 +# Tokens per program of _causal_conv1d_fwd_kernel; a block_table enumerates +# (sequence, chunk) pairs in these units. +CAUSAL_CONV1D_FWD_BLOCK_M = 8 + + @triton.jit() def _causal_conv1d_fwd_kernel( # continuous batching # Pointers to matrices @@ -26,6 +31,7 @@ def _causal_conv1d_fwd_kernel( # continuous batching has_initial_states_ptr, query_start_loc_ptr, o_ptr, # (dim, seqlen) - actually pointing to x_ptr + block_table_ptr, # (num_blocks, 2) int32 (seq, chunk) rows; USE_BLOCK_TABLE only # Matrix dimensions dim: tl.constexpr, seqlen: tl.int32, # cu_seqlen @@ -55,6 +61,7 @@ def _causal_conv1d_fwd_kernel( # continuous batching NP2_STATELEN: tl.constexpr, BLOCK_M: tl.constexpr, BLOCK_N: tl.constexpr, + USE_BLOCK_TABLE: tl.constexpr = False, ): conv_states_ptr = initial_states_ptr conv_state_indices_ptr = cache_indices_ptr @@ -68,9 +75,17 @@ def _causal_conv1d_fwd_kernel( # continuous batching # one program handles one chunk in a single sequence # rather than mixing sequences - to make updating initial_states across sequences efficiently - # single-sequence id - idx_seq = tl.program_id(0) - chunk_offset = tl.program_id(1) + if USE_BLOCK_TABLE: + # CUDA-graph safe launch: the grid is a fixed upper bound on the + # (sequence, BLOCK_M-chunk) pairs of a token bucket and each program + # reads its pair from the table; padded rows carry pad_slot_id. + idx_block = tl.program_id(0) + idx_seq = tl.load(block_table_ptr + idx_block * 2) + chunk_offset = tl.load(block_table_ptr + idx_block * 2 + 1) + else: + # single-sequence id + idx_seq = tl.program_id(0) + chunk_offset = tl.program_id(1) # BLOCK_N elements along the feature-dimension (channel) idx_feats = tl.program_id(2) * BLOCK_N + tl.arange(0, BLOCK_N) @@ -402,6 +417,7 @@ def causal_conv1d_fn( activation: Optional[str] = "silu", pad_slot_id: int = PAD_SLOT_ID, validate_data=False, + block_table: Optional[torch.Tensor] = None, **kwargs, ): """support varlen + continuous batching when x is 2D tensor @@ -515,6 +531,9 @@ def causal_conv1d_fn( assert is_channel_last, "Need to run in channel-last layout" def grid(META): + if block_table is not None: + # (num_blocks, 1, feature blocks): no host-side sequence lengths. + return (block_table.shape[0], 1, triton.cdiv(dim, META["BLOCK_N"])) max_seq_len = max(seq_lens_cpu) return ( len(seq_lens_cpu), # batch_size @@ -532,6 +551,7 @@ def causal_conv1d_fn( has_initial_state, query_start_loc, out, + block_table, # Matrix dimensions dim, cu_seqlen, @@ -560,8 +580,9 @@ def causal_conv1d_fn( USE_PAD_SLOT=pad_slot_id is not None, NP2_STATELEN=np2_statelen, # launch_cooperative_grid=True - BLOCK_M=8, + BLOCK_M=CAUSAL_CONV1D_FWD_BLOCK_M, BLOCK_N=256, + USE_BLOCK_TABLE=block_table is not None, num_stages=2, ) return out diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py index 91535405fa..7283e0a280 100644 --- a/python/sglang/srt/environ.py +++ b/python/sglang/srt/environ.py @@ -1499,6 +1499,13 @@ class Envs: # Eager forward wraps the ForwardBatch's own tensors instead of copying them # into the CUDA graph buffer registry (no per-iter device-to-device copy). SGLANG_EAGER_INPUT_NO_COPY = EnvBool(False) + # Breakable prefill CUDA graphs capture the Triton KDA extend instead of + # breaking the graph at every KDA layer (layers/attention/linear/ + # kda_prefill_graph.py); this kill-switch restores the eager break. + SGLANG_DISABLE_KDA_PREFILL_GRAPH_EXTEND = EnvBool(False) + # Sequences a captured KDA extend bucket accepts; a larger prefill batch on + # any dp rank runs the step eagerly. Bounds the padded grids and scratch. + SGLANG_KDA_PREFILL_GRAPH_MAX_SEQS = EnvInt(128) # =================================================================== # Tokenizer, request state, embeddings, and reasoning controls diff --git a/python/sglang/srt/layers/attention/base_attn_backend.py b/python/sglang/srt/layers/attention/base_attn_backend.py index bcd7ec0f50..1502623ef6 100644 --- a/python/sglang/srt/layers/attention/base_attn_backend.py +++ b/python/sglang/srt/layers/attention/base_attn_backend.py @@ -152,6 +152,9 @@ class AttentionBackend(ABC): # those tensor addresses. Such backends opt in here, create the metadata # object during capture, and refresh its dynamic fields before each replay. use_captured_forward_metadata_for_breakable_cuda_graph: bool = False + # Most requests one replay of that captured metadata can carry (None: no + # bound). Fixed at init: the dp-attention replay vote checks it too. + prefill_cuda_graph_max_batch_size: Optional[int] = None # Backends may keep MIXED prefill eager under DP attention when replaying # the EXTEND graph is a known serving-performance regression. diff --git a/python/sglang/srt/layers/attention/hybrid_linear_attn_backend.py b/python/sglang/srt/layers/attention/hybrid_linear_attn_backend.py index 59bbe57901..4a424a52d0 100644 --- a/python/sglang/srt/layers/attention/hybrid_linear_attn_backend.py +++ b/python/sglang/srt/layers/attention/hybrid_linear_attn_backend.py @@ -35,6 +35,11 @@ from sglang.srt.layers.attention.mamba.replay_state_indices_validator import ( ) from sglang.srt.layers.radix_attention import RadixAttention from sglang.srt.mem_cache.memory_pool import HybridReqToTokenPool +from sglang.srt.model_executor.cuda_graph_config import ( + Backend, + Phase, + check_cuda_graph_backend, +) from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode from sglang.srt.model_executor.model_runner import ModelRunner from sglang.srt.runtime_context import get_exec, get_memory, get_spec @@ -68,6 +73,10 @@ class MambaAttnBackendBase(AttentionBackend): # else — update_mamba_state_after_mtp_verify keys the fused branch on it. accept_lens_pool: Optional[torch.Tensor] = None + # Most sequences the prefill graph tables hold (larger batches replay no + # graph); set at init by backends whose supports_prefill_graph_extend holds. + prefill_graph_max_seqs: int + def __init__(self, model_runner: ModelRunner): self.validate_mis_support(model_runner.server_args) super().__init__() @@ -109,6 +118,21 @@ class MambaAttnBackendBase(AttentionBackend): # skips the per-cycle method dispatch. self._graph_seq_len_fill_value = self.get_cuda_graph_seq_len_fill_value() + def supports_prefill_graph_extend(self) -> bool: + """Whether breakable prefill CUDA graphs can capture this backend's + extend on static per-bucket tables instead of breaking at each layer.""" + return False + + def prefill_graph_extend_active(self) -> bool: + """Whether the extend being captured or replayed runs on those tables.""" + return False + + def init_prefill_graph_metadata(self, forward_batch: ForwardBatch): + raise NotImplementedError() + + def refresh_prefill_graph_metadata(self, meta, forward_batch: ForwardBatch): + raise NotImplementedError() + @property def mamba_chunk_size(self) -> int: if self._mamba_chunk_size is None: @@ -1178,6 +1202,22 @@ class HybridLinearAttnBackend(AttentionBackend): self.extend_dummy_seqs_capped_by_req_pool = getattr( full_attn_backend, "extend_dummy_seqs_capped_by_req_pool", False ) or getattr(linear_attn_backend, "extend_dummy_seqs_capped_by_req_pool", False) + # Breakable prefill CUDA graphs capture the linear extend when the linear + # backend can run it on static tables (linear/kda_prefill_graph.py); + # those tables follow the runner's captured-metadata contract. + self.use_captured_forward_metadata_for_breakable_cuda_graph = ( + check_cuda_graph_backend(Phase.PREFILL, Backend.BREAKABLE) + and linear_attn_backend.supports_prefill_graph_extend() + and not full_attn_backend.use_captured_forward_metadata_for_breakable_cuda_graph + ) + if self.use_captured_forward_metadata_for_breakable_cuda_graph: + self.prefill_cuda_graph_max_batch_size = ( + linear_attn_backend.prefill_graph_max_seqs + ) + logger.info( + "Breakable prefill CUDA graphs capture the %s extend.", + type(linear_attn_backend).__name__, + ) @property def data_type(self): @@ -1230,6 +1270,49 @@ class HybridLinearAttnBackend(AttentionBackend): for attn_backend in self.attn_backend_list: attn_backend.init_forward_metadata_in_graph(forward_batch) + @staticmethod + def _is_plain_extend(forward_batch: ForwardBatch) -> bool: + mode = forward_batch.forward_mode + return ( + mode.is_extend() + and not mode.is_target_verify() + and not mode.is_draft_extend_v2() + ) + + def linear_extend_in_graph(self) -> bool: + """Whether the linear layers of the prefill being captured run inside + the graph (no eager break).""" + return self.linear_attn_backend.prefill_graph_extend_active() + + def init_forward_metadata_for_breakable_cuda_graph_capture( + self, forward_batch: ForwardBatch + ): + self.full_attn_backend.init_forward_metadata(forward_batch) + if not self._is_plain_extend(forward_batch): + if not forward_batch.forward_mode.is_draft_extend_v2(): + self.linear_attn_backend.init_forward_metadata(forward_batch) + return None + return self.linear_attn_backend.init_prefill_graph_metadata(forward_batch) + + def prepare_forward_metadata_for_breakable_cuda_graph_replay( + self, + capture_metadata, + forward_batch: ForwardBatch, + *, + static_forward_batch: Optional[ForwardBatch] = None, + ) -> None: + self.full_attn_backend.init_forward_metadata(forward_batch) + if capture_metadata is not None: + self.linear_attn_backend.refresh_prefill_graph_metadata( + capture_metadata, forward_batch + ) + elif not forward_batch.forward_mode.is_draft_extend_v2(): + self.linear_attn_backend.init_forward_metadata(forward_batch) + if static_forward_batch is not None: + self.prepare_prefill_shared_read_snapshot( + forward_batch, num_qo_tokens=static_forward_batch.positions.shape[0] + ) + def get_indexer_metadata(self, layer_id: int, forward_batch: ForwardBatch): if layer_id in self.full_attn_layers: return self.full_attn_backend.get_indexer_metadata(layer_id, forward_batch) diff --git a/python/sglang/srt/layers/attention/linear/kda_backend.py b/python/sglang/srt/layers/attention/linear/kda_backend.py index c14c199233..a542ee3543 100644 --- a/python/sglang/srt/layers/attention/linear/kda_backend.py +++ b/python/sglang/srt/layers/attention/linear/kda_backend.py @@ -10,6 +10,12 @@ from sglang.kernels.ops.mamba.causal_conv1d_triton import ( ) from sglang.srt.environ import envs from sglang.srt.layers.attention.hybrid_linear_attn_backend import MambaAttnBackendBase +from sglang.srt.layers.attention.linear.kda_prefill_graph import ( + KDAPrefillGraphMetadata, + KDAPrefillTrackRows, + build_track_rows, + kda_extend_in_graph, +) from sglang.srt.layers.attention.linear.kernels.kda_flashinfer import ( build_fused_accept_indices, ) @@ -43,6 +49,7 @@ from sglang.srt.runtime_context import ( get_memory, get_platform, get_spec, + mamba_cache_chunk_size, ) @@ -454,8 +461,13 @@ class KDAAttnBackend(MambaAttnBackendBase): # force-flush path. needs_cpu_seq_lens: bool = False + # Set while a breakable prefill CUDA graph captures or replays a bucket: + # forward_extend then runs the graph-safe chain on these static tables. + prefill_graph_metadata: Optional[KDAPrefillGraphMetadata] = None + def __init__(self, model_runner: ModelRunner): super().__init__(model_runner) + self.prefill_graph_max_seqs = envs.SGLANG_KDA_PREFILL_GRAPH_MAX_SEQS.get() # Needed by the extra_buffer track path: _init_track_conv_indices reads # conv_states_shape[-1] as the conv window length (kernel_size - 1). # The KDA pool stores conv states as [kernel-1, dim] — transposed vs @@ -590,7 +602,110 @@ class KDAAttnBackend(MambaAttnBackendBase): metadata.fused_accept_num_accepted, ) + def supports_prefill_graph_extend(self) -> bool: + return ( + not envs.SGLANG_DISABLE_KDA_PREFILL_GRAPH_EXTEND.get() + # Only the Triton chain guards padded rows (-1 state slots, chunks past T). + and isinstance(self.kernel_dispatcher.extend_kernel, TritonKDAKernel) + # Fused-accept staging (flashinfer verify) writes per extend. + and self.accept_lens_pool is None + ) + + def prefill_graph_extend_active(self) -> bool: + return self.prefill_graph_metadata is not None + + def init_prefill_graph_metadata( + self, forward_batch: ForwardBatch + ) -> KDAPrefillGraphMetadata: + cache_indices = self._prefill_graph_cache_indices(forward_batch) + meta = KDAPrefillGraphMetadata.allocate( + num_tokens=forward_batch.positions.shape[0], + max_seqs_cap=self.prefill_graph_max_seqs, + # Any replay of this bucket may carry prefix-cache snapshots (the + # extra buffer is only on with the radix cache). + track=get_exec().mamba.enable_mamba_extra_buffer, + cache_index_dtype=cache_indices.dtype, + device=self.device, + ) + self._fill_prefill_graph_metadata( + meta=meta, forward_batch=forward_batch, cache_indices=cache_indices + ) + return meta + + def refresh_prefill_graph_metadata( + self, meta: KDAPrefillGraphMetadata, forward_batch: ForwardBatch + ) -> None: + self._fill_prefill_graph_metadata( + meta=meta, + forward_batch=forward_batch, + cache_indices=self._prefill_graph_cache_indices(forward_batch), + ) + + def _prefill_graph_cache_indices(self, forward_batch: ForwardBatch): + # Same slot resolution as MambaAttnBackendBase._forward_metadata. + cache_indices = self._translate_mamba_indices( + self.req_to_token_pool.get_mamba_indices(forward_batch.req_pool_indices) + ) + real_bs = forward_batch._original_batch_size + if real_bs is not None and real_bs < cache_indices.shape[0]: + cache_indices = cache_indices.clone() + cache_indices[real_bs:] = -1 + return cache_indices + + def _fill_prefill_graph_metadata( + self, + *, + meta: KDAPrefillGraphMetadata, + forward_batch: ForwardBatch, + cache_indices: torch.Tensor, + ) -> None: + if forward_batch.mamba_track_indices is not None: + # Same one-time in-place translation as the eager metadata path. + forward_batch.mamba_track_indices = self._translate_mamba_indices( + forward_batch.mamba_track_indices + ) + track_rows = None + if forward_batch.mamba_track_mask is not None: + assert meta.track, ( + "prefix-cache track batch replayed on a bucket captured without " + "track tables" + ) + track_rows = self._prefill_graph_track_rows(forward_batch) + meta.fill( + extend_seq_lens=list(forward_batch.extend_seq_lens_cpu), + cache_indices=cache_indices, + extend_prefix_lens=forward_batch.extend_prefix_lens, + track_rows=track_rows, + ) + self.prefill_graph_metadata = meta + + def _prefill_graph_track_rows( + self, forward_batch: ForwardBatch + ) -> Optional[KDAPrefillTrackRows]: + bs = forward_batch.batch_size + if self._has_cpu_prefill_track_metadata(forward_batch): + track_mask = list(forward_batch.mamba_prefill_track_mask_cpu) + track_seq_lens = list(forward_batch.mamba_track_seqlens_cpu) + extend_prefix_lens = list(forward_batch.extend_prefix_lens_cpu) + else: + # The same host copies the eager path takes without CPU metadata. + track_mask = forward_batch.mamba_track_mask[:bs].tolist() + track_seq_lens = forward_batch.mamba_track_seqlens[:bs].tolist() + extend_prefix_lens = forward_batch.extend_prefix_lens[:bs].tolist() + return build_track_rows( + track_mask=track_mask, + track_seq_lens=track_seq_lens, + extend_seq_lens=list(forward_batch.extend_seq_lens_cpu), + extend_prefix_lens=extend_prefix_lens, + state_chunk_size=self.mamba_chunk_size, + cache_chunk_size=mamba_cache_chunk_size(), + conv_len=self.conv_states_shape[-1], + dst=forward_batch.mamba_track_indices, + ) + def init_forward_metadata(self, forward_batch: ForwardBatch): + # An eager forward never runs on a graph bucket's tables. + self.prefill_graph_metadata = None super().init_forward_metadata(forward_batch) if self.forward_metadata.has_mamba_track_mask: if self.forward_metadata.mamba_track_mask_indices is None: @@ -908,6 +1023,21 @@ class KDAAttnBackend(MambaAttnBackendBase): if forward_batch.forward_mode.is_target_verify(): return self._forward_target_verify(layer, forward_batch, mixed_qkv, a, b) + if self.prefill_graph_metadata is not None: + # Breakable prefill CUDA graph bucket: host-free chain on static + # tables refreshed before each replay. + layer_cache = self.req_to_token_pool.mamba2_layer_cache(layer.layer_id) + return kda_extend_in_graph( + layer=layer, + meta=self.prefill_graph_metadata, + conv_pool=layer_cache.conv[0], + ssm_states=layer_cache.temporal, + kernel_dispatcher=self.kernel_dispatcher, + mixed_qkv=mixed_qkv, + a=a, + b=b, + ) + query_start_loc = self.forward_metadata.query_start_loc cache_indices = self.forward_metadata.mamba_cache_indices diff --git a/python/sglang/srt/layers/attention/linear/kda_prefill_graph.py b/python/sglang/srt/layers/attention/linear/kda_prefill_graph.py new file mode 100644 index 0000000000..636d6d9c77 --- /dev/null +++ b/python/sglang/srt/layers/attention/linear/kda_prefill_graph.py @@ -0,0 +1,389 @@ +"""KDA extend captured inside breakable prefill CUDA graphs. + +A breakable prefill CUDA graph (BCG) is captured once per token bucket and +replayed for any batch that pads into that bucket. The eager KDA extend derives +its launch grids and index tables from host-side sequence lengths +(``prepare_chunk_indices``, ``causal_conv1d_fn``'s ``seq_lens_cpu``, the +prefix-cache track plan), so without this module BCG breaks the graph at every +KDA layer and runs the whole chain -- about sixteen kernels per layer plus their +Python -- eagerly. + +For each bucket this module allocates static tables sized by the bucket's +bounds: + +* ``max_seqs`` sequences (``min(bucket, cap)``; batches with more sequences run + eagerly, through the runner's replay vote on + ``AttentionBackend.prefill_cuda_graph_max_batch_size``), +* ``max_chunks = cdiv(bucket, 64) + max_seqs`` delta-rule chunks, +* ``max_conv_blocks = cdiv(bucket, 8) + max_seqs`` conv blocks. + +Before every replay ``fill`` rebuilds the tables from the batch's CPU lengths +(one host-to-device copy plus a few device gathers, no synchronization). The +captured kernels launch fixed grids over the bounds; padded rows point past +their sequence (chunk tables) or carry ``PAD_SLOT_ID`` (conv blocks, state +slots) and return immediately. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Optional + +import msgspec +import numpy as np +import torch +import triton + +from sglang.kernels.ops.attention.fla.kda_prefill_graph_track import ( + TRACK_CHUNK_STATE, + TRACK_FINAL_STATE, + kda_track_conv_window, + kda_track_ssm_state, +) +from sglang.kernels.ops.mamba.causal_conv1d_triton import ( + CAUSAL_CONV1D_FWD_BLOCK_M, + PAD_SLOT_ID, + causal_conv1d_fn, +) +from sglang.srt.layers.attention.mamba.prefill_track_metadata import ( + build_prefill_track_plan, +) + +if TYPE_CHECKING: + from sglang.srt.layers.attention.linear.kda_backend import KDAKernelDispatcher + from sglang.srt.layers.radix_linear_attention import RadixLinearAttention + +# Delta-rule chunk size of chunk_kda (fla/kda.py, fla/chunk_delta_h.py). +KDA_CHUNK_SIZE = 64 +# Chunk index of padded chunk_indices rows: past the end of any sequence, so +# every varlen chunk kernel returns on its `i_t * BT >= T` guard. +_PAD_CHUNK = 1 << 20 + + +class KDAPrefillTrackRows(msgspec.Struct, frozen=True): + """One batch's prefix-cache track plan, in batch rows.""" + + # (row, chunk): the snapshot is the state at that chunk boundary. + chunk_rows: list[tuple[int, int]] + # Rows whose snapshot is the post-extend state (chunk-aligned length). + final_rows: list[int] + # (row, aligned_len) of every tracked row: its conv window ends there. + conv_rows: list[tuple[int, int]] + # Conv window length (kernel_size - 1). + conv_len: int + # [bs] physical track slots, on the device. + dst: torch.Tensor + + +class KDAPrefillGraphMetadata(msgspec.Struct, frozen=True): + """Static per-bucket tables read by the captured KDA extend.""" + + num_tokens: int + max_seqs: int + max_chunks: int + max_conv_blocks: int + # Allocates and records the prefix-cache snapshot copies. + track: bool + + # One int32 device buffer, refreshed by a single copy, and its views. + packed: torch.Tensor + cu_seqlens: torch.Tensor # [max_seqs + 1] + chunk_indices: torch.Tensor # [max_chunks, 2] (seq, chunk) + chunk_offsets: torch.Tensor # [max_seqs + 1] + conv_block_table: torch.Tensor # [max_conv_blocks, 2] (seq, block) + track_chunk_idx: torch.Tensor # [max_seqs] + track_mode: torch.Tensor # [max_seqs] + track_conv_start: torch.Tensor # [max_seqs] + num_logical_tokens: torch.Tensor # [1] + host: np.ndarray # int32 staging for `packed` + + # Filled on the device from the batch. + cache_indices: torch.Tensor # [max_seqs], PAD_SLOT_ID past the batch + has_initial_state: torch.Tensor # [max_seqs] bool + track_dst: torch.Tensor # [max_seqs] int64 + token_index: torch.Tensor # [num_tokens] int32 arange + + @classmethod + def allocate( + cls, + *, + num_tokens: int, + max_seqs_cap: int, + track: bool, + cache_index_dtype: torch.dtype, + device: torch.device, + ) -> KDAPrefillGraphMetadata: + max_seqs = max(1, min(num_tokens, max_seqs_cap)) + max_chunks = triton.cdiv(num_tokens, KDA_CHUNK_SIZE) + max_seqs + max_conv_blocks = triton.cdiv(num_tokens, CAUSAL_CONV1D_FWD_BLOCK_M) + max_seqs + sizes = [ + max_seqs + 1, # cu_seqlens + 2 * max_chunks, # chunk_indices + max_seqs + 1, # chunk_offsets + 2 * max_conv_blocks, # conv_block_table + max_seqs, # track_chunk_idx + max_seqs, # track_mode + max_seqs, # track_conv_start + 1, # num_logical_tokens + ] + packed = torch.zeros(sum(sizes), dtype=torch.int32, device=device) + views = torch.split(packed, sizes) + return cls( + num_tokens=num_tokens, + max_seqs=max_seqs, + max_chunks=max_chunks, + max_conv_blocks=max_conv_blocks, + track=track, + packed=packed, + cu_seqlens=views[0], + chunk_indices=views[1].view(max_chunks, 2), + chunk_offsets=views[2], + conv_block_table=views[3].view(max_conv_blocks, 2), + track_chunk_idx=views[4], + track_mode=views[5], + track_conv_start=views[6], + num_logical_tokens=views[7], + host=np.zeros(sum(sizes), dtype=np.int32), + cache_indices=torch.full( + (max_seqs,), PAD_SLOT_ID, dtype=cache_index_dtype, device=device + ), + has_initial_state=torch.zeros(max_seqs, dtype=torch.bool, device=device), + track_dst=torch.zeros(max_seqs, dtype=torch.int64, device=device), + token_index=torch.arange(num_tokens, dtype=torch.int32, device=device), + ) + + def can_fit(self, extend_seq_lens: list[int]) -> bool: + return ( + len(extend_seq_lens) <= self.max_seqs + and sum(extend_seq_lens) <= self.num_tokens + ) + + def fill( + self, + *, + extend_seq_lens: list[int], + cache_indices: torch.Tensor, + extend_prefix_lens: torch.Tensor, + track_rows: Optional[KDAPrefillTrackRows], + ) -> None: + """Rebuild the tables for this batch. ``cache_indices`` and + ``extend_prefix_lens`` are the batch's device tensors.""" + bs = len(extend_seq_lens) + assert self.can_fit(extend_seq_lens), ( + f"batch of {bs} sequences / {sum(extend_seq_lens)} tokens exceeds the " + f"captured bounds ({self.max_seqs} / {self.num_tokens})" + ) + self._write_host_tables( + lens=np.asarray(extend_seq_lens, dtype=np.int64), track_rows=track_rows + ) + + # A pinned block from the caching host allocator keeps the copy + # asynchronous and is not reused before the copy completes. + self.packed.copy_(torch.from_numpy(self.host).pin_memory(), non_blocking=True) + self.cache_indices[:bs].copy_(cache_indices[:bs]) + self.cache_indices[bs:].fill_(PAD_SLOT_ID) + torch.gt(extend_prefix_lens[:bs], 0, out=self.has_initial_state[:bs]) + self.has_initial_state[bs:].fill_(False) + if track_rows is not None: + self.track_dst[:bs].copy_(track_rows.dst[:bs]) + + def _write_host_tables( + self, *, lens: np.ndarray, track_rows: Optional[KDAPrefillTrackRows] + ) -> None: + """Write every host-built table into ``self.host`` in ``packed`` order.""" + bs = lens.shape[0] + total = int(lens.sum()) + host = self.host + + # Padded sequences are empty and sit at the end. + cu = np.empty(self.max_seqs + 1, dtype=np.int64) + cu[0] = 0 + np.cumsum(lens, out=cu[1 : bs + 1]) + cu[bs + 1 :] = total + o = self._write(o=0, values=cu) + + # Sequence-major (seq, chunk) pairs: chunk_gla_fwd_o reads the per-chunk + # states by row, chunk_gated_delta_rule_fwd_h writes them at + # chunk_offsets[seq] + chunk. + n_chunks = (lens + KDA_CHUNK_SIZE - 1) // KDA_CHUNK_SIZE + o = self._write_pairs( + o=o, counts=n_chunks, rows=self.max_chunks, pad=(0, _PAD_CHUNK) + ) + offsets = np.empty(self.max_seqs + 1, dtype=np.int64) + offsets[0] = 0 + np.cumsum(n_chunks, out=offsets[1 : bs + 1]) + offsets[bs + 1 :] = offsets[bs] + o = self._write(o=o, values=offsets) + + n_blocks = (lens + CAUSAL_CONV1D_FWD_BLOCK_M - 1) // CAUSAL_CONV1D_FWD_BLOCK_M + o = self._write_pairs( + o=o, counts=n_blocks, rows=self.max_conv_blocks, pad=(PAD_SLOT_ID, 0) + ) + + track_chunk_idx = np.full(self.max_seqs, -1, dtype=np.int64) + track_mode = np.zeros(self.max_seqs, dtype=np.int64) + track_conv_start = np.zeros(self.max_seqs, dtype=np.int64) + if track_rows is not None: + for row, chunk in track_rows.chunk_rows: + track_chunk_idx[row] = chunk + track_mode[row] = TRACK_CHUNK_STATE + for row in track_rows.final_rows: + track_mode[row] = TRACK_FINAL_STATE + for row, aligned_len in track_rows.conv_rows: + track_conv_start[row] = cu[row] + aligned_len - track_rows.conv_len + o = self._write(o=o, values=track_chunk_idx) + o = self._write(o=o, values=track_mode) + o = self._write(o=o, values=track_conv_start) + o = self._write(o=o, values=np.array([total])) + assert o == host.shape[0] + + def _write(self, *, o: int, values: np.ndarray) -> int: + self.host[o : o + values.shape[0]] = values + return o + values.shape[0] + + def _write_pairs( + self, *, o: int, counts: np.ndarray, rows: int, pad: tuple[int, int] + ) -> int: + """Write (seq, index) rows, `counts[seq]` indices per sequence, into a + [rows, 2] table at `o`, padding the tail with `pad`.""" + n = int(counts.sum()) + table = self.host[o : o + 2 * rows].reshape(rows, 2) + if n: + table[:n, 0] = np.repeat(np.arange(counts.shape[0]), counts) + table[:n, 1] = np.arange(n) - np.repeat(np.cumsum(counts) - counts, counts) + table[n:, 0] = pad[0] + table[n:, 1] = pad[1] + return o + 2 * rows + + +def build_track_rows( + *, + track_mask: list[bool], + track_seq_lens: list[int], + extend_seq_lens: list[int], + extend_prefix_lens: list[int], + state_chunk_size: int, + cache_chunk_size: int, + conv_len: int, + dst: torch.Tensor, +) -> Optional[KDAPrefillTrackRows]: + """The eager track selection (_init_track_ssm_indices_from_cpu and + _init_track_conv_indices) as per-row lists; None when nothing is tracked.""" + if not any(track_mask): + return None + plan = build_prefill_track_plan( + track_mask, + track_seq_lens, + extend_seq_lens, + extend_prefix_lens, + state_chunk_size, + mamba2=False, + ) + conv_rows = [ + ( + row, + (track_seq_lens[row] - extend_prefix_lens[row]) + // cache_chunk_size + * cache_chunk_size, + ) + for row in plan.tracked_rows + ] + return KDAPrefillTrackRows( + chunk_rows=[(row, plan.chunk_indices[row]) for row in plan.unaligned_rows], + final_rows=list(plan.final_rows), + conv_rows=conv_rows, + conv_len=conv_len, + dst=dst, + ) + + +def kda_extend_in_graph( + *, + layer: RadixLinearAttention, + meta: KDAPrefillGraphMetadata, + conv_pool: torch.Tensor, + ssm_states: torch.Tensor, + kernel_dispatcher: KDAKernelDispatcher, + mixed_qkv: torch.Tensor, + a: torch.Tensor, + b: torch.Tensor, +) -> torch.Tensor: + """KDAAttnBackend.forward_extend with launch grids that depend only on the + bucket. ``conv_pool`` is the layer's [slots, kernel-1, dim] conv cache. + Returns [1, num_tokens, H, V]; rows past the batch's tokens are zero.""" + if meta.track: + # Snapshot the conv window before the conv kernel updates the cache; + # the source is the raw input, the destinations are track slots. + kda_track_conv_window( + conv_pool=conv_pool, + x=mixed_qkv, + track_mode=meta.track_mode, + track_conv_start=meta.track_conv_start, + track_dst=meta.track_dst, + num_tokens=meta.num_logical_tokens, + ) + + qkv = causal_conv1d_fn( + mixed_qkv.transpose(0, 1), + layer.conv_weights, + layer.bias, + activation="silu", + conv_states=conv_pool.transpose(-1, -2), + has_initial_state=meta.has_initial_state, + cache_indices=meta.cache_indices, + query_start_loc=meta.cu_seqlens, + seq_lens_cpu=None, + block_table=meta.conv_block_table, + ).transpose(0, 1) + q, k, v = qkv.split([layer.q_dim, layer.k_dim, layer.v_dim], dim=-1) + q = q.unflatten(-1, (-1, layer.head_q_dim)).unsqueeze(0) + k = k.unflatten(-1, (-1, layer.head_k_dim)).unsqueeze(0) + v = v.unflatten(-1, (-1, layer.head_v_dim)).unsqueeze(0) + + gate_was_flat = a.ndim == 3 + if gate_was_flat: + a = a.unflatten(-1, (-1, layer.head_k_dim)) + + h_track = None + if meta.track: + h_track = torch.empty( + (meta.max_seqs, *ssm_states.shape[1:]), + dtype=torch.float32, + device=ssm_states.device, + ) + core_attn_out = kernel_dispatcher.extend( + q=q, + k=k, + v=v, + g=a, + beta=b, + ssm_states=ssm_states, + cache_indices=meta.cache_indices, + query_start_loc=meta.cu_seqlens, + A_log=layer.A_log, + dt_bias=layer.dt_bias, + lower_bound=layer.lower_bound, + beta_is_raw=gate_was_flat, + extend_seq_lens_cpu=None, + is_spec_decode=False, + return_intermediate_states=False, + track_state=h_track, + track_chunk_idx=meta.track_chunk_idx if meta.track else None, + chunk_indices=meta.chunk_indices, + chunk_offsets=meta.chunk_offsets, + ) + + if meta.track: + kda_track_ssm_state( + ssm_pool=ssm_states, + h_track=h_track, + track_mode=meta.track_mode, + track_dst=meta.track_dst, + cache_indices=meta.cache_indices, + ) + + # Rows past the batch belong to no sequence and were never written; keep the + # layer's padded output finite for the MoE and the collectives. + core_attn_out.masked_fill_( + (meta.token_index >= meta.num_logical_tokens).view(1, -1, 1, 1), 0 + ) + return core_attn_out diff --git a/python/sglang/srt/layers/attention/linear/kernels/kda_triton.py b/python/sglang/srt/layers/attention/linear/kernels/kda_triton.py index e0e807aa47..c035de4ac9 100644 --- a/python/sglang/srt/layers/attention/linear/kernels/kda_triton.py +++ b/python/sglang/srt/layers/attention/linear/kernels/kda_triton.py @@ -251,4 +251,6 @@ class TritonKDAKernel(LinearAttnKernelBase): output_intermediate_states=return_intermediate_states, track_state=kwargs.get("track_state"), track_chunk_idx=kwargs.get("track_chunk_idx"), + chunk_indices=kwargs.get("chunk_indices"), + chunk_offsets=kwargs.get("chunk_offsets"), ) diff --git a/python/sglang/srt/layers/radix_linear_attention.py b/python/sglang/srt/layers/radix_linear_attention.py index 39201effbe..3f0333772c 100644 --- a/python/sglang/srt/layers/radix_linear_attention.py +++ b/python/sglang/srt/layers/radix_linear_attention.py @@ -93,7 +93,7 @@ class RadixLinearAttention(nn.Module): dtype=mixed_qkv.dtype, device=mixed_qkv.device, ) - if is_in_breakable_cuda_graph(): + if is_in_breakable_cuda_graph() and not _linear_extend_in_graph(): bcg_unified_linear_attention_with_output( mixed_qkv, a, @@ -148,6 +148,20 @@ class RadixLinearAttention(nn.Module): ) +def _linear_extend_in_graph() -> bool: + """Whether the backend runs this breakable prefill capture's linear extend + on static tables inside the graph (no eager break at the layer).""" + from sglang.srt.layers.attention.hybrid_linear_attn_backend import ( + HybridLinearAttnBackend, + ) + + backend = get_attn_backend() + return ( + isinstance(backend, HybridLinearAttnBackend) + and backend.linear_extend_in_graph() + ) + + def _linear_attention_with_output_impl( mixed_qkv: torch.Tensor, a: torch.Tensor, diff --git a/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py b/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py index 552b735283..916ecc2eb1 100644 --- a/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py +++ b/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py @@ -289,6 +289,7 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner): _use_draft_input_embeds = False _backend_can_run_prefill_cuda_graph = None + _captured_attn_metadata_max_bs: Optional[int] = None dllm_attention = None def __init__(self, model_runner: ModelRunner): @@ -612,6 +613,11 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner): self.use_captured_attn_metadata = model_runner.attn_backend.use_captured_forward_metadata_for_breakable_cuda_graph else: self.use_captured_attn_metadata = False + self._captured_attn_metadata_max_bs = ( + model_runner.attn_backend.prefill_cuda_graph_max_batch_size + if self.use_captured_attn_metadata + else None + ) self.attn_metadata_buffers: Optional[Dict[ShapeKey, object]] = ( {} if self.use_captured_attn_metadata else None ) @@ -1274,6 +1280,11 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner): return False if self._is_full_backend and batch_size > self._capture_req_slots: return False + if ( + self._captured_attn_metadata_max_bs is not None + and batch_size > self._captured_attn_metadata_max_bs + ): + return False # LoRA replays need prepare_lora_batch's static metadata. lora_manager # keeps LoRA prefill eager on every rank under dp attention, so the # schedule-time vote derives this from enable_lora alone. diff --git a/test/registered/kernels/ops/attention/test_kda_prefill_graph.py b/test/registered/kernels/ops/attention/test_kda_prefill_graph.py new file mode 100644 index 0000000000..098ee3c208 --- /dev/null +++ b/test/registered/kernels/ops/attention/test_kda_prefill_graph.py @@ -0,0 +1,273 @@ +"""The CUDA-graph captured KDA extend (linear/kda_prefill_graph.py) against the +eager KDAAttnBackend.forward_extend. + +One bucket is captured once, then replayed for batches of different shapes: +several sequences, cached prefixes, padding past the batch, chunk-aligned and +unaligned prefix-cache snapshots, and a batch without tracking. Each replay must +match the eager path bit for bit on the output, the conv / SSM state pools and +the track slots, and must leave every other pool slot untouched. +""" + +import unittest +from types import SimpleNamespace +from unittest.mock import patch + +import torch + +from sglang.srt.layers.attention.linear.kda_backend import ( + KDAAttnBackend, + KDAKernelDispatcher, +) +from sglang.srt.layers.attention.linear.kda_prefill_graph import KDAPrefillGraphMetadata +from sglang.srt.layers.attention.linear.utils import LinearAttnKernelBackend +from sglang.srt.model_executor.forward_batch_info import ForwardMode +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.test_utils import CustomTestCase + +register_cuda_ci(est_time=60, stage="base-b-kernel-unit", runner_config="1-gpu-large") + +HEADS, HEAD_DIM, CONV = 2, 128, 4 +DIM = 3 * HEADS * HEAD_DIM +BUCKET, MAX_SEQS = 512, 16 +NUM_SLOTS = 48 +CHUNK = 64 + + +def _layer(): + g = torch.Generator(device="cuda").manual_seed(7) + + def randn(*shape, dtype=torch.bfloat16, scale=0.2): + return (torch.randn(*shape, generator=g, device="cuda") * scale).to(dtype) + + return SimpleNamespace( + layer_id=0, + num_q_heads=HEADS, + num_k_heads=HEADS, + num_v_heads=HEADS, + head_q_dim=HEAD_DIM, + head_k_dim=HEAD_DIM, + head_v_dim=HEAD_DIM, + q_dim=HEADS * HEAD_DIM, + k_dim=HEADS * HEAD_DIM, + v_dim=HEADS * HEAD_DIM, + conv_weights=randn(DIM, CONV), + bias=randn(DIM), + A_log=randn(1, 1, HEADS, 1, dtype=torch.float32), + dt_bias=randn(HEADS * HEAD_DIM, dtype=torch.float32), + lower_bound=-5.0, + ) + + +def _backend(conv_pool, ssm_pool): + """The production backend on test pools: slot of request i is i + 1.""" + backend = KDAAttnBackend.__new__(KDAAttnBackend) + cache = SimpleNamespace(conv=[conv_pool], temporal=ssm_pool) + backend.req_to_token_pool = SimpleNamespace( + mamba2_layer_cache=lambda layer_id: cache, + get_mamba_indices=lambda req_pool_indices: (req_pool_indices + 1).to( + torch.int32 + ), + translate_mamba_indices=lambda indices: indices, + ) + backend.device = torch.device("cuda") + backend._mamba_chunk_size = CHUNK + backend.conv_states_shape = conv_pool.transpose(-1, -2).shape + backend.accept_lens_pool = None + backend.prefill_graph_metadata = None + backend.kernel_dispatcher = KDAKernelDispatcher( + LinearAttnKernelBackend.TRITON, + LinearAttnKernelBackend.TRITON, + LinearAttnKernelBackend.TRITON, + ) + return backend + + +def _batch(extend_lens, prefix_lens, track=None): + """A ForwardBatch stand-in. ``track[row] = tracked length`` (from the + prefix) marks prefix-cache snapshot rows; their track slots follow the + cache slots (NUM_SLOTS // 2 + row).""" + bs = len(extend_lens) + dev = "cuda" + starts = [sum(extend_lens[:i]) for i in range(bs)] + batch = SimpleNamespace( + forward_mode=ForwardMode.EXTEND, + batch_size=bs, + _original_batch_size=None, + req_pool_indices=torch.arange(bs, device=dev), + extend_seq_lens=torch.tensor(extend_lens, device=dev), + extend_prefix_lens=torch.tensor(prefix_lens, device=dev), + extend_start_loc=torch.tensor(starts, device=dev), + extend_seq_lens_cpu=list(extend_lens), + extend_prefix_lens_cpu=list(prefix_lens), + tbo_parent_token_range=None, + mamba_track_mask=None, + mamba_track_indices=None, + mamba_track_seqlens=None, + mamba_prefill_track_mask_cpu=None, + mamba_track_seqlens_cpu=None, + ) + if track is not None: + mask = [row in track for row in range(bs)] + track_lens = [prefix_lens[r] + track.get(r, 0) for r in range(bs)] + batch.mamba_prefill_track_mask_cpu = mask + batch.mamba_track_seqlens_cpu = track_lens + batch.mamba_track_mask = torch.tensor(mask, device=dev) + batch.mamba_track_seqlens = torch.tensor(track_lens, device=dev) + batch.mamba_track_indices = torch.arange( + NUM_SLOTS // 2, NUM_SLOTS // 2 + bs, device=dev, dtype=torch.int32 + ) + + def aligned_lens(): + lens = batch.mamba_track_seqlens - batch.extend_prefix_lens + return (lens // CHUNK) * CHUNK + + batch.mamba_track_aligned_lens = aligned_lens + return batch + + +class TestKDAPrefillGraph(CustomTestCase): + def setUp(self): + self.layer = _layer() + g = torch.Generator(device="cuda").manual_seed(11) + self.conv_pool = ( + torch.randn(NUM_SLOTS, CONV - 1, DIM, generator=g, device="cuda") * 0.3 + ).to(torch.bfloat16) + self.ssm_pool = ( + torch.randn( + NUM_SLOTS, HEADS, HEAD_DIM, HEAD_DIM, generator=g, device="cuda" + ) + * 0.05 + ) + self.backend = _backend(self.conv_pool, self.ssm_pool) + # Static inputs the captured graph reads. + self.mixed = torch.zeros(BUCKET, DIM, device="cuda", dtype=torch.bfloat16) + self.a = torch.zeros(1, BUCKET, HEADS * HEAD_DIM, device="cuda") + self.a = self.a.to(torch.bfloat16) + self.b = torch.zeros(1, BUCKET, HEADS, device="cuda", dtype=torch.bfloat16) + + def _fill(self, meta, batch): + with patch( + "sglang.srt.layers.attention.linear.kda_backend.mamba_cache_chunk_size", + return_value=CHUNK, + ): + self.backend._fill_prefill_graph_metadata( + meta=meta, + forward_batch=batch, + cache_indices=self.backend._prefill_graph_cache_indices(batch), + ) + + def _capture(self): + meta = KDAPrefillGraphMetadata.allocate( + num_tokens=BUCKET, + max_seqs_cap=MAX_SEQS, + track=True, + cache_index_dtype=torch.int32, + device=torch.device("cuda"), + ) + capture_batch = _batch([BUCKET], [0]) + self._fill(meta, capture_batch) + snapshot = (self.conv_pool.clone(), self.ssm_pool.clone()) + + def run(): + return self.backend.forward_extend( + self.layer, capture_batch, self.mixed, self.a, self.b + ) + + stream = torch.cuda.Stream() + stream.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(stream): + run() # warmup: Triton compile / autotune outside the capture + torch.cuda.current_stream().wait_stream(stream) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + out = run() + # Warmup and capture ran on the pools; restore them. + self.conv_pool.copy_(snapshot[0]) + self.ssm_pool.copy_(snapshot[1]) + return meta, graph, out + + def _check_replay(self, meta, graph, out, *, lens, prefixes, track, seed): + batch = _batch(lens, prefixes, track=track) + total = sum(batch.extend_seq_lens_cpu) + g = torch.Generator(device="cuda").manual_seed(seed) + # Fresh requests start from zeroed slots (the pool clears on alloc). + for row, prefix in enumerate(batch.extend_prefix_lens_cpu): + if prefix == 0: + self.conv_pool[row + 1].zero_() + self.ssm_pool[row + 1].zero_() + # Garbage past the batch must not leak into anything. + self.mixed.copy_(torch.randn(BUCKET, DIM, generator=g, device="cuda")) + self.a.copy_( + torch.randn(1, BUCKET, HEADS * HEAD_DIM, generator=g, device="cuda") + ) + self.b.copy_(torch.randn(1, BUCKET, HEADS, generator=g, device="cuda")) + + # Eager reference on cloned pools. + ref_conv, ref_ssm = self.conv_pool.clone(), self.ssm_pool.clone() + ref_backend = _backend(ref_conv, ref_ssm) + ref_batch = _batch(lens, prefixes, track=track) + ref_backend.init_forward_metadata(ref_batch) + ref_out = ref_backend.forward_extend( + self.layer, + ref_batch, + self.mixed[:total], + self.a[:, :total], + self.b[:, :total], + ) + + self._fill(meta, batch) + graph.replay() + torch.cuda.synchronize() + + self.assertTrue(torch.equal(out[:, :total], ref_out[:, :total])) + self.assertTrue(bool((out[:, total:] == 0).all()), "padded rows not zeroed") + self.assertTrue(torch.equal(self.conv_pool, ref_conv), "conv pool differs") + self.assertTrue(torch.equal(self.ssm_pool, ref_ssm), "ssm pool differs") + + def test_replays_match_eager(self): + meta, graph, out = self._capture() + cases = [ + # (extend lens, prefix lens, track {row: tracked length from prefix}). + # Tracked lengths are >= one chunk and within the row's extend, as + # the scheduler sets them; a chunk multiple snapshots the final state. + ([37, 100, 5, 192], [0, 64, 130, 0], {1: 70, 3: 192}), + ([BUCKET], [256], {0: 300}), + ( + [1, 2, 3, 5, 8, 13, 21, 34, 3, 9, 1, 30, 17, 4, 64, 20], + [0, 5, 0, 64, 0, 3, 0, 128, 0, 0, 7, 0, 0, 1, 0, 64], + {14: 64}, + ), + ([200, 11], [0, 0], None), + ([1], [900], None), + ] + for i, (lens, prefixes, track) in enumerate(cases): + with self.subTest(case=i): + self._check_replay( + meta, + graph, + out, + lens=lens, + prefixes=prefixes, + track=track, + seed=100 + i, + ) + + def test_capacity(self): + meta = KDAPrefillGraphMetadata.allocate( + num_tokens=BUCKET, + max_seqs_cap=MAX_SEQS, + track=False, + cache_index_dtype=torch.int32, + device=torch.device("cuda"), + ) + self.assertTrue(meta.can_fit([BUCKET // MAX_SEQS] * MAX_SEQS)) + self.assertFalse(meta.can_fit([1] * (MAX_SEQS + 1))) + self.assertFalse(meta.can_fit([BUCKET + 1])) + # Every (sequence, chunk) pair of a worst-case batch fits the tables. + lens = [1] * (MAX_SEQS - 1) + [BUCKET - MAX_SEQS + 1] + chunks = sum((n + CHUNK - 1) // CHUNK for n in lens) + self.assertLessEqual(chunks, meta.max_chunks) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/managers/scheduler_components/test_dp_attn.py b/test/registered/unit/managers/scheduler_components/test_dp_attn.py index c50fca132d..039be3466d 100644 --- a/test/registered/unit/managers/scheduler_components/test_dp_attn.py +++ b/test/registered/unit/managers/scheduler_components/test_dp_attn.py @@ -13,6 +13,7 @@ maybe_stub_sgl_kernel() from sglang.srt.constants import HEALTH_CHECK_RID_PREFIX # noqa: E402 from sglang.srt.environ import envs # noqa: E402 from sglang.srt.managers.scheduler_components import dp_attn # noqa: E402 +from sglang.srt.model_executor.cuda_graph_config import Backend # noqa: E402 from sglang.srt.model_executor.forward_batch_info import ForwardMode # noqa: E402 from sglang.srt.observability.metrics_collector import DPBalanceStats # noqa: E402 from sglang.srt.speculative.spec_info import SpeculativeAlgorithm # noqa: E402 @@ -496,6 +497,54 @@ class TestPrefillCudaGraphVote(CustomTestCase): runner.can_replay_locally.assert_called_once() self.assertTrue(runner.can_replay_locally.call_args.kwargs["is_mixed"]) + def test_batch_over_captured_metadata_bound_votes_eager(self): + """An extend or mixed batch over the captured attention metadata's + request bound votes eager, so no dp rank replays while another falls + back at forward time.""" + runner_cls = dp_attn.PrefillCudaGraphRunner + runner = Mock(spec=runner_cls) + runner.enable_lora = False + runner.max_context_size = None + runner._qwen_bcg_hc_sidechannel = False + runner._is_full_backend = False + runner._captured_attn_metadata_max_bs = 2 + runner.prefer_eager_mixed_prefill = False + runner.prefill_backend_name = Backend.BREAKABLE + runner.has_mha_companion_layers = False + runner._has_uncapturable_chunked_prefix.return_value = False + runner.max_num_tokens = 64 + runner.capture_num_tokens = [64] + runner._pad_to_bucket = runner_cls._pad_to_bucket + runner.can_replay_locally.side_effect = lambda **kwargs: ( + runner_cls.can_replay_locally(self=runner, **kwargs) + ) + + def vote(mode, batch_size): + batch = SimpleNamespace( + forward_mode=mode, + extend_num_tokens=48, + multimodal_inputs=None, + input_embeds=None, + replace_embeds=None, + prefix_lens=[0] * batch_size, + return_logprob=False, + batch_size=lambda: batch_size, + ) + return dp_attn._local_prefill_cuda_graph_vote( + local_batch=batch, + prefill_graph_runner=runner, + coordinated_prefill=True, + breakable_prefill=True, + spec_algorithm=SpeculativeAlgorithm.NONE, + model_config=object(), + ) + + # Mixed batches (--enable-mixed-chunk) count their decode rows too. + for mode in (ForwardMode.EXTEND, ForwardMode.MIXED): + with self.subTest(mode=mode): + self.assertTrue(vote(mode, batch_size=2)) + self.assertFalse(vote(mode, batch_size=3)) + @patch.object(dp_attn, "get_parallel") @patch.object(dp_attn, "all_gather_single") def test_multimodal_ranks_vote_eager(self, gather, parallel):