[Linear Attention] Expose GDN/KDA prefill hooks and auxiliary cache accounting (#40227)

Co-authored-by: Ke Bao <ispobaoke@gmail.com>
This commit is contained in:
paulzhang-tm
2026-09-30 17:19:02 +08:00
committed by GitHub
co-authored by Ke Bao
parent 9b7464c8dd
commit 8055ccd2cd
6 changed files with 314 additions and 40 deletions
@@ -38,6 +38,7 @@ if TYPE_CHECKING:
from sglang.srt.model_executor.model_runner import ModelRunner
ATTENTION_BACKENDS = {}
HYBRID_GDN_SM100_BACKENDS = {"triton", "trtllm_mha", "fa4", "flashinfer"}
def register_attention_backend(name):
@@ -438,7 +439,7 @@ def attn_backend_wrapper(runner: "ModelRunner", full_attn_backend: "AttentionBac
# GDN models. In particular, quantized KV recipes use it
# to expose an FP8 dequant workspace while a different
# backend (for example TRT-LLM GenMHA) owns decode.
allowed = {"triton", "trtllm_mha", "fa4", "flashinfer"}
allowed = HYBRID_GDN_SM100_BACKENDS
prefill_be = runner.prefill_attention_backend_str
decode_be = runner.decode_attention_backend_str
assert prefill_be in allowed and decode_be in allowed, (
@@ -515,6 +515,7 @@ class GDNAttnBackend(MambaAttnBackendBase):
needs_cpu_seq_lens: bool = False
supports_mis: bool = True
requires_contiguous_prefill_state: bool = True
def __init__(self, model_runner: ModelRunner):
_validate_gdn_linear_attn_backends(model_runner.linear_attn_backends)
@@ -881,6 +882,32 @@ class GDNAttnBackend(MambaAttnBackendBase):
return (core_attn_out, z) if return_z else core_attn_out
def _convolve_prefill(
self,
layer: RadixLinearAttention,
forward_batch: ForwardBatch,
mixed_qkv: torch.Tensor,
conv_states: torch.Tensor,
cache_indices: torch.Tensor,
) -> torch.Tensor:
return causal_conv1d_fn(
mixed_qkv.transpose(0, 1),
layer.conv_weights,
layer.bias,
activation=layer.activation,
conv_states=conv_states,
has_initial_state=forward_batch.extend_prefix_lens > 0,
cache_indices=cache_indices,
query_start_loc=self.forward_metadata.query_start_loc,
seq_lens_cpu=forward_batch.extend_seq_lens_cpu,
).transpose(0, 1)[: mixed_qkv.shape[0]]
@staticmethod
def _prefill_gates(
layer: RadixLinearAttention, a: torch.Tensor, b: torch.Tensor
) -> tuple[torch.Tensor, torch.Tensor]:
return fused_gdn_gating(layer.A_log, a, b, layer.dt_bias)
def forward_extend(
self,
layer: RadixLinearAttention,
@@ -938,8 +965,6 @@ class GDNAttnBackend(MambaAttnBackendBase):
and getattr(mamba_pool, "replayssm_cache_base", None) is not None
and not getattr(mamba_pool, "replayssm_is_kda", False)
)
else:
has_initial_states = forward_batch.extend_prefix_lens > 0
# Page-major envelope: the prefill kernels (CUDA causal_conv1d_fwd,
# chunk_gated_delta_rule) write state back in place assuming a contiguous
@@ -953,7 +978,8 @@ class GDNAttnBackend(MambaAttnBackendBase):
# and chunk_gated_delta_rule kernels honor the pool's real slot stride +
# int64 indexing, like packed_decode / causal_conv1d_update already do.
needs_state_gather = (
(not is_target_verify)
self.requires_contiguous_prefill_state
and (not is_target_verify)
and (not is_cpu())
and (not conv_states.is_contiguous() or not ssm_states.is_contiguous())
)
@@ -991,26 +1017,21 @@ class GDNAttnBackend(MambaAttnBackendBase):
)
mixed_qkv = mixed_qkv_processed.transpose(1, 2).view(seq_len, -1)
else:
mixed_qkv = mixed_qkv.transpose(0, 1)
if forward_metadata.has_mamba_track_mask:
mixed_qkv_to_track = mixed_qkv[
:, forward_metadata.track_conv_indices
].transpose(0, 1)
forward_metadata.track_conv_indices
].transpose(1, 2)
conv_states[forward_metadata.conv_states_mask_indices] = (
mixed_qkv_to_track
)
mixed_qkv = causal_conv1d_fn(
mixed_qkv = self._convolve_prefill(
layer,
forward_batch,
mixed_qkv,
layer.conv_weights,
layer.bias,
activation=layer.activation,
conv_states=conv_states_contig,
has_initial_state=has_initial_states,
cache_indices=state_cache_indices,
query_start_loc=query_start_loc,
seq_lens_cpu=forward_batch.extend_seq_lens_cpu,
).transpose(0, 1)[:seq_len]
conv_states_contig,
state_cache_indices,
)
actual_seq_len = mixed_qkv.shape[0]
qkv_dim = layer.q_dim + layer.k_dim + layer.v_dim
@@ -1098,7 +1119,21 @@ class GDNAttnBackend(MambaAttnBackendBase):
retrieve_parent_token=retrieve_parent_token,
)
else:
g, beta = fused_gdn_gating(layer.A_log, a, b, layer.dt_bias)
g, beta = self._prefill_gates(layer, a, b)
h_track_buf = None
if (
forward_metadata.has_mamba_track_mask
and forward_metadata.track_ssm_h_src.numel() > 0
and self.kernel_dispatcher.extend_kernel.supports_track_state_snapshot
):
# Snapshot-capable kernels can supply just the selected FP32
# checkpoint instead of materializing all per-chunk states.
assert forward_metadata.track_chunk_idx is not None
h_track_buf = torch.empty(
(len(cache_indices), *ssm_states.shape[1:]),
dtype=torch.float32,
device=ssm_states.device,
)
core_attn_out, last_recurrent_state, h = self.kernel_dispatcher.extend(
q=query,
k=key,
@@ -1116,6 +1151,14 @@ class GDNAttnBackend(MambaAttnBackendBase):
forward_metadata.state_checkpoint_every_n_tokens
),
output=kwargs.get("linear_attn_output"),
layer_id=layer.layer_id,
extend_prefix_lens=forward_batch.extend_prefix_lens,
track_state=h_track_buf,
track_chunk_idx=(
forward_metadata.track_chunk_idx
if h_track_buf is not None
else None
),
)
if is_npu() and last_recurrent_state is not None:
@@ -1132,7 +1175,11 @@ class GDNAttnBackend(MambaAttnBackendBase):
if forward_metadata.has_mamba_track_mask:
self._track_mamba_state_extend(
forward_batch, h, ssm_states, forward_metadata
forward_batch,
h,
ssm_states,
forward_metadata,
h_track_buf=h_track_buf,
)
return core_attn_out
@@ -793,6 +793,28 @@ class KDAAttnBackend(MambaAttnBackendBase):
return core_attn_out
def _convolve_prefill(
self,
layer: RadixLinearAttention,
forward_batch: ForwardBatch,
mixed_qkv: torch.Tensor,
conv_states: torch.Tensor,
cache_indices: torch.Tensor,
) -> torch.Tensor:
# Depthwise conv is channel-independent, so one packed call over the
# full qkv width matches the decode path and saves two kernel launches.
return causal_conv1d_fn(
mixed_qkv.transpose(0, 1),
layer.conv_weights,
layer.bias,
activation="silu",
conv_states=conv_states,
has_initial_state=forward_batch.extend_prefix_lens > 0,
cache_indices=cache_indices,
query_start_loc=self.forward_metadata.query_start_loc,
seq_lens_cpu=forward_batch.extend_seq_lens_cpu,
).transpose(0, 1)
def forward_extend(
self,
layer: RadixLinearAttention,
@@ -820,7 +842,6 @@ class KDAAttnBackend(MambaAttnBackendBase):
raise RuntimeError(
"extend_prefix_lens cannot be None in non-TARGET_VERIFY mode."
)
has_initial_state = forward_batch.extend_prefix_lens > 0
physical_num_tokens = mixed_qkv.shape[0]
logical_num_tokens = self.forward_metadata.logical_num_tokens
@@ -841,19 +862,9 @@ class KDAAttnBackend(MambaAttnBackendBase):
self.forward_metadata.conv_states_mask_indices
] = mixed_qkv[self.forward_metadata.track_conv_indices]
# Depthwise conv is channel-independent, so one packed call over the
# full qkv width matches the decode path and saves two kernel launches.
qkv = causal_conv1d_fn(
mixed_qkv.transpose(0, 1),
layer.conv_weights,
layer.bias,
activation="silu",
conv_states=conv_states,
has_initial_state=has_initial_state,
cache_indices=cache_indices,
query_start_loc=query_start_loc,
seq_lens_cpu=forward_batch.extend_seq_lens_cpu,
).transpose(0, 1)
qkv = self._convolve_prefill(
layer, forward_batch, mixed_qkv, conv_states, cache_indices
)
q, k, v = qkv.split([layer.q_dim, layer.k_dim, layer.v_dim], dim=-1)
q = q.unflatten(-1, (-1, layer.head_q_dim)).unsqueeze(0) # n (h d) -> 1 n h d
@@ -907,6 +918,8 @@ class KDAAttnBackend(MambaAttnBackendBase):
lower_bound=layer.lower_bound,
beta_is_raw=gate_was_flat,
extend_seq_lens_cpu=forward_batch.extend_seq_lens_cpu,
extend_prefix_lens=forward_batch.extend_prefix_lens,
layer_id=layer.layer_id,
# draft_extend_v2 must stay rollback-able, so kernels that commit state
# in place (e.g. FlashKDA) must not run for it.
is_spec_decode=forward_batch.forward_mode.is_draft_extend_v2(),
@@ -285,6 +285,7 @@ class KVCacheConfigurator:
memory_pool_config: Optional[MemoryPoolConfig]
draft_model_idx: Optional[int] = None
kv_cache_dtype_str: Optional[str] = None
extra_mamba_cache_bytes_per_req: int = 0
mambaish_config: Optional[Any] = field(init=False)
hybrid_gdn_config: Optional[Any] = field(init=False)
hybrid_kda_config: Optional[Any] = field(init=False)
@@ -2479,6 +2480,9 @@ class KVCacheConfigurator:
else:
replayssm_fixed_bytes = 0
replayssm_ring_per_slot = replayssm_ring_per_req
extra_per_slot = replayssm_ring_per_slot + int(
self.extra_mamba_cache_bytes_per_req * pp_layer_scale
)
if has_spec_dec:
assert get_spec().speculative_num_draft_tokens is not None
assert get_schedule().max_running_requests is not None
@@ -2530,8 +2534,7 @@ class KVCacheConfigurator:
per_req = stage_per_req
# Solve jointly for max_mamba_cache_size (K), including the pool's
# +1 padding slot on both buffers (see memory_pool.py):
# (K + 1) * per_req + (K / ratio + 1) * D * per_req = mamba_budget_bytes
# +1 padding slot on both buffers (see memory_pool.py).
mamba_budget = (
total_rest_memory
* get_schedule().mamba_full_memory_ratio
@@ -2546,8 +2549,8 @@ class KVCacheConfigurator:
get_context().override(
"mamba_pool.memory_budget_spec",
max_mamba_cache_size=int(
(mamba_budget_bytes - per_req * (1 + D))
// (per_req * (1 + D / ratio))
(mamba_budget_bytes - per_req * (1 + D) - extra_per_slot)
// (per_req * (1 + D / ratio) + extra_per_slot)
),
)
# Intermediate memory is included in mamba_budget, subtract it
@@ -2559,7 +2562,7 @@ class KVCacheConfigurator:
intermediate_size = per_req * (capped_reqs + 1) * D
total_rest_memory = total_rest_memory - (intermediate_size / (1 << 30))
else:
per_slot = per_req + replayssm_ring_per_slot
per_slot = per_req + extra_per_slot
get_context().override(
"mamba_pool.memory_budget",
max_mamba_cache_size=int(
@@ -2586,8 +2589,7 @@ class KVCacheConfigurator:
# +1 accounts for each pool's padding slot.
mamba_state_memory = (
(get_schedule().max_mamba_cache_size + 1)
* (stage_per_req + replayssm_ring_per_slot)
(get_schedule().max_mamba_cache_size + 1) * (stage_per_req + extra_per_slot)
+ replayssm_fixed_bytes
) / (1 << 30)
return total_rest_memory - mamba_state_memory