[Linear Attention] Expose GDN/KDA prefill hooks and auxiliary cache accounting (#40227)
Co-authored-by: Ke Bao <ispobaoke@gmail.com>
This commit is contained in:
@@ -38,6 +38,7 @@ if TYPE_CHECKING:
|
||||
from sglang.srt.model_executor.model_runner import ModelRunner
|
||||
|
||||
ATTENTION_BACKENDS = {}
|
||||
HYBRID_GDN_SM100_BACKENDS = {"triton", "trtllm_mha", "fa4", "flashinfer"}
|
||||
|
||||
|
||||
def register_attention_backend(name):
|
||||
@@ -438,7 +439,7 @@ def attn_backend_wrapper(runner: "ModelRunner", full_attn_backend: "AttentionBac
|
||||
# GDN models. In particular, quantized KV recipes use it
|
||||
# to expose an FP8 dequant workspace while a different
|
||||
# backend (for example TRT-LLM GenMHA) owns decode.
|
||||
allowed = {"triton", "trtllm_mha", "fa4", "flashinfer"}
|
||||
allowed = HYBRID_GDN_SM100_BACKENDS
|
||||
prefill_be = runner.prefill_attention_backend_str
|
||||
decode_be = runner.decode_attention_backend_str
|
||||
assert prefill_be in allowed and decode_be in allowed, (
|
||||
|
||||
@@ -515,6 +515,7 @@ class GDNAttnBackend(MambaAttnBackendBase):
|
||||
|
||||
needs_cpu_seq_lens: bool = False
|
||||
supports_mis: bool = True
|
||||
requires_contiguous_prefill_state: bool = True
|
||||
|
||||
def __init__(self, model_runner: ModelRunner):
|
||||
_validate_gdn_linear_attn_backends(model_runner.linear_attn_backends)
|
||||
@@ -881,6 +882,32 @@ class GDNAttnBackend(MambaAttnBackendBase):
|
||||
|
||||
return (core_attn_out, z) if return_z else core_attn_out
|
||||
|
||||
def _convolve_prefill(
|
||||
self,
|
||||
layer: RadixLinearAttention,
|
||||
forward_batch: ForwardBatch,
|
||||
mixed_qkv: torch.Tensor,
|
||||
conv_states: torch.Tensor,
|
||||
cache_indices: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
return causal_conv1d_fn(
|
||||
mixed_qkv.transpose(0, 1),
|
||||
layer.conv_weights,
|
||||
layer.bias,
|
||||
activation=layer.activation,
|
||||
conv_states=conv_states,
|
||||
has_initial_state=forward_batch.extend_prefix_lens > 0,
|
||||
cache_indices=cache_indices,
|
||||
query_start_loc=self.forward_metadata.query_start_loc,
|
||||
seq_lens_cpu=forward_batch.extend_seq_lens_cpu,
|
||||
).transpose(0, 1)[: mixed_qkv.shape[0]]
|
||||
|
||||
@staticmethod
|
||||
def _prefill_gates(
|
||||
layer: RadixLinearAttention, a: torch.Tensor, b: torch.Tensor
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
return fused_gdn_gating(layer.A_log, a, b, layer.dt_bias)
|
||||
|
||||
def forward_extend(
|
||||
self,
|
||||
layer: RadixLinearAttention,
|
||||
@@ -938,8 +965,6 @@ class GDNAttnBackend(MambaAttnBackendBase):
|
||||
and getattr(mamba_pool, "replayssm_cache_base", None) is not None
|
||||
and not getattr(mamba_pool, "replayssm_is_kda", False)
|
||||
)
|
||||
else:
|
||||
has_initial_states = forward_batch.extend_prefix_lens > 0
|
||||
|
||||
# Page-major envelope: the prefill kernels (CUDA causal_conv1d_fwd,
|
||||
# chunk_gated_delta_rule) write state back in place assuming a contiguous
|
||||
@@ -953,7 +978,8 @@ class GDNAttnBackend(MambaAttnBackendBase):
|
||||
# and chunk_gated_delta_rule kernels honor the pool's real slot stride +
|
||||
# int64 indexing, like packed_decode / causal_conv1d_update already do.
|
||||
needs_state_gather = (
|
||||
(not is_target_verify)
|
||||
self.requires_contiguous_prefill_state
|
||||
and (not is_target_verify)
|
||||
and (not is_cpu())
|
||||
and (not conv_states.is_contiguous() or not ssm_states.is_contiguous())
|
||||
)
|
||||
@@ -991,26 +1017,21 @@ class GDNAttnBackend(MambaAttnBackendBase):
|
||||
)
|
||||
mixed_qkv = mixed_qkv_processed.transpose(1, 2).view(seq_len, -1)
|
||||
else:
|
||||
mixed_qkv = mixed_qkv.transpose(0, 1)
|
||||
if forward_metadata.has_mamba_track_mask:
|
||||
mixed_qkv_to_track = mixed_qkv[
|
||||
:, forward_metadata.track_conv_indices
|
||||
].transpose(0, 1)
|
||||
forward_metadata.track_conv_indices
|
||||
].transpose(1, 2)
|
||||
conv_states[forward_metadata.conv_states_mask_indices] = (
|
||||
mixed_qkv_to_track
|
||||
)
|
||||
|
||||
mixed_qkv = causal_conv1d_fn(
|
||||
mixed_qkv = self._convolve_prefill(
|
||||
layer,
|
||||
forward_batch,
|
||||
mixed_qkv,
|
||||
layer.conv_weights,
|
||||
layer.bias,
|
||||
activation=layer.activation,
|
||||
conv_states=conv_states_contig,
|
||||
has_initial_state=has_initial_states,
|
||||
cache_indices=state_cache_indices,
|
||||
query_start_loc=query_start_loc,
|
||||
seq_lens_cpu=forward_batch.extend_seq_lens_cpu,
|
||||
).transpose(0, 1)[:seq_len]
|
||||
conv_states_contig,
|
||||
state_cache_indices,
|
||||
)
|
||||
|
||||
actual_seq_len = mixed_qkv.shape[0]
|
||||
qkv_dim = layer.q_dim + layer.k_dim + layer.v_dim
|
||||
@@ -1098,7 +1119,21 @@ class GDNAttnBackend(MambaAttnBackendBase):
|
||||
retrieve_parent_token=retrieve_parent_token,
|
||||
)
|
||||
else:
|
||||
g, beta = fused_gdn_gating(layer.A_log, a, b, layer.dt_bias)
|
||||
g, beta = self._prefill_gates(layer, a, b)
|
||||
h_track_buf = None
|
||||
if (
|
||||
forward_metadata.has_mamba_track_mask
|
||||
and forward_metadata.track_ssm_h_src.numel() > 0
|
||||
and self.kernel_dispatcher.extend_kernel.supports_track_state_snapshot
|
||||
):
|
||||
# Snapshot-capable kernels can supply just the selected FP32
|
||||
# checkpoint instead of materializing all per-chunk states.
|
||||
assert forward_metadata.track_chunk_idx is not None
|
||||
h_track_buf = torch.empty(
|
||||
(len(cache_indices), *ssm_states.shape[1:]),
|
||||
dtype=torch.float32,
|
||||
device=ssm_states.device,
|
||||
)
|
||||
core_attn_out, last_recurrent_state, h = self.kernel_dispatcher.extend(
|
||||
q=query,
|
||||
k=key,
|
||||
@@ -1116,6 +1151,14 @@ class GDNAttnBackend(MambaAttnBackendBase):
|
||||
forward_metadata.state_checkpoint_every_n_tokens
|
||||
),
|
||||
output=kwargs.get("linear_attn_output"),
|
||||
layer_id=layer.layer_id,
|
||||
extend_prefix_lens=forward_batch.extend_prefix_lens,
|
||||
track_state=h_track_buf,
|
||||
track_chunk_idx=(
|
||||
forward_metadata.track_chunk_idx
|
||||
if h_track_buf is not None
|
||||
else None
|
||||
),
|
||||
)
|
||||
|
||||
if is_npu() and last_recurrent_state is not None:
|
||||
@@ -1132,7 +1175,11 @@ class GDNAttnBackend(MambaAttnBackendBase):
|
||||
|
||||
if forward_metadata.has_mamba_track_mask:
|
||||
self._track_mamba_state_extend(
|
||||
forward_batch, h, ssm_states, forward_metadata
|
||||
forward_batch,
|
||||
h,
|
||||
ssm_states,
|
||||
forward_metadata,
|
||||
h_track_buf=h_track_buf,
|
||||
)
|
||||
|
||||
return core_attn_out
|
||||
|
||||
@@ -793,6 +793,28 @@ class KDAAttnBackend(MambaAttnBackendBase):
|
||||
|
||||
return core_attn_out
|
||||
|
||||
def _convolve_prefill(
|
||||
self,
|
||||
layer: RadixLinearAttention,
|
||||
forward_batch: ForwardBatch,
|
||||
mixed_qkv: torch.Tensor,
|
||||
conv_states: torch.Tensor,
|
||||
cache_indices: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
# Depthwise conv is channel-independent, so one packed call over the
|
||||
# full qkv width matches the decode path and saves two kernel launches.
|
||||
return causal_conv1d_fn(
|
||||
mixed_qkv.transpose(0, 1),
|
||||
layer.conv_weights,
|
||||
layer.bias,
|
||||
activation="silu",
|
||||
conv_states=conv_states,
|
||||
has_initial_state=forward_batch.extend_prefix_lens > 0,
|
||||
cache_indices=cache_indices,
|
||||
query_start_loc=self.forward_metadata.query_start_loc,
|
||||
seq_lens_cpu=forward_batch.extend_seq_lens_cpu,
|
||||
).transpose(0, 1)
|
||||
|
||||
def forward_extend(
|
||||
self,
|
||||
layer: RadixLinearAttention,
|
||||
@@ -820,7 +842,6 @@ class KDAAttnBackend(MambaAttnBackendBase):
|
||||
raise RuntimeError(
|
||||
"extend_prefix_lens cannot be None in non-TARGET_VERIFY mode."
|
||||
)
|
||||
has_initial_state = forward_batch.extend_prefix_lens > 0
|
||||
|
||||
physical_num_tokens = mixed_qkv.shape[0]
|
||||
logical_num_tokens = self.forward_metadata.logical_num_tokens
|
||||
@@ -841,19 +862,9 @@ class KDAAttnBackend(MambaAttnBackendBase):
|
||||
self.forward_metadata.conv_states_mask_indices
|
||||
] = mixed_qkv[self.forward_metadata.track_conv_indices]
|
||||
|
||||
# Depthwise conv is channel-independent, so one packed call over the
|
||||
# full qkv width matches the decode path and saves two kernel launches.
|
||||
qkv = causal_conv1d_fn(
|
||||
mixed_qkv.transpose(0, 1),
|
||||
layer.conv_weights,
|
||||
layer.bias,
|
||||
activation="silu",
|
||||
conv_states=conv_states,
|
||||
has_initial_state=has_initial_state,
|
||||
cache_indices=cache_indices,
|
||||
query_start_loc=query_start_loc,
|
||||
seq_lens_cpu=forward_batch.extend_seq_lens_cpu,
|
||||
).transpose(0, 1)
|
||||
qkv = self._convolve_prefill(
|
||||
layer, forward_batch, mixed_qkv, conv_states, cache_indices
|
||||
)
|
||||
q, k, v = qkv.split([layer.q_dim, layer.k_dim, layer.v_dim], dim=-1)
|
||||
|
||||
q = q.unflatten(-1, (-1, layer.head_q_dim)).unsqueeze(0) # n (h d) -> 1 n h d
|
||||
@@ -907,6 +918,8 @@ class KDAAttnBackend(MambaAttnBackendBase):
|
||||
lower_bound=layer.lower_bound,
|
||||
beta_is_raw=gate_was_flat,
|
||||
extend_seq_lens_cpu=forward_batch.extend_seq_lens_cpu,
|
||||
extend_prefix_lens=forward_batch.extend_prefix_lens,
|
||||
layer_id=layer.layer_id,
|
||||
# draft_extend_v2 must stay rollback-able, so kernels that commit state
|
||||
# in place (e.g. FlashKDA) must not run for it.
|
||||
is_spec_decode=forward_batch.forward_mode.is_draft_extend_v2(),
|
||||
|
||||
@@ -285,6 +285,7 @@ class KVCacheConfigurator:
|
||||
memory_pool_config: Optional[MemoryPoolConfig]
|
||||
draft_model_idx: Optional[int] = None
|
||||
kv_cache_dtype_str: Optional[str] = None
|
||||
extra_mamba_cache_bytes_per_req: int = 0
|
||||
mambaish_config: Optional[Any] = field(init=False)
|
||||
hybrid_gdn_config: Optional[Any] = field(init=False)
|
||||
hybrid_kda_config: Optional[Any] = field(init=False)
|
||||
@@ -2479,6 +2480,9 @@ class KVCacheConfigurator:
|
||||
else:
|
||||
replayssm_fixed_bytes = 0
|
||||
replayssm_ring_per_slot = replayssm_ring_per_req
|
||||
extra_per_slot = replayssm_ring_per_slot + int(
|
||||
self.extra_mamba_cache_bytes_per_req * pp_layer_scale
|
||||
)
|
||||
if has_spec_dec:
|
||||
assert get_spec().speculative_num_draft_tokens is not None
|
||||
assert get_schedule().max_running_requests is not None
|
||||
@@ -2530,8 +2534,7 @@ class KVCacheConfigurator:
|
||||
per_req = stage_per_req
|
||||
|
||||
# Solve jointly for max_mamba_cache_size (K), including the pool's
|
||||
# +1 padding slot on both buffers (see memory_pool.py):
|
||||
# (K + 1) * per_req + (K / ratio + 1) * D * per_req = mamba_budget_bytes
|
||||
# +1 padding slot on both buffers (see memory_pool.py).
|
||||
mamba_budget = (
|
||||
total_rest_memory
|
||||
* get_schedule().mamba_full_memory_ratio
|
||||
@@ -2546,8 +2549,8 @@ class KVCacheConfigurator:
|
||||
get_context().override(
|
||||
"mamba_pool.memory_budget_spec",
|
||||
max_mamba_cache_size=int(
|
||||
(mamba_budget_bytes - per_req * (1 + D))
|
||||
// (per_req * (1 + D / ratio))
|
||||
(mamba_budget_bytes - per_req * (1 + D) - extra_per_slot)
|
||||
// (per_req * (1 + D / ratio) + extra_per_slot)
|
||||
),
|
||||
)
|
||||
# Intermediate memory is included in mamba_budget, subtract it
|
||||
@@ -2559,7 +2562,7 @@ class KVCacheConfigurator:
|
||||
intermediate_size = per_req * (capped_reqs + 1) * D
|
||||
total_rest_memory = total_rest_memory - (intermediate_size / (1 << 30))
|
||||
else:
|
||||
per_slot = per_req + replayssm_ring_per_slot
|
||||
per_slot = per_req + extra_per_slot
|
||||
get_context().override(
|
||||
"mamba_pool.memory_budget",
|
||||
max_mamba_cache_size=int(
|
||||
@@ -2586,8 +2589,7 @@ class KVCacheConfigurator:
|
||||
|
||||
# +1 accounts for each pool's padding slot.
|
||||
mamba_state_memory = (
|
||||
(get_schedule().max_mamba_cache_size + 1)
|
||||
* (stage_per_req + replayssm_ring_per_slot)
|
||||
(get_schedule().max_mamba_cache_size + 1) * (stage_per_req + extra_per_slot)
|
||||
+ replayssm_fixed_bytes
|
||||
) / (1 << 30)
|
||||
return total_rest_memory - mamba_state_memory
|
||||
|
||||
Reference in New Issue
Block a user