[KDA] Support Helion MTP verification and harden prefill, decode (#42857)

This commit is contained in:
Jzz1943
2026-10-11 12:47:57 +08:00
committed by GitHub
parent 65e6bb6be7
commit 260551ebe9
5 changed files with 352 additions and 19 deletions
@@ -0,0 +1,145 @@
"""Helion KDA verification with per-candidate state snapshots."""
from __future__ import annotations
import helion
import helion.language as hl
import torch
@helion.kernel(
static_shapes=False,
config=helion.Config(
block_sizes=[8],
num_warps=1,
num_stages=1,
indexing="pointer",
),
ignore_warnings=[helion.exc.ProcessGroupNameNotFound],
)
def _kda_verify(
q: torch.Tensor,
k: torch.Tensor,
v: torch.Tensor,
a: torch.Tensor,
b: torch.Tensor,
A_log: torch.Tensor,
dt_bias: torch.Tensor,
state_pool: torch.Tensor,
state_indices: torch.Tensor,
cu_seqlens: torch.Tensor,
snapshots: torch.Tensor,
snapshot_indices: torch.Tensor,
parents: torch.Tensor | None,
out: torch.Tensor,
lower_bound: float,
bounded_gate: hl.constexpr,
) -> torch.Tensor:
N = cu_seqlens.size(0) - 1
H = hl.specialize(q.size(1))
HV = hl.specialize(v.size(1))
K = hl.specialize(q.size(2))
V = hl.specialize(v.size(2))
block_v = hl.register_block_size(1, V)
for tile_n, tile_h, tile_v in hl.tile([N, HV, V], block_size=[1, 1, block_v]):
n, hv = tile_n.id, tile_h.id
h = hv // (HV // H)
keys = hl.arange(K)
slot = state_indices[n].long()
scratch = snapshot_indices[n].long()
start = cu_seqlens[n].long()
end = cu_seqlens[n + 1].long()
state = hl.load(
state_pool, [slot, hv, tile_v.index, keys], extra_mask=slot >= 0
).float()
for token in range(start, end):
step = token - start
if parents is not None:
use_parent = (step > 0) & (scratch >= 0)
parent = parents[n, step].long()
parent_state = hl.load(
snapshots,
[scratch, parent, hv, tile_v.index, keys],
extra_mask=use_parent,
).float()
state = torch.where(use_parent, parent_state, state)
raw_gate = a[token, hv, keys].float() + dt_bias[hv, keys].float()
A = torch.exp(A_log[hv].float())
if bounded_gate:
log_decay = lower_bound * torch.sigmoid(A * raw_gate)
else:
softplus = torch.where(
raw_gate <= 20.0, torch.log(1.0 + torch.exp(raw_gate)), raw_gate
)
log_decay = -A * softplus
beta = torch.sigmoid(b[token, hv].float())
query = q[token, h, keys].float()
key = k[token, h, keys].float()
query = query / torch.sqrt((query * query).sum() + 1e-6)
key = key / torch.sqrt((key * key).sum() + 1e-6)
state = state * torch.exp(log_decay)[None, :]
value = v[token, hv, tile_v].float()
residual = (value - (state * key[None, :]).sum(-1)) * beta
state = state + residual[:, None] * key[None, :]
out[token, hv, tile_v] = (state * (query * K**-0.5)[None, :]).sum(-1)
if scratch >= 0:
snapshots[scratch, step, hv, tile_v.index, keys] = state
return out
def helion_kda_verify(
*,
q: torch.Tensor,
k: torch.Tensor,
v: torch.Tensor,
a: torch.Tensor,
b: torch.Tensor,
A_log: torch.Tensor,
dt_bias: torch.Tensor,
ssm_states: torch.Tensor,
cache_indices: torch.Tensor,
query_start_loc: torch.Tensor,
intermediate_states_buffer: torch.Tensor,
intermediate_state_indices: torch.Tensor,
cache_steps: int,
retrieve_parent_token: torch.Tensor | None = None,
lower_bound: float | None = None,
) -> torch.Tensor:
"""Verify packed sequences without mutating the committed state pool.
Chain verification keeps an FP32 accumulator across candidates; tree
verification reads each parent's snapshot, matching the reference contract.
Negative state/snapshot indices are CUDA Graph padding slots.
"""
if q.ndim != 4 or q.shape[0] != 1:
raise ValueError("Helion KDA verify expects packed [1, T, H, K] inputs")
tokens, heads, key_dim = q.shape[1:]
value_heads, value_dim = v.shape[2:]
if value_heads % heads or k.shape != q.shape:
raise ValueError("Inconsistent Helion KDA verify query/key head layout")
if intermediate_states_buffer is None or cache_steps <= 0:
raise ValueError("Helion KDA verify requires per-candidate state snapshots")
if intermediate_states_buffer.shape[1] < cache_steps:
raise ValueError(
"Helion KDA verify snapshot capacity is smaller than cache_steps"
)
out = torch.zeros_like(v)
_kda_verify(
q[0],
k[0],
v[0],
a.reshape(tokens, value_heads, key_dim),
b.reshape(tokens, value_heads),
A_log.reshape(-1),
dt_bias.reshape(value_heads, key_dim),
ssm_states,
cache_indices,
query_start_loc,
intermediate_states_buffer,
intermediate_state_indices,
retrieve_parent_token,
out[0],
0.0 if lower_bound is None else lower_bound,
lower_bound is not None,
)
return out
@@ -110,7 +110,11 @@ class KDAKernelDispatcher:
triton_kernel = TritonKDAKernel()
self.triton_kernel = triton_kernel
helion_kernel = None
if decode_backend.is_helion() or prefill_backend.is_helion():
if (
decode_backend.is_helion()
or prefill_backend.is_helion()
or verify_backend.is_helion()
):
if not is_cuda():
raise ValueError("KDA Helion backend requires CUDA")
from sglang.srt.layers.attention.linear.kernels.kda_helion import (
@@ -157,9 +161,13 @@ class KDAKernelDispatcher:
# the KDA correctness tests assert against.
# flashinfer: recurrent_kda (SM100, chain only); reuses the decode kernel
# when decode is also flashinfer.
# helion: recurrent chain + tree verify with per-candidate snapshots.
# nv_cutedsl: fused Kimi-K3/DSpARK dense verify.
if verify_backend.is_triton():
self.verify_kernel = triton_kernel
elif verify_backend.is_helion():
assert helion_kernel is not None
self.verify_kernel = helion_kernel
elif verify_backend.is_flashinfer():
if decode_backend.is_flashinfer():
self.verify_kernel = self.decode_kernel
@@ -182,7 +190,7 @@ class KDAKernelDispatcher:
else:
raise ValueError(
f"Unsupported KDA verify backend: {verify_backend}. "
"KDA verify supports 'triton', 'nv_cutedsl', or 'flashinfer'."
"KDA verify supports 'triton', 'helion', 'nv_cutedsl', or 'flashinfer'."
)
if prefill_backend.is_triton():
@@ -262,6 +270,9 @@ class KDAKernelDispatcher:
self.supports_packed_decode = getattr(
self.decode_kernel, "supports_packed_decode", False
)
self.supports_packed_decode_safe_gate = getattr(
self.decode_kernel, "supports_packed_decode_safe_gate", False
)
rank0_log(
f"KDA kernel dispatcher: decode={self.decode_kernel.__class__.__name__}, "
@@ -361,15 +372,15 @@ class KDAKernelDispatcher:
lower_bound: Optional[float] = None,
**kwargs,
) -> torch.Tensor:
"""MTP / speculative-decode verify, routed to ``self.verify_kernel``
(FlashInfer decode -> recurrent_kda; Triton / CuTe DSL decode -> the Triton
fused KDA verify)."""
if lower_bound is not None and not isinstance(
self.verify_kernel, TritonKDAKernel
"""MTP verification routed to the explicitly selected verify backend."""
if (
lower_bound is not None
and not isinstance(self.verify_kernel, TritonKDAKernel)
and not getattr(self.verify_kernel, "supports_safe_gate_verify", False)
):
raise NotImplementedError(
"lower_bound (safe gate) target verify is only supported by "
f"TritonKDAKernel; got {self.verify_kernel.__class__.__name__}."
"lower_bound (safe gate) target verify is not supported by "
f"{self.verify_kernel.__class__.__name__}."
)
return self.verify_kernel.target_verify(
A_log=A_log,
@@ -931,9 +942,9 @@ class KDAAttnBackend(MambaAttnBackendBase):
)
# The packed kernel assumes one token per request.
if (
self.kernel_dispatcher.supports_packed_decode
and getattr(layer, "lower_bound", None) is None
if self.kernel_dispatcher.supports_packed_decode and (
getattr(layer, "lower_bound", None) is None
or self.kernel_dispatcher.supports_packed_decode_safe_gate
):
assert qkv.shape[0] == cache_indices.shape[0], (
"KDA packed decode requires one token per sequence (T=1): "
@@ -11,15 +11,17 @@ from sglang.srt.layers.attention.linear.kernels.kernel_backend import (
class HelionKDAKernel(LinearAttnKernelBase):
"""KDA packed decode and prefill implemented with Helion kernels.
"""KDA packed decode, prefill, and verification with Helion kernels.
The generic decode interface delegates to Triton, and the dispatcher routes
speculative target verification directly to Triton. The one-token decode
The generic decode interface delegates to Triton. The one-token decode
and ReplaySSM paths use :meth:`packed_decode`, while prefill uses
:meth:`extend`.
:meth:`extend`. Verification writes per-candidate snapshots without
updating the committed state pool.
"""
supports_packed_decode = True
supports_packed_decode_safe_gate = True
supports_safe_gate_verify = True
supports_track_state_snapshot: bool = True
def __init__(
@@ -157,6 +159,51 @@ class HelionKDAKernel(LinearAttnKernelBase):
**kwargs,
)
def target_verify(
self,
*,
q: torch.Tensor,
k: torch.Tensor,
v: torch.Tensor,
a: torch.Tensor,
b: torch.Tensor,
A_log: torch.Tensor,
dt_bias: torch.Tensor,
ssm_states: torch.Tensor,
cache_indices: torch.Tensor,
query_start_loc: torch.Tensor,
intermediate_states_buffer: torch.Tensor,
intermediate_state_indices: torch.Tensor,
cache_steps: int,
retrieve_parent_token: torch.Tensor | None = None,
lower_bound: float | None = None,
cache_ring: bool = False,
**kwargs,
) -> torch.Tensor:
if cache_ring:
raise NotImplementedError(
"Helion KDA verify does not support ReplaySSM rings"
)
from sglang.kernels.ops.attention.helion.kda_verify import helion_kda_verify
return helion_kda_verify(
q=q,
k=k,
v=v,
a=a,
b=b,
A_log=A_log,
dt_bias=dt_bias,
ssm_states=ssm_states,
cache_indices=cache_indices,
query_start_loc=query_start_loc,
intermediate_states_buffer=intermediate_states_buffer,
intermediate_state_indices=intermediate_state_indices,
cache_steps=cache_steps,
retrieve_parent_token=retrieve_parent_token,
lower_bound=lower_bound,
)
def extend(
self,
q: torch.Tensor,