[KDA] Add an opt-in decode-parity mode to the Triton multi-token recurrence (#42251)

Co-authored-by: merrymercy <merrymercy@users.noreply.github.com>
Co-authored-by: jiayisuse <jiayisuse@users.noreply.github.com>
This commit is contained in:
metamergebot
2026-10-02 14:59:56 -07:00
committed by GitHub
co-authored by merrymercy jiayisuse
parent 19794f5743
commit e8fab85a02
2 changed files with 260 additions and 14 deletions
@@ -49,6 +49,73 @@ def _select_recurrent_launch_config(
return min(triton.next_power_of_2(v), 32), 1
_EX2_APPROX_FTZ_F32 = tl.constexpr("ex2.approx.ftz.f32 $0, $1;")
_LG2_APPROX_FTZ_F32 = tl.constexpr("lg2.approx.ftz.f32 $0, $1;")
_UNARY_F32_REGISTERS = tl.constexpr("=f,f")
@triton.jit
def _ex2_approx_ftz_f32(x):
return tl.inline_asm_elementwise(
asm=_EX2_APPROX_FTZ_F32,
constraints=_UNARY_F32_REGISTERS,
args=(x,),
dtype=tl.float32,
is_pure=True,
pack=1,
)
@triton.jit
def _lg2_approx_ftz_f32(x):
return tl.inline_asm_elementwise(
asm=_LG2_APPROX_FTZ_F32,
constraints=_UNARY_F32_REGISTERS,
args=(x,),
dtype=tl.float32,
is_pure=True,
pack=1,
)
@triton.jit
def _cutedsl_exp(x):
return _ex2_approx_ftz_f32(x * 1.4426950408889634)
@triton.jit
def _cutedsl_qk_sum_squares(x, BK: tl.constexpr):
"""Lane-striped Q/K sum of squares, as FlashInfer's ``recurrent_kda`` CuTe
DSL decode kernel reduces it: lane ``l`` owns the elements ``l + i * 32``
and a 32-lane butterfly combines the lanes. Keeping both reduction
dimensions explicit stops Triton from flattening K into a different
reduction tree.
"""
NS: tl.constexpr = BK // 32
tl.static_assert(
NS >= 1 and NS <= 4,
"match_cutedsl_decode needs 32 <= BK <= 128 (head_k_dim <= 128)",
)
# The decode kernel accumulates its per-lane squares with packed f32x2 FMA,
# so the even and odd i streams stay separate until both are complete. A
# plain tl.sum(x * x) is mathematically equivalent, but can move a BF16
# recurrent state across a rounding boundary after several verify steps.
lane_values = tl.reshape(x, (NS, 32), can_reorder=False)
row = tl.arange(0, NS)[:, None]
even0 = tl.sum(tl.where(row == 0, lane_values, 0.0), axis=0)
odd0 = tl.sum(tl.where(row == 1, lane_values, 0.0), axis=0)
zeros = tl.zeros((32,), tl.float32)
even_sum = tl.fma(even0, even0, zeros)
odd_sum = tl.fma(odd0, odd0, zeros)
if NS == 4:
even1 = tl.sum(tl.where(row == 2, lane_values, 0.0), axis=0)
odd1 = tl.sum(tl.where(row == 3, lane_values, 0.0), axis=0)
even_sum = tl.fma(even1, even1, even_sum)
odd_sum = tl.fma(odd1, odd1, odd_sum)
lane_sum = even_sum + odd_sum
return tl.sum(lane_sum, axis=0)
@triton.jit(do_not_specialize=["T"])
def fused_sigmoid_gating_delta_rule_update_kernel(
A_log,
@@ -113,6 +180,8 @@ def fused_sigmoid_gating_delta_rule_update_kernel(
CACHE_RING: tl.constexpr = False,
SPLIT_N_HV_GRID: tl.constexpr = False,
USE_GDC: tl.constexpr = False,
MATCH_CUTEDSL_DECODE: tl.constexpr = False,
ROUND_STATE_TO_BF16: tl.constexpr = False,
):
"""
Fused kernel that combines sigmoid gating computation with recurrent delta rule update.
@@ -247,18 +316,32 @@ def fused_sigmoid_gating_delta_rule_update_kernel(
# KDA safe gate: lower_bound * sigmoid(exp(A_log) * (a + dt_bias))
b_g = lower_bound * tl.sigmoid(tl.exp(b_A_log) * x)
else:
# Compute g = -exp(A_log) * softplus(a + dt_bias)
beta_x = softplus_beta * x
# Apply softplus with numerical stability
softplus_x = tl.where(
beta_x <= softplus_threshold,
(1.0 / softplus_beta) * tl.log(1.0 + tl.exp(beta_x)),
x,
)
b_g = -tl.exp(b_A_log) * softplus_x
if MATCH_CUTEDSL_DECODE and IS_KDA:
# The CuTe decode kernel forms the multiplicative gate directly
# in base 2. Algebraically this is exp(-A * softplus(x)), but
# the instruction sequence decides on which side of a BF16
# state-rounding boundary the result lands.
b_gate_multiplier = _ex2_approx_ftz_f32(
(-_cutedsl_exp(b_A_log))
* _lg2_approx_ftz_f32(1.0 + _cutedsl_exp(x))
)
b_g = tl.log(b_gate_multiplier)
else:
# Compute g = -exp(A_log) * softplus(a + dt_bias)
beta_x = softplus_beta * x
# Apply softplus with numerical stability
softplus_x = tl.where(
beta_x <= softplus_threshold,
(1.0 / softplus_beta) * tl.log(1.0 + tl.exp(beta_x)),
x,
)
b_g = -tl.exp(b_A_log) * softplus_x
# Compute beta = sigmoid(b)
b_beta = 1.0 / (1.0 + tl.exp(-b_b))
if MATCH_CUTEDSL_DECODE:
b_beta = b_b
else:
b_beta = 1.0 / (1.0 + tl.exp(-b_b))
# fused ring-write: stash this step's raw inputs + in-kernel gate/beta
# into the per-slot ring for the commit fold to replay. Must sit here --
@@ -325,14 +408,30 @@ def fused_sigmoid_gating_delta_rule_update_kernel(
# Apply L2 normalization if enabled
if USE_QK_L2NORM_IN_KERNEL:
b_q = b_q / (tl.sqrt(tl.sum(b_q * b_q) + 1e-6))
b_k = b_k / (tl.sqrt(tl.sum(b_k * b_k) + 1e-6))
if MATCH_CUTEDSL_DECODE:
q_sum_sq = _cutedsl_qk_sum_squares(b_q, BK)
k_sum_sq = _cutedsl_qk_sum_squares(b_k, BK)
b_q *= tl.rsqrt(q_sum_sq + 1e-6)
b_k *= tl.rsqrt(k_sum_sq + 1e-6)
else:
b_q = b_q / (tl.sqrt(tl.sum(b_q * b_q) + 1e-6))
b_k = b_k / (tl.sqrt(tl.sum(b_k * b_k) + 1e-6))
b_q = b_q * scale
# Apply gating to hidden state: h *= exp(g)
if IS_KDA:
b_h *= tl.exp(b_g[:, None])
if MATCH_CUTEDSL_DECODE:
# The CuTe decode kernel forms the decayed state with packed
# FMA and a zero addend. Keep that rounding instead of letting
# the compiler pick a plain multiply.
b_h = tl.fma(
b_h,
b_gate_multiplier[:, None],
tl.zeros((BK, BV), tl.float32),
)
else:
b_h *= tl.exp(b_g[:, None])
else:
b_h *= tl.exp(b_g)
@@ -343,7 +442,10 @@ def fused_sigmoid_gating_delta_rule_update_kernel(
b_v *= b_beta
# Update hidden state: h += k[:, None] * v[None, :]
b_h += b_k[:, None] * b_v[None, :]
if MATCH_CUTEDSL_DECODE:
b_h = tl.fma(b_k[:, None], b_v[None, :], b_h)
else:
b_h += b_k[:, None] * b_v[None, :]
# Compute output: o = sum(h * q, dim=0)
b_o = tl.sum(b_h * b_q[:, None], 0)
@@ -363,6 +465,13 @@ def fused_sigmoid_gating_delta_rule_update_kernel(
)
tl.store(cache_ptr, b_h.to(cache_ptr.dtype.element_ty), mask=mask_h)
# A BF16 recurrent-state decode persists its state after every token.
# Multi-token verification must make that precision boundary visible to
# the next recurrence step instead of retaining the FP32 accumulator for
# the whole candidate chain. FP32 state pools must retain the accumulator.
if ROUND_STATE_TO_BF16:
b_h = b_h.to(tl.bfloat16).to(tl.float32)
step_idx += 1
# Update pointers for next timestep
@@ -419,6 +528,12 @@ def fused_sigmoid_gating_delta_rule_update(
replayssm_rawk: Optional[torch.Tensor] = None,
replayssm_g: Optional[torch.Tensor] = None,
replayssm_beta: Optional[torch.Tensor] = None,
# Reproduce FlashInfer's `recurrent_kda` CuTe DSL decode kernel step for
# step: `b` is then the already-sigmoided FP32 beta, the gate is formed in
# base 2 with ex2/lg2.approx, Q/K norms reduce lane-striped, the state
# decays and updates through FMA, and a BF16 state pool is rounded after
# every token.
match_cutedsl_decode: bool = False,
):
"""
Fused triton implementation of sigmoid gating delta rule update.
@@ -430,6 +545,11 @@ def fused_sigmoid_gating_delta_rule_update(
- target_verify: multi-step with intermediate state caching, optional tree attention,
and optional state update disable
"""
if match_cutedsl_decode and (not is_kda or lower_bound is not None):
raise ValueError(
"match_cutedsl_decode is defined for the KDA softplus gate only "
"(is_kda=True, lower_bound=None)."
)
B, T, H, K, V = *k.shape, v.shape[-1]
stride_q = q.stride()[1]
stride_k = k.stride()[1]
@@ -508,6 +628,14 @@ def fused_sigmoid_gating_delta_rule_update(
# KDA and GDN recurrent paths.
pdl_kwargs = {"USE_GDC": True, "launch_pdl": True} if is_arch_support_pdl() else {}
# CuTeDSL specializes on the persisted state dtype. Its BF16 decode path
# writes BF16 after every token and reloads FP32 for the next recurrence;
# its FP32 path has no such rounding boundary. Keep CuTe math parity and
# state-persistence precision as separate compile-time choices.
round_state_to_bf16 = match_cutedsl_decode and (
initial_state_source is None or initial_state_source.dtype == torch.bfloat16
)
fused_sigmoid_gating_delta_rule_update_kernel[grid](
A_log=A_log,
a=a,
@@ -568,6 +696,8 @@ def fused_sigmoid_gating_delta_rule_update(
MAX_CACHE_LEN=max_cache_len,
CACHE_RING=cache_ring,
SPLIT_N_HV_GRID=split_n_hv_grid,
MATCH_CUTEDSL_DECODE=match_cutedsl_decode,
ROUND_STATE_TO_BF16=round_state_to_bf16,
num_warps=num_warps,
num_stages=num_stages,
**pdl_kwargs,