From e8fab85a02c23e01260fb1adb030b00886f5999d Mon Sep 17 00:00:00 2001 From: metamergebot Date: Fri, 2 Oct 2026 14:59:56 -0700 Subject: [PATCH] [KDA] Add an opt-in decode-parity mode to the Triton multi-token recurrence (#42251) Co-authored-by: merrymercy Co-authored-by: jiayisuse --- .../fla/fused_sigmoid_gating_recurrent.py | 158 ++++++++++++++++-- test/registered/attention/test_kda_kernels.py | 116 +++++++++++++ 2 files changed, 260 insertions(+), 14 deletions(-) diff --git a/python/sglang/kernels/ops/attention/fla/fused_sigmoid_gating_recurrent.py b/python/sglang/kernels/ops/attention/fla/fused_sigmoid_gating_recurrent.py index 1fdfe7df1c..da6d8b265a 100644 --- a/python/sglang/kernels/ops/attention/fla/fused_sigmoid_gating_recurrent.py +++ b/python/sglang/kernels/ops/attention/fla/fused_sigmoid_gating_recurrent.py @@ -49,6 +49,73 @@ def _select_recurrent_launch_config( return min(triton.next_power_of_2(v), 32), 1 +_EX2_APPROX_FTZ_F32 = tl.constexpr("ex2.approx.ftz.f32 $0, $1;") +_LG2_APPROX_FTZ_F32 = tl.constexpr("lg2.approx.ftz.f32 $0, $1;") +_UNARY_F32_REGISTERS = tl.constexpr("=f,f") + + +@triton.jit +def _ex2_approx_ftz_f32(x): + return tl.inline_asm_elementwise( + asm=_EX2_APPROX_FTZ_F32, + constraints=_UNARY_F32_REGISTERS, + args=(x,), + dtype=tl.float32, + is_pure=True, + pack=1, + ) + + +@triton.jit +def _lg2_approx_ftz_f32(x): + return tl.inline_asm_elementwise( + asm=_LG2_APPROX_FTZ_F32, + constraints=_UNARY_F32_REGISTERS, + args=(x,), + dtype=tl.float32, + is_pure=True, + pack=1, + ) + + +@triton.jit +def _cutedsl_exp(x): + return _ex2_approx_ftz_f32(x * 1.4426950408889634) + + +@triton.jit +def _cutedsl_qk_sum_squares(x, BK: tl.constexpr): + """Lane-striped Q/K sum of squares, as FlashInfer's ``recurrent_kda`` CuTe + DSL decode kernel reduces it: lane ``l`` owns the elements ``l + i * 32`` + and a 32-lane butterfly combines the lanes. Keeping both reduction + dimensions explicit stops Triton from flattening K into a different + reduction tree. + """ + NS: tl.constexpr = BK // 32 + tl.static_assert( + NS >= 1 and NS <= 4, + "match_cutedsl_decode needs 32 <= BK <= 128 (head_k_dim <= 128)", + ) + # The decode kernel accumulates its per-lane squares with packed f32x2 FMA, + # so the even and odd i streams stay separate until both are complete. A + # plain tl.sum(x * x) is mathematically equivalent, but can move a BF16 + # recurrent state across a rounding boundary after several verify steps. + lane_values = tl.reshape(x, (NS, 32), can_reorder=False) + row = tl.arange(0, NS)[:, None] + even0 = tl.sum(tl.where(row == 0, lane_values, 0.0), axis=0) + odd0 = tl.sum(tl.where(row == 1, lane_values, 0.0), axis=0) + zeros = tl.zeros((32,), tl.float32) + even_sum = tl.fma(even0, even0, zeros) + odd_sum = tl.fma(odd0, odd0, zeros) + if NS == 4: + even1 = tl.sum(tl.where(row == 2, lane_values, 0.0), axis=0) + odd1 = tl.sum(tl.where(row == 3, lane_values, 0.0), axis=0) + even_sum = tl.fma(even1, even1, even_sum) + odd_sum = tl.fma(odd1, odd1, odd_sum) + lane_sum = even_sum + odd_sum + return tl.sum(lane_sum, axis=0) + + @triton.jit(do_not_specialize=["T"]) def fused_sigmoid_gating_delta_rule_update_kernel( A_log, @@ -113,6 +180,8 @@ def fused_sigmoid_gating_delta_rule_update_kernel( CACHE_RING: tl.constexpr = False, SPLIT_N_HV_GRID: tl.constexpr = False, USE_GDC: tl.constexpr = False, + MATCH_CUTEDSL_DECODE: tl.constexpr = False, + ROUND_STATE_TO_BF16: tl.constexpr = False, ): """ Fused kernel that combines sigmoid gating computation with recurrent delta rule update. @@ -247,18 +316,32 @@ def fused_sigmoid_gating_delta_rule_update_kernel( # KDA safe gate: lower_bound * sigmoid(exp(A_log) * (a + dt_bias)) b_g = lower_bound * tl.sigmoid(tl.exp(b_A_log) * x) else: - # Compute g = -exp(A_log) * softplus(a + dt_bias) - beta_x = softplus_beta * x - # Apply softplus with numerical stability - softplus_x = tl.where( - beta_x <= softplus_threshold, - (1.0 / softplus_beta) * tl.log(1.0 + tl.exp(beta_x)), - x, - ) - b_g = -tl.exp(b_A_log) * softplus_x + if MATCH_CUTEDSL_DECODE and IS_KDA: + # The CuTe decode kernel forms the multiplicative gate directly + # in base 2. Algebraically this is exp(-A * softplus(x)), but + # the instruction sequence decides on which side of a BF16 + # state-rounding boundary the result lands. + b_gate_multiplier = _ex2_approx_ftz_f32( + (-_cutedsl_exp(b_A_log)) + * _lg2_approx_ftz_f32(1.0 + _cutedsl_exp(x)) + ) + b_g = tl.log(b_gate_multiplier) + else: + # Compute g = -exp(A_log) * softplus(a + dt_bias) + beta_x = softplus_beta * x + # Apply softplus with numerical stability + softplus_x = tl.where( + beta_x <= softplus_threshold, + (1.0 / softplus_beta) * tl.log(1.0 + tl.exp(beta_x)), + x, + ) + b_g = -tl.exp(b_A_log) * softplus_x # Compute beta = sigmoid(b) - b_beta = 1.0 / (1.0 + tl.exp(-b_b)) + if MATCH_CUTEDSL_DECODE: + b_beta = b_b + else: + b_beta = 1.0 / (1.0 + tl.exp(-b_b)) # fused ring-write: stash this step's raw inputs + in-kernel gate/beta # into the per-slot ring for the commit fold to replay. Must sit here -- @@ -325,14 +408,30 @@ def fused_sigmoid_gating_delta_rule_update_kernel( # Apply L2 normalization if enabled if USE_QK_L2NORM_IN_KERNEL: - b_q = b_q / (tl.sqrt(tl.sum(b_q * b_q) + 1e-6)) - b_k = b_k / (tl.sqrt(tl.sum(b_k * b_k) + 1e-6)) + if MATCH_CUTEDSL_DECODE: + q_sum_sq = _cutedsl_qk_sum_squares(b_q, BK) + k_sum_sq = _cutedsl_qk_sum_squares(b_k, BK) + b_q *= tl.rsqrt(q_sum_sq + 1e-6) + b_k *= tl.rsqrt(k_sum_sq + 1e-6) + else: + b_q = b_q / (tl.sqrt(tl.sum(b_q * b_q) + 1e-6)) + b_k = b_k / (tl.sqrt(tl.sum(b_k * b_k) + 1e-6)) b_q = b_q * scale # Apply gating to hidden state: h *= exp(g) if IS_KDA: - b_h *= tl.exp(b_g[:, None]) + if MATCH_CUTEDSL_DECODE: + # The CuTe decode kernel forms the decayed state with packed + # FMA and a zero addend. Keep that rounding instead of letting + # the compiler pick a plain multiply. + b_h = tl.fma( + b_h, + b_gate_multiplier[:, None], + tl.zeros((BK, BV), tl.float32), + ) + else: + b_h *= tl.exp(b_g[:, None]) else: b_h *= tl.exp(b_g) @@ -343,7 +442,10 @@ def fused_sigmoid_gating_delta_rule_update_kernel( b_v *= b_beta # Update hidden state: h += k[:, None] * v[None, :] - b_h += b_k[:, None] * b_v[None, :] + if MATCH_CUTEDSL_DECODE: + b_h = tl.fma(b_k[:, None], b_v[None, :], b_h) + else: + b_h += b_k[:, None] * b_v[None, :] # Compute output: o = sum(h * q, dim=0) b_o = tl.sum(b_h * b_q[:, None], 0) @@ -363,6 +465,13 @@ def fused_sigmoid_gating_delta_rule_update_kernel( ) tl.store(cache_ptr, b_h.to(cache_ptr.dtype.element_ty), mask=mask_h) + # A BF16 recurrent-state decode persists its state after every token. + # Multi-token verification must make that precision boundary visible to + # the next recurrence step instead of retaining the FP32 accumulator for + # the whole candidate chain. FP32 state pools must retain the accumulator. + if ROUND_STATE_TO_BF16: + b_h = b_h.to(tl.bfloat16).to(tl.float32) + step_idx += 1 # Update pointers for next timestep @@ -419,6 +528,12 @@ def fused_sigmoid_gating_delta_rule_update( replayssm_rawk: Optional[torch.Tensor] = None, replayssm_g: Optional[torch.Tensor] = None, replayssm_beta: Optional[torch.Tensor] = None, + # Reproduce FlashInfer's `recurrent_kda` CuTe DSL decode kernel step for + # step: `b` is then the already-sigmoided FP32 beta, the gate is formed in + # base 2 with ex2/lg2.approx, Q/K norms reduce lane-striped, the state + # decays and updates through FMA, and a BF16 state pool is rounded after + # every token. + match_cutedsl_decode: bool = False, ): """ Fused triton implementation of sigmoid gating delta rule update. @@ -430,6 +545,11 @@ def fused_sigmoid_gating_delta_rule_update( - target_verify: multi-step with intermediate state caching, optional tree attention, and optional state update disable """ + if match_cutedsl_decode and (not is_kda or lower_bound is not None): + raise ValueError( + "match_cutedsl_decode is defined for the KDA softplus gate only " + "(is_kda=True, lower_bound=None)." + ) B, T, H, K, V = *k.shape, v.shape[-1] stride_q = q.stride()[1] stride_k = k.stride()[1] @@ -508,6 +628,14 @@ def fused_sigmoid_gating_delta_rule_update( # KDA and GDN recurrent paths. pdl_kwargs = {"USE_GDC": True, "launch_pdl": True} if is_arch_support_pdl() else {} + # CuTeDSL specializes on the persisted state dtype. Its BF16 decode path + # writes BF16 after every token and reloads FP32 for the next recurrence; + # its FP32 path has no such rounding boundary. Keep CuTe math parity and + # state-persistence precision as separate compile-time choices. + round_state_to_bf16 = match_cutedsl_decode and ( + initial_state_source is None or initial_state_source.dtype == torch.bfloat16 + ) + fused_sigmoid_gating_delta_rule_update_kernel[grid]( A_log=A_log, a=a, @@ -568,6 +696,8 @@ def fused_sigmoid_gating_delta_rule_update( MAX_CACHE_LEN=max_cache_len, CACHE_RING=cache_ring, SPLIT_N_HV_GRID=split_n_hv_grid, + MATCH_CUTEDSL_DECODE=match_cutedsl_decode, + ROUND_STATE_TO_BF16=round_state_to_bf16, num_warps=num_warps, num_stages=num_stages, **pdl_kwargs, diff --git a/test/registered/attention/test_kda_kernels.py b/test/registered/attention/test_kda_kernels.py index 83b8d6de4c..6efc99aead 100644 --- a/test/registered/attention/test_kda_kernels.py +++ b/test/registered/attention/test_kda_kernels.py @@ -647,5 +647,121 @@ class TestKDAPackedDecode(unittest.TestCase): ) +@unittest.skipIf(not torch.cuda.is_available(), "Test requires CUDA") +class TestKDAVerifyDecodeParity(CustomTestCase): + """With ``match_cutedsl_decode`` a T-token verify launch must reproduce the + T single-token launches a decode loop runs over a BF16 state pool, bit for + bit; the default path carries FP32 state across the chain and drifts.""" + + @staticmethod + def _run_decode_loop_and_verify(B, T, H, K, state_dtype, match_decode): + torch.manual_seed(1234) + device = get_device() + q = torch.randn(1, B * T, H, K, dtype=torch.bfloat16, device=device) + k = torch.randn(1, B * T, H, K, dtype=torch.bfloat16, device=device) + v = torch.randn(1, B * T, H, K, dtype=torch.bfloat16, device=device) + a = torch.randn(B * T, H * K, dtype=torch.bfloat16, device=device) + b = torch.randn(1, B * T, H, dtype=torch.bfloat16, device=device) + if match_decode: + # The parity contract: beta arrives already sigmoided in FP32. + b = b.float().sigmoid() + A_log = torch.randn(H, dtype=torch.float32, device=device) + dt_bias = torch.randn(H * K, dtype=torch.float32, device=device) + num_slots = B + 2 + pool = torch.randn(num_slots, H, K, K, dtype=torch.float32, device=device).to( + state_dtype + ) + slots = torch.arange(B, dtype=torch.int32, device=device) + common = dict( + A_log=A_log, + dt_bias=dt_bias, + initial_state_indices=slots, + use_qk_l2norm_in_kernel=True, + softplus_beta=1.0, + softplus_threshold=20.0, + is_kda=True, + match_cutedsl_decode=match_decode, + ) + + # Token t of request i sits at row i * T + t. + decode_pool = pool.clone() + decode_cu_seqlens = torch.arange(B + 1, dtype=torch.int32, device=device) + decode_outputs, decode_states = [], [] + for t in range(T): + rows = [i * T + t for i in range(B)] + decode_outputs.append( + fused_sigmoid_gating_delta_rule_update( + q=q[:, rows].contiguous(), + k=k[:, rows].contiguous(), + v=v[:, rows].contiguous(), + a=a[rows].contiguous(), + b=b[:, rows].contiguous(), + initial_state_source=decode_pool, + cu_seqlens=decode_cu_seqlens, + **common, + ) + ) + decode_states.append(decode_pool[slots].clone()) + decode_output = torch.stack(decode_outputs, dim=2).reshape(1, B * T, H, K) + decode_state = torch.stack(decode_states, dim=1) + + verify_pool = pool.clone() + verify_state = torch.zeros( + num_slots, T, H, K, K, dtype=state_dtype, device=device + ) + verify_output = fused_sigmoid_gating_delta_rule_update( + q=q, + k=k, + v=v, + a=a, + b=b, + initial_state_source=verify_pool, + cu_seqlens=torch.arange(0, B * T + 1, T, dtype=torch.int32, device=device), + disable_state_update=True, + intermediate_states_buffer=verify_state, + intermediate_state_indices=slots, + cache_steps=T, + retrieve_parent_token=None, + **common, + ) + return decode_output, decode_state, verify_output, verify_state[slots] + + def test_bf16_state_verify_matches_single_token_launches(self): + for K in (128, 64): + with self.subTest(head_dim=K): + decode_output, decode_state, verify_output, verify_state = ( + self._run_decode_loop_and_verify( + B=2, + T=16, + H=4, + K=K, + state_dtype=torch.bfloat16, + match_decode=True, + ) + ) + self.assertTrue(torch.equal(verify_output, decode_output)) + self.assertTrue(torch.equal(verify_state, decode_state)) + + def test_default_path_keeps_fp32_state_across_the_chain(self): + decode_output, decode_state, verify_output, verify_state = ( + self._run_decode_loop_and_verify( + B=2, T=16, H=4, K=128, state_dtype=torch.bfloat16, match_decode=False + ) + ) + self.assertFalse(torch.equal(verify_state, decode_state)) + # Still the same recurrence: the drift is BF16 rounding, not a bug. + torch.testing.assert_close( + verify_output.float(), decode_output.float(), atol=2e-2, rtol=1e-2 + ) + + def test_fp32_state_pool_is_not_rounded(self): + _, decode_state, _, verify_state = self._run_decode_loop_and_verify( + B=2, T=16, H=4, K=128, state_dtype=torch.float32, match_decode=True + ) + self.assertTrue(torch.equal(verify_state, decode_state)) + rounded = verify_state.to(torch.bfloat16).to(torch.float32) + self.assertFalse(torch.equal(verify_state, rounded)) + + if __name__ == "__main__": unittest.main()