diff --git a/python/sglang/kernels/ops/attention/fla/fused_sigmoid_gating_recurrent.py b/python/sglang/kernels/ops/attention/fla/fused_sigmoid_gating_recurrent.py index da6d8b265a..374182c437 100644 --- a/python/sglang/kernels/ops/attention/fla/fused_sigmoid_gating_recurrent.py +++ b/python/sglang/kernels/ops/attention/fla/fused_sigmoid_gating_recurrent.py @@ -411,11 +411,15 @@ def fused_sigmoid_gating_delta_rule_update_kernel( if MATCH_CUTEDSL_DECODE: q_sum_sq = _cutedsl_qk_sum_squares(b_q, BK) k_sum_sq = _cutedsl_qk_sum_squares(b_k, BK) - b_q *= tl.rsqrt(q_sum_sq + 1e-6) - b_k *= tl.rsqrt(k_sum_sq + 1e-6) + q_rsqrt = tl.rsqrt(q_sum_sq + 1e-6) + k_rsqrt = tl.rsqrt(k_sum_sq + 1e-6) else: - b_q = b_q / (tl.sqrt(tl.sum(b_q * b_q) + 1e-6)) - b_k = b_k / (tl.sqrt(tl.sum(b_k * b_k) + 1e-6)) + q_sum_sq = tl.sum(b_q * b_q) + k_sum_sq = tl.sum(b_k * b_k) + q_rsqrt = 1.0 / tl.sqrt(q_sum_sq + 1e-6) + k_rsqrt = 1.0 / tl.sqrt(k_sum_sq + 1e-6) + b_q *= q_rsqrt + b_k *= k_rsqrt b_q = b_q * scale diff --git a/test/registered/attention/test_kda_kernels.py b/test/registered/attention/test_kda_kernels.py index 6efc99aead..dae5b0f733 100644 --- a/test/registered/attention/test_kda_kernels.py +++ b/test/registered/attention/test_kda_kernels.py @@ -1,6 +1,8 @@ import unittest import torch +import triton +import triton.language as tl from sglang.kernels.ops.attention.fla.cumsum import chunk_local_cumsum from sglang.kernels.ops.attention.fla.fused_recurrent import ( @@ -23,6 +25,68 @@ register_cuda_ci(est_time=25, stage="base-b", runner_config="1-gpu-large") register_amd_ci(est_time=12, stage="stage-b", runner_config="1-gpu-large-amd") +@triton.jit +def _normalize_qk_reference(X, Y, K: tl.constexpr, BK: tl.constexpr): + offsets = tl.program_id(0) * K + tl.arange(0, BK) + x = tl.load(X + offsets, tl.arange(0, BK) < K, other=0).to(tl.float32) + reciprocal_norm = 1.0 / tl.sqrt(tl.sum(x * x) + 1e-6) + tl.store(Y + offsets, x * reciprocal_norm, tl.arange(0, BK) < K) + + +@unittest.skipIf(not torch.cuda.is_available(), "Test requires CUDA") +class TestRecurrentQKNormalization(unittest.TestCase): + def test_fused_matches_pre_normalized_inputs(self): + torch.manual_seed(42) + device = get_device() + for is_kda in (False, True): + for K in (32, 64, 128): + with self.subTest(is_kda=is_kda, head_dim=K): + B, T, H, V = 2, 4, 2, 32 + # Exact sums of squares isolate normalization arithmetic + # from the standalone kernel's different reduction layout. + q = torch.randint(-8, 9, (B, T, H, K), device=device).float() + k = torch.randint(-8, 9, q.shape, device=device).float() + normalized_q, normalized_k = ( + torch.empty_like(q), + torch.empty_like(k), + ) + for raw, normalized in ((q, normalized_q), (k, normalized_k)): + _normalize_qk_reference[(B * T * H,)]( + raw, normalized, K, triton.next_power_of_2(K), num_warps=1 + ) + + state = torch.randn(B, H, V, K, device=device) + reference_state = state.clone() + gate_dim = H * K if is_kda else H + common = dict( + A_log=torch.randn(H, device=device), + a=torch.randn(B, T, gate_dim, device=device), + dt_bias=torch.randn(gate_dim, device=device), + softplus_beta=1.0, + softplus_threshold=20.0, + v=torch.randn(B, T, H, V, device=device), + b=torch.randn(B, T, H, device=device), + initial_state_indices=torch.arange(B, device=device), + is_kda=is_kda, + ) + expected = fused_sigmoid_gating_delta_rule_update( + q=normalized_q, + k=normalized_k, + initial_state_source=reference_state, + use_qk_l2norm_in_kernel=False, + **common, + ) + actual = fused_sigmoid_gating_delta_rule_update( + q=q, + k=k, + initial_state_source=state, + use_qk_l2norm_in_kernel=True, + **common, + ) + self.assertTrue(torch.equal(actual, expected)) + self.assertTrue(torch.equal(state, reference_state)) + + @unittest.skipIf( not (torch.cuda.is_available() or torch.xpu.is_available()), "Test requires CUDA or XPU",