diff --git a/python/sglang/kernels/ops/attention/cutedsl_gdn.py b/python/sglang/kernels/ops/attention/cutedsl_gdn.py index 968d0cef03..f7241c4885 100644 --- a/python/sglang/kernels/ops/attention/cutedsl_gdn.py +++ b/python/sglang/kernels/ops/attention/cutedsl_gdn.py @@ -15,10 +15,11 @@ logger = logging.getLogger(__name__) _compiled_kernels: Dict[Tuple, object] = {} _cu_seqlens_cache: Dict[Tuple, torch.Tensor] = {} TILE_K = 128 +# K-contiguous smem rows; the 8-float pad makes each warp's 8 K x 4 V reads +# hit distinct banks and keeps rows 16B-aligned for cp.async. +TILE_K_PADDED = 136 TILE_V = 32 -TILE_V_PADDED = 36 TILE_V_SMALL = 16 -TILE_V_SMALL_PADDED = 20 NUM_STAGES = 2 NUM_THREADS = 128 NUM_BLOCKS_PER_STATE_SMALL = 8 @@ -262,12 +263,11 @@ def _define_kernels(): for k_iter in range(NUM_K_ITERS_SMALL): flat_idx = tidx + k_iter * 128 - k_write = flat_idx // TILE_V_SMALL - v_write = flat_idx % TILE_V_SMALL - if k_write < TILE_K: - h_val = sData[(k_write, v_write, stage)] - v_global_write = v_tile * TILE_V_SMALL + v_write - h0_source[(pool_idx, i_hv, k_write, v_global_write)] = h_val + k_write = flat_idx % TILE_K + v_write = flat_idx // TILE_K + h_val = sData[(k_write, v_write, stage)] + v_global_write = v_tile * TILE_V_SMALL + v_write + h0_source[(pool_idx, i_hv, k_write, v_global_write)] = h_val cute.arch.barrier() @@ -495,12 +495,11 @@ def _define_kernels(): for k_iter in range(NUM_K_ITERS_SMALL): flat_idx = tidx + k_iter * 128 - k_write = flat_idx // TILE_V_SMALL - v_write = flat_idx % TILE_V_SMALL - if k_write < TILE_K: - h_val = sData[(k_write, v_write, stage)] - v_global_write = v_tile * TILE_V_SMALL + v_write - h0_source[(pool_idx, i_hv, k_write, v_global_write)] = h_val + k_write = flat_idx % TILE_K + v_write = flat_idx // TILE_K + h_val = sData[(k_write, v_write, stage)] + v_global_write = v_tile * TILE_V_SMALL + v_write + h0_source[(pool_idx, i_hv, k_write, v_global_write)] = h_val cute.arch.barrier() @@ -713,12 +712,11 @@ def _define_kernels(): for k_iter in range(NUM_K_ITERS): flat_idx = tidx + k_iter * 256 - k_write = flat_idx // TILE_V - v_write = flat_idx % TILE_V - if k_write < TILE_K: - h_val = sData[(k_write, v_write, stage)] - v_global_write = v_tile * TILE_V + v_write - h0_source[(pool_idx, i_hv, k_write, v_global_write)] = h_val + k_write = flat_idx % TILE_K + v_write = flat_idx // TILE_K + h_val = sData[(k_write, v_write, stage)] + v_global_write = v_tile * TILE_V + v_write + h0_source[(pool_idx, i_hv, k_write, v_global_write)] = h_val cute.arch.barrier() @@ -931,12 +929,11 @@ def _define_kernels(): for k_iter in range(NUM_K_ITERS): flat_idx = tidx + k_iter * 256 - k_write = flat_idx // TILE_V - v_write = flat_idx % TILE_V - if k_write < TILE_K: - h_val = sData[(k_write, v_write, stage)] - v_global_write = v_tile * TILE_V + v_write - h0_source[(pool_idx, i_hv, k_write, v_global_write)] = h_val + k_write = flat_idx % TILE_K + v_write = flat_idx // TILE_K + h_val = sData[(k_write, v_write, stage)] + v_global_write = v_tile * TILE_V + v_write + h0_source[(pool_idx, i_hv, k_write, v_global_write)] = h_val cute.arch.barrier() @@ -991,15 +988,15 @@ def _create_jit_functions(): num_v_tiles_small = cute.ceil_div(v_dim, TILE_V_SMALL) smem_layout_small = cute.make_layout( (TILE_K, TILE_V_SMALL, NUM_STAGES), - stride=(TILE_V_SMALL_PADDED, 1, TILE_K * TILE_V_SMALL_PADDED), + stride=(1, TILE_K_PADDED, TILE_K_PADDED * TILE_V_SMALL), ) - thread_layout_small = cute.make_layout((32, 4), stride=(4, 1)) - val_layout_small = cute.make_layout((1, 4)) + thread_layout_small = cute.make_layout((32, 4), stride=(1, 32)) + val_layout_small = cute.make_layout((4, 1)) tiled_copy_load_small = cute.make_tiled_copy_tv( copy_atom, thread_layout_small, val_layout_small ) smem_bytes_small = ( - 4 * TILE_K * TILE_V_SMALL_PADDED * NUM_STAGES + 4 * TILE_K_PADDED * TILE_V_SMALL * NUM_STAGES + 4 * TILE_V_SMALL + 4 * TILE_K * 2 + 64 @@ -1070,15 +1067,15 @@ def _create_jit_functions(): num_v_tiles_small = cute.ceil_div(v_dim, TILE_V_SMALL) smem_layout_small = cute.make_layout( (TILE_K, TILE_V_SMALL, NUM_STAGES), - stride=(TILE_V_SMALL_PADDED, 1, TILE_K * TILE_V_SMALL_PADDED), + stride=(1, TILE_K_PADDED, TILE_K_PADDED * TILE_V_SMALL), ) - thread_layout_small = cute.make_layout((32, 4), stride=(4, 1)) - val_layout_small = cute.make_layout((1, 4)) + thread_layout_small = cute.make_layout((32, 4), stride=(1, 32)) + val_layout_small = cute.make_layout((4, 1)) tiled_copy_load_small = cute.make_tiled_copy_tv( copy_atom, thread_layout_small, val_layout_small ) smem_bytes_small = ( - 4 * TILE_K * TILE_V_SMALL_PADDED * NUM_STAGES + 4 * TILE_K_PADDED * TILE_V_SMALL * NUM_STAGES + 4 * TILE_V_SMALL + 4 * TILE_K * 2 + 64 @@ -1149,13 +1146,13 @@ def _create_jit_functions(): num_v_tiles = cute.ceil_div(v_dim, TILE_V) base_smem_layout = cute.make_layout( (TILE_K, TILE_V, NUM_STAGES), - stride=(TILE_V_PADDED, 1, TILE_K * TILE_V_PADDED), + stride=(1, TILE_K_PADDED, TILE_K_PADDED * TILE_V), ) - thread_layout = cute.make_layout((32, 8), stride=(8, 1)) - val_layout = cute.make_layout((1, 4)) + thread_layout = cute.make_layout((32, 8), stride=(1, 32)) + val_layout = cute.make_layout((4, 1)) tiled_copy_load = cute.make_tiled_copy_tv(copy_atom, thread_layout, val_layout) smem_bytes = ( - 4 * TILE_K * TILE_V_PADDED * NUM_STAGES + 4 * TILE_V + 4 * TILE_K * 2 + 64 + 4 * TILE_K_PADDED * TILE_V * NUM_STAGES + 4 * TILE_V + 4 * TILE_K * 2 + 64 ) gdn_large( @@ -1223,13 +1220,13 @@ def _create_jit_functions(): num_v_tiles = cute.ceil_div(v_dim, TILE_V) base_smem_layout = cute.make_layout( (TILE_K, TILE_V, NUM_STAGES), - stride=(TILE_V_PADDED, 1, TILE_K * TILE_V_PADDED), + stride=(1, TILE_K_PADDED, TILE_K_PADDED * TILE_V), ) - thread_layout = cute.make_layout((32, 8), stride=(8, 1)) - val_layout = cute.make_layout((1, 4)) + thread_layout = cute.make_layout((32, 8), stride=(1, 32)) + val_layout = cute.make_layout((4, 1)) tiled_copy_load = cute.make_tiled_copy_tv(copy_atom, thread_layout, val_layout) smem_bytes = ( - 4 * TILE_K * TILE_V_PADDED * NUM_STAGES + 4 * TILE_V + 4 * TILE_K * 2 + 64 + 4 * TILE_K_PADDED * TILE_V * NUM_STAGES + 4 * TILE_V + 4 * TILE_K * 2 + 64 ) gdn_large_varlen( @@ -1304,7 +1301,10 @@ def _get_compiled_kernel(N, H, HV, K, V, pool_size, use_small_batch, is_varlen_d A_log = torch.zeros(HV, dtype=torch.float32, device="cuda") dt_bias = torch.zeros(HV, dtype=torch.bfloat16, device="cuda") - h0_source = torch.zeros(pool_size, HV, K, V, dtype=torch.float32, device="cuda") + # from_dlpack bakes this layout into the kernel; it must match the runtime view. + h0_source = torch.zeros( + pool_size, HV, V, K, dtype=torch.float32, device="cuda" + ).transpose(-1, -2) h0_indices = torch.zeros(N, dtype=torch.int32, device="cuda") cu_seqlens_tensor = from_dlpack(cu_seqlens, assumed_align=16) @@ -1386,7 +1386,10 @@ def cutedsl_fused_sigmoid_gating_delta_rule_update( softplus_beta: float = 1.0, softplus_threshold: float = 20.0, ) -> torch.Tensor: - """CuTe DSL implementation of fused sigmoid gating delta rule update.""" + """CuTe DSL implementation of fused sigmoid gating delta rule update. + + ``initial_state_source`` is the ``[pool_size, HV, V, K]`` state pool. + """ B_q, T_q, H, K = q.shape HV = v.shape[2] @@ -1401,7 +1404,7 @@ def cutedsl_fused_sigmoid_gating_delta_rule_update( if initial_state_source.dim() == 1: pool_size = initial_state_source.numel() // (HV * K * V) - h0_source = initial_state_source.view(pool_size, HV, K, V) + h0_source = initial_state_source.view(pool_size, HV, V, K) elif initial_state_source.dim() == 4: pool_size = initial_state_source.shape[0] h0_source = initial_state_source @@ -1409,6 +1412,8 @@ def cutedsl_fused_sigmoid_gating_delta_rule_update( raise ValueError( f"Unexpected initial_state_source shape: {initial_state_source.shape}" ) + # The kernels index (pool, HV, K, V). + h0_source = h0_source.transpose(-1, -2) if is_varlen_decode: if a.dim() == 3: @@ -1462,7 +1467,7 @@ def cutedsl_fused_sigmoid_gating_delta_rule_update( ).mark_layout_dynamic(leading_dim=0) h0_source_tensor = from_dlpack( h0_source.detach(), assumed_align=16 - ).mark_layout_dynamic(leading_dim=h0_source.ndim - 1) + ).mark_layout_dynamic(leading_dim=2) h0_indices_tensor = from_dlpack( initial_state_indices.detach(), assumed_align=16 ).mark_layout_dynamic(leading_dim=0) diff --git a/test/registered/kernels/ops/attention/test_cutedsl_gdn.py b/test/registered/kernels/ops/attention/test_cutedsl_gdn.py index 4887e7607e..d7cae7dbe7 100644 --- a/test/registered/kernels/ops/attention/test_cutedsl_gdn.py +++ b/test/registered/kernels/ops/attention/test_cutedsl_gdn.py @@ -29,10 +29,12 @@ try: except ImportError: TRITON_AVAILABLE = False -register_cuda_ci(est_time=5, stage="base-b-kernel-unit", runner_config="1-gpu-large") +register_cuda_ci(est_time=10, stage="base-b-kernel-unit", runner_config="1-gpu-large") -def run_triton_kernel(A_log, dt_bias, q, k, v, a, b, initial_state, indices, scale): +def run_triton_kernel( + A_log, dt_bias, q, k, v, a, b, initial_state, indices, scale, cu_seqlens=None +): return fused_sigmoid_gating_delta_rule_update( A_log=A_log, a=a, @@ -47,67 +49,57 @@ def run_triton_kernel(A_log, dt_bias, q, k, v, a, b, initial_state, indices, sca initial_state_indices=indices, scale=scale, use_qk_l2norm_in_kernel=True, - cu_seqlens=None, + cu_seqlens=cu_seqlens, ) @pytest.mark.skipif(not CUTEDSL_AVAILABLE, reason="CuTe DSL not available") @pytest.mark.skipif(not TRITON_AVAILABLE, reason="Triton kernel not available") -@pytest.mark.skip( - reason=( - "Temporary CI workaround: CuTe DSL GDN precision is currently unstable " - "against the Triton reference and needs follow-up investigation." - ) -) +@pytest.mark.parametrize("pool_view", ["4d", "1d"]) +@pytest.mark.parametrize("varlen", [False, True]) @pytest.mark.parametrize("B", [16, 128]) -def test_cutedsl_gdn_precision(B: int): - """Test precision of CuTe DSL GDN kernel against Triton reference.""" +def test_cutedsl_gdn_precision(B: int, varlen: bool, pool_view: str): + """CuTe DSL decode must read and update the Triton [pool, HV, V, K] state pool.""" torch.manual_seed(2025) T, H, K, V, HV = 1, 16, 128, 128, 32 scale = K**-0.5 + lead = (1, B) if varlen else (B, T) + gate_shape = (B, HV) if varlen else (B, T, HV) A_log = torch.randn(HV, dtype=torch.float32, device="cuda") dt_bias = torch.randn(HV, dtype=torch.bfloat16, device="cuda") - a = torch.randn(B, T, HV, dtype=torch.bfloat16, device="cuda") - b = torch.randn(B, T, HV, dtype=torch.bfloat16, device="cuda") - q = torch.randn(B, T, H, K, dtype=torch.bfloat16, device="cuda") - k = torch.randn(B, T, H, K, dtype=torch.bfloat16, device="cuda") - v = torch.randn(B, T, HV, V, dtype=torch.bfloat16, device="cuda") - indices = torch.arange(B, dtype=torch.int32, device="cuda") - state_cutedsl = torch.randn(B, HV, K, V, dtype=torch.float32, device="cuda") - state_triton = state_cutedsl.clone().reshape(-1).contiguous() - - # Warmup compilation - _ = cutedsl_gdn.cutedsl_fused_sigmoid_gating_delta_rule_update( - A_log, dt_bias, q, k, v, a, b, state_cutedsl.clone(), indices, scale=scale + pool_size = 2 * B + 1 + indices = torch.randperm(pool_size, device="cuda")[:B].to(torch.int32) + indices[B // 2] = -1 + valid = indices >= 0 + cu_seqlens = ( + torch.arange(B + 1, dtype=torch.int32, device="cuda") if varlen else None ) - torch.cuda.synchronize() + state_cutedsl = torch.randn(pool_size, HV, V, K, dtype=torch.float32, device="cuda") + state_triton = state_cutedsl.clone() + pool = state_cutedsl.view(-1) if pool_view == "1d" else state_cutedsl - # Fresh state for actual test - state_cutedsl = torch.randn(B, HV, K, V, dtype=torch.float32, device="cuda") - state_triton = state_cutedsl.clone().reshape(-1).contiguous() + for _ in range(3): + a = torch.randn(*gate_shape, dtype=torch.bfloat16, device="cuda") + b = torch.randn(*gate_shape, dtype=torch.bfloat16, device="cuda") + q = torch.randn(*lead, H, K, dtype=torch.bfloat16, device="cuda") + k = torch.randn(*lead, H, K, dtype=torch.bfloat16, device="cuda") + v = torch.randn(*lead, HV, V, dtype=torch.bfloat16, device="cuda") - out_cutedsl = cutedsl_gdn.cutedsl_fused_sigmoid_gating_delta_rule_update( - A_log, dt_bias, q, k, v, a, b, state_cutedsl, indices, scale=scale - ) - out_triton = run_triton_kernel( - A_log, dt_bias, q, k, v, a, b, state_triton, indices, scale - ) + out_cutedsl = cutedsl_gdn.cutedsl_fused_sigmoid_gating_delta_rule_update( + A_log, dt_bias, q, k, v, a, b, pool, indices, cu_seqlens, scale=scale + ) + out_triton = run_triton_kernel( + A_log, dt_bias, q, k, v, a, b, state_triton, indices, scale, cu_seqlens + ) - # Check precision: diff > 0.1 must be < 1% of elements - abs_diff = (out_triton.float() - out_cutedsl.float()).abs() - max_diff = abs_diff.max().item() - mean_diff = abs_diff.mean().item() - fail_rate = (abs_diff > 0.1).float().mean().item() * 100 - has_nan = torch.isnan(out_cutedsl).any() or torch.isinf(out_cutedsl).any() - - kernel_type = "SmallBatch" if B < 32 else "LargeBatch" - print( - f"\n B={B} ({kernel_type}): max_diff={max_diff:.2e}, mean_diff={mean_diff:.2e}, fail_rate={fail_rate:.2f}%" - ) - - assert not has_nan, "Output contains NaN/Inf" - assert fail_rate < 1.0, f"Fail rate {fail_rate:.2f}% >= 1%" + torch.testing.assert_close( + out_cutedsl.reshape(B, HV, V)[valid].float(), + out_triton.reshape(B, HV, V)[valid].float(), + atol=1e-3, + rtol=1e-2, + ) + torch.testing.assert_close(state_cutedsl, state_triton, atol=1e-4, rtol=1e-4) @pytest.mark.skipif( @@ -171,7 +163,7 @@ def test_cutedsl_gdn_performance(B: int): A_log_t = from_dlpack(A_log, assumed_align=16) dt_bias_t = from_dlpack(dt_bias, assumed_align=16) - h0_t = from_dlpack(state_cutedsl, assumed_align=16) + h0_t = from_dlpack(state_cutedsl.transpose(-1, -2), assumed_align=16) idx_t = from_dlpack(indices, assumed_align=16) o_t = from_dlpack(o_cutedsl, assumed_align=16) cu_t = from_dlpack(cu_seqlens, assumed_align=16)