diff --git a/python/sglang/kernels/ops/attention/dsa/dequant_k_cache.py b/python/sglang/kernels/ops/attention/dsa/dequant_k_cache.py index 7ce302350d..56649c1e7a 100644 --- a/python/sglang/kernels/ops/attention/dsa/dequant_k_cache.py +++ b/python/sglang/kernels/ops/attention/dsa/dequant_k_cache.py @@ -5,13 +5,25 @@ import triton import triton.language as tl -def dequantize_k_cache(quant_k_cache): - return _dequantize_k_cache_fast_wrapped(quant_k_cache) +def _infer_dsa_dims(dim_quant: int, tile_size: int = 128) -> tuple[int, int]: + """Recover the NoPE and RoPE widths from a packed DSA cache row.""" + for dim_rope in (0, 64): + scaled_width = (dim_quant - 2 * dim_rope) * tile_size + if scaled_width % (tile_size + 4): + continue + dim_nope = scaled_width // (tile_size + 4) + if dim_nope > 0 and dim_nope % tile_size == 0: + return dim_nope, dim_rope + raise ValueError(f"Unsupported packed DSA KV row width: {dim_quant}") + + +def dequantize_k_cache(quant_k_cache, dv: int | None = None): + return _dequantize_k_cache_fast_wrapped(quant_k_cache, dv=dv) def _dequantize_k_cache_fast_wrapped( quant_k_cache: torch.Tensor, - dv: int = 512, + dv: int | None = None, tile_size: int = 128, ) -> torch.Tensor: original_ndim = quant_k_cache.ndim @@ -19,12 +31,15 @@ def _dequantize_k_cache_fast_wrapped( # set block_size = 1 quant_k_cache = quant_k_cache.unsqueeze(1) num_blocks, block_size, _, dim_quant = quant_k_cache.shape - assert dv == 512 - assert dim_quant == 656 assert tile_size == 128 + inferred_dv, dim_rope = _infer_dsa_dims(dim_quant) + if dv is None: + dv = inferred_dv + if dv != inferred_dv: + raise ValueError(f"dv={dv} does not match packed row width {dim_quant}") quant_k_cache = quant_k_cache.view((-1, dim_quant)) - output = _dequantize_k_cache_fast(quant_k_cache) + output = _dequantize_k_cache_fast(quant_k_cache, dim_nope=dv, dim_rope=dim_rope) if original_ndim == 3: return output.view(num_blocks, 1, -1) @@ -32,14 +47,17 @@ def _dequantize_k_cache_fast_wrapped( return output.view(num_blocks, block_size, 1, -1) -def _dequantize_k_cache_fast(quant_k_cache, group_size: int = 128): +def _dequantize_k_cache_fast( + quant_k_cache, + dim_nope: int, + dim_rope: int, + group_size: int = 128, +): num_tokens, dim_quant = quant_k_cache.shape assert quant_k_cache.dtype == torch.float8_e4m3fn - dim_nope = 512 - dim_rope = 64 num_tiles = dim_nope // group_size - assert dim_quant == 656 + assert dim_quant == dim_nope + num_tiles * 4 + dim_rope * 2 output = torch.empty( (num_tokens, dim_nope + dim_rope), @@ -47,8 +65,7 @@ def _dequantize_k_cache_fast(quant_k_cache, group_size: int = 128): device=quant_k_cache.device, ) - num_blocks_per_token = triton.cdiv(dim_nope + dim_rope, group_size) - assert num_blocks_per_token == 5 + num_blocks_per_token = num_tiles + triton.cdiv(dim_rope, group_size) assert dim_nope % group_size == 0 @@ -138,18 +155,14 @@ def dequantize_k_cache_paged( output: [num_tokens, 1, dim_nope + dim_rope], the de-quantized k-cache """ dim_quant = quant_k_cache.shape[-1] - assert dim_quant == 656, ( - f"dim_quant: {dim_quant} != 656 detected in dequantize_k_cache_paged" - ) + dim_nope, dim_rope = _infer_dsa_dims(dim_quant) quant_k_cache = quant_k_cache.view((-1, dim_quant)) # num_tokens can exceed kv_cache_size due to prefix sharing (multiple seqs share same KV slots) # Index bounds validated in dsa_backend.init_forward_metadata num_tokens = page_table_1_flattened.shape[0] assert quant_k_cache.dtype == torch.float8_e4m3fn - dim_nope = 512 - dim_rope = 64 - num_tiles = dim_nope // group_size # 512 // 128 = 4 + num_tiles = dim_nope // group_size output = torch.empty( (num_tokens, 1, dim_nope + dim_rope), @@ -157,9 +170,7 @@ def dequantize_k_cache_paged( device=quant_k_cache.device, ) - # cdiv(512 + 64, 128) = 5 - num_blocks_per_token = triton.cdiv(dim_nope + dim_rope, group_size) - assert num_blocks_per_token == 5 + num_blocks_per_token = num_tiles + triton.cdiv(dim_rope, group_size) assert dim_nope % group_size == 0 @@ -258,10 +269,11 @@ def gather_dequant_requant_fp8_paged( extra_rows: int = 0, out: Optional[torch.Tensor] = None, ) -> torch.Tensor: - """Gather paged fp8 KV tokens and re-pack into flat [576] fp8 layout. + """Gather paged FP8 KV tokens and re-pack into a flat raw FP8 layout. - The paged KV cache stores 656 bytes per token: - [512 nope_fp8 | 16 scales_f32 (4 groups) | 128 rope_bf16_bytes] + Each packed cache row stores: + [dim_nope FP8 bytes | 4 scale bytes per group | 2 bytes per RoPE element] + For example, 256+0 uses 264 bytes and 512+64 uses 656 bytes. This kernel gathers the requested tokens, de-quantises nope with the per-group scales, and re-quantises to per-tensor fp8 (scale=1.0). Rope is cast bf16->fp8. The whole operation is fused into a single @@ -275,7 +287,7 @@ def gather_dequant_requant_fp8_paged( persistent (dirty) buffer via ``out``. Args: - quant_k_cache: [total_num_tokens, 1, 656] fp8_e4m3fn + quant_k_cache: [total_num_tokens, 1, dim_quant] fp8_e4m3fn page_table_1_flattened: [num_tokens] int32 group_size: per-group dequant tile size (default 128) extra_rows: number of zero-filled landing-pad rows to append at @@ -283,21 +295,20 @@ def gather_dequant_requant_fp8_paged( kernel which over-reads past end-of-buffer for masked indices) out: optional pre-allocated destination of shape - [num_tokens + extra_rows, 1, 576] (or [.., 576]) fp8_e4m3fn, - contiguous. Contents may be arbitrary (fully overwritten). + [num_tokens + extra_rows, 1, dim_nope + dim_rope] (or the + equivalent 2D shape) fp8_e4m3fn, contiguous. Contents may be + arbitrary (fully overwritten). Returns: - output: [num_tokens + extra_rows, 1, 576] fp8_e4m3fn + output: [num_tokens + extra_rows, 1, dim_nope + dim_rope] fp8_e4m3fn """ dim_quant = quant_k_cache.shape[-1] - assert dim_quant == 656 + dim_nope, dim_rope = _infer_dsa_dims(dim_quant) quant_k_cache = quant_k_cache.view((-1, dim_quant)) num_tokens = page_table_1_flattened.shape[0] assert quant_k_cache.dtype == torch.float8_e4m3fn - dim_nope = 512 - dim_rope = 64 - num_tiles = dim_nope // group_size # 4 - out_dim = dim_nope + dim_rope # 576 + num_tiles = dim_nope // group_size + out_dim = dim_nope + dim_rope assert num_tiles * group_size == dim_nope total_rows = num_tokens + extra_rows @@ -417,13 +428,17 @@ def _gather_dequant_requant_fp8_paged_vec_kernel( tl.store(dst_q, y, mask=row_in_range[:, None, None]) # b. rope: [T, R] bf16 -> fp8; pad rows: 0.0 -> byte 0x00 - offs_r = tl.arange(0, DIM_ROPE) - src_r = input_rope_ptr + paged[:, None] * input_rope_stride_0 + offs_r[None, :] - data = tl.load(src_r, mask=is_real[:, None], other=0.0).to(tl.float8e4nv) - dst_r = ( - output_ptr + offs_t64[:, None] * output_stride_0 + DIM_NOPE + offs_r[None, :] - ) - tl.store(dst_r, data, mask=row_in_range[:, None]) + if DIM_ROPE > 0: + offs_r = tl.arange(0, DIM_ROPE) + src_r = input_rope_ptr + paged[:, None] * input_rope_stride_0 + offs_r[None, :] + data = tl.load(src_r, mask=is_real[:, None], other=0.0).to(tl.float8e4nv) + dst_r = ( + output_ptr + + offs_t64[:, None] * output_stride_0 + + DIM_NOPE + + offs_r[None, :] + ) + tl.store(dst_r, data, mask=row_in_range[:, None]) def gather_dequant_requant_fp8_paged_legacy( @@ -441,15 +456,13 @@ def gather_dequant_requant_fp8_paged_legacy( (token, 128-elem slice). """ dim_quant = quant_k_cache.shape[-1] - assert dim_quant == 656 + dim_nope, dim_rope = _infer_dsa_dims(dim_quant) quant_k_cache = quant_k_cache.view((-1, dim_quant)) num_tokens = page_table_1_flattened.shape[0] assert quant_k_cache.dtype == torch.float8_e4m3fn - dim_nope = 512 - dim_rope = 64 - num_tiles = dim_nope // group_size # 4 - out_dim = dim_nope + dim_rope # 576 + num_tiles = dim_nope // group_size + out_dim = dim_nope + dim_rope assert num_tiles * group_size == dim_nope total_rows = num_tokens + extra_rows @@ -462,8 +475,7 @@ def gather_dequant_requant_fp8_paged_legacy( device=quant_k_cache.device, ) - num_blocks_per_token = triton.cdiv(dim_nope + dim_rope, group_size) # 5 - assert num_blocks_per_token == 5 + num_blocks_per_token = num_tiles + triton.cdiv(dim_rope, group_size) input_nope_q = quant_k_cache[:, :dim_nope] input_nope_s = quant_k_cache[:, dim_nope : dim_nope + num_tiles * 4].view( diff --git a/python/sglang/kernels/ops/attention/dsa/quant_k_cache.py b/python/sglang/kernels/ops/attention/dsa/quant_k_cache.py index d7b1e5a50c..d2e524093e 100644 --- a/python/sglang/kernels/ops/attention/dsa/quant_k_cache.py +++ b/python/sglang/kernels/ops/attention/dsa/quant_k_cache.py @@ -54,8 +54,8 @@ def _gather_dsa_kv_scales( block_start += num_programs * BLOCK -def quantize_k_cache(cache_k): - return _quantize_k_cache_fast_wrapped(cache_k) +def quantize_k_cache(cache_k, dv: int | None = None): + return _quantize_k_cache_fast_wrapped(cache_k, dv=dv) def quantize_k_cache_separate( @@ -71,7 +71,7 @@ def quantize_k_cache_separate( Args: k_nope: (num_tokens, dim_nope) or (num_tokens, 1, dim_nope) - Must have dim_nope=512 for FP8 MLA quantization + dim_nope must be divisible by 128 k_rope: (num_tokens, dim_rope) or (num_tokens, 1, dim_rope) Must have dim_rope=64 for FP8 MLA quantization, or dim_rope=0 for no-PE MLA (empty rope); None is treated @@ -80,8 +80,8 @@ def quantize_k_cache_separate( Returns: Tuple of (nope_part, rope_part) where: - - nope_part: (num_tokens, 1, 528) as uint8 view, contains [nope_fp8(512) | scales(16)] - - rope_part: (num_tokens, 1, 128) as uint8 view, contains [rope_bf16_bytes(128)] + - nope_part: uint8 view containing [nope_fp8 | one FP32 scale per tile] + - rope_part: uint8 view containing the raw BF16 RoPE bytes (empty, (num_tokens, 1, 0), when dim_rope=0) These two tensors can be directly passed to set_mla_kv_buffer_triton(kv_buffer, loc, nope_part, rope_part) @@ -100,8 +100,10 @@ def quantize_k_cache_separate( dim_rope = k_rope_2d.shape[1] # Validate dimensions for FP8 MLA - if dim_nope != 512: - raise ValueError(f"Expected dim_nope=512 for FP8 MLA, got {dim_nope}") + if dim_nope % tile_size != 0: + raise ValueError( + f"Expected dim_nope divisible by {tile_size} for FP8 MLA, got {dim_nope}" + ) if dim_rope not in (0, 64): raise ValueError(f"Expected dim_rope=64 (or 0 for no-PE MLA), got {dim_rope}") if k_rope_2d.shape[0] != num_tokens: @@ -170,14 +172,24 @@ def _quantize_k_cache_ref( def _quantize_k_cache_fast_wrapped( input_k_cache: torch.Tensor, - dv: int = 512, + dv: int | None = None, tile_size: int = 128, ) -> torch.Tensor: # TODO the final API may be 2D instead of 4D, thus we convert them here num_blocks, block_size, _, dim_nope_and_rope = input_k_cache.shape - assert dv == 512 - assert dim_nope_and_rope == 512 + 64 assert tile_size == 128 + if dv is None: + if dim_nope_and_rope == 256: + dv = 256 + elif dim_nope_and_rope == 512 + 64: + dv = 512 + else: + raise ValueError( + "Cannot infer FP8 MLA NoPE width from total width " + f"{dim_nope_and_rope}; pass dv explicitly" + ) + if dv % tile_size != 0 or dv > dim_nope_and_rope: + raise ValueError(f"Invalid dv={dv} for input width {dim_nope_and_rope}") input_k_cache = input_k_cache.view((-1, dim_nope_and_rope)) # TODO deliberately split into two tensors, then upstream can provide the two tensors instead of concat into one @@ -201,8 +213,8 @@ def _quantize_k_cache_fast(k_nope, k_rope, group_size: int = 128): num_tokens, dim_nope = k_nope.shape num_tokens_, dim_rope = k_rope.shape assert num_tokens == num_tokens_ - assert dim_nope == 512 - assert dim_rope == 64 + assert dim_nope % group_size == 0 + assert dim_rope in (0, 64) assert k_nope.dtype == k_rope.dtype num_tiles = dim_nope // group_size @@ -218,8 +230,7 @@ def _quantize_k_cache_fast(k_nope, k_rope, group_size: int = 128): output_nope_s = output[..., dim_nope : dim_nope + num_tiles * 4].view(torch.float32) output_rope = output[..., dim_nope + num_tiles * 4 :].view(torch.bfloat16) - num_blocks_per_token = triton.cdiv(dim_nope + dim_rope, group_size) - assert num_blocks_per_token == 5 + num_blocks_per_token = num_tiles + triton.cdiv(dim_rope, group_size) assert dim_nope % group_size == 0 NUM_NOPE_BLOCKS = dim_nope // group_size @@ -252,8 +263,8 @@ def _quantize_k_cache_fast_separate(k_nope, k_rope, group_size: int = 128): This avoids packing/unpacking and enables direct use with set_mla_kv_buffer_triton. - :param k_nope: (num_tokens, dim_nope 512) bfloat16 - :param k_rope: (num_tokens, dim_rope 64) bfloat16 + :param k_nope: (num_tokens, dim_nope) bfloat16; width divisible by 128 + :param k_rope: (num_tokens, dim_rope) bfloat16; width 0 or 64 :param group_size: quantization tile size (default 128, kernel is tuned for this value) :return: Tuple of (nope_part_u8, rope_part_u8) - nope_part_u8: (num_tokens, 1, nope_part_bytes) uint8, layout [nope_fp8(dim_nope) | scales(num_tiles*4)] diff --git a/python/sglang/kernels/ops/attention/dsa/tilelang_kernel.py b/python/sglang/kernels/ops/attention/dsa/tilelang_kernel.py index 58dcd749a2..bd9a37b531 100644 --- a/python/sglang/kernels/ops/attention/dsa/tilelang_kernel.py +++ b/python/sglang/kernels/ops/attention/dsa/tilelang_kernel.py @@ -861,6 +861,7 @@ def sparse_mla_fwd_decode_partial( BI = block_I D = dim D_tail = tail_dim + has_tail = D_tail > 0 q_shape = [batch, seq_len, heads, dim + tail_dim] kv_shape = [batch, seq_len_kv, kv_group, dim + tail_dim] @@ -884,13 +885,16 @@ def sparse_mla_fwd_decode_partial( with T.Kernel(seq_len * REPLICATE_H, N_GROUPS, threads=threads) as (bx, by): if _q_in_shared: Q_buf = T.alloc_shared([H_per_block, D], dtype) - Q_tail_buf = T.alloc_shared([H_per_block, D_tail], dtype) + if has_tail: + Q_tail_buf = T.alloc_shared([H_per_block, D_tail], dtype) else: Q_buf = T.alloc_fragment([H_per_block, D], dtype) - Q_tail_buf = T.alloc_fragment([H_per_block, D_tail], dtype) + if has_tail: + Q_tail_buf = T.alloc_fragment([H_per_block, D_tail], dtype) KV_shared = T.alloc_shared([BI, D], dtype) - K_tail_shared = T.alloc_shared([BI, D_tail], dtype) + if has_tail: + K_tail_shared = T.alloc_shared([BI, D_tail], dtype) S_shared = T.alloc_shared([H_per_block, BI], dtype) mask = T.alloc_fragment([BI], T.bool) @@ -913,7 +917,8 @@ def sparse_mla_fwd_decode_partial( H1 = H0 + H_per_block T.copy(Q[b_i, s_i, H0:H1, :D], Q_buf) - T.copy(Q[b_i, s_i, H0:H1, D:], Q_tail_buf) + if has_tail: + T.copy(Q[b_i, s_i, H0:H1, D:], Q_tail_buf) for k_i in T.Pipelined(inner_iter, num_stages=num_stages): topk_block_i = group_i * inner_iter + k_i @@ -925,11 +930,12 @@ def sparse_mla_fwd_decode_partial( KV_shared[bi_i, d_i] = KV[ b_i, T.if_then_else(idx >= 0, idx, 0), g_i, d_i ] - for bi_i, d_i in T.Parallel(BI, D_tail): - idx = Indices[b_i, s_i, g_i, topk_block_i * BI + bi_i] - K_tail_shared[bi_i, d_i] = KV[ - b_i, T.if_then_else(idx >= 0, idx, 0), g_i, D + d_i - ] + if has_tail: + for bi_i, d_i in T.Parallel(BI, D_tail): + idx = Indices[b_i, s_i, g_i, topk_block_i * BI + bi_i] + K_tail_shared[bi_i, d_i] = KV[ + b_i, T.if_then_else(idx >= 0, idx, 0), g_i, D + d_i + ] for h_i, bi_i in T.Parallel(H_per_block, BI): acc_s[h_i, bi_i] = T.if_then_else( @@ -943,13 +949,14 @@ def sparse_mla_fwd_decode_partial( transpose_B=True, policy=T.GemmWarpPolicy.FullCol, ) - T.gemm( - Q_tail_buf, - K_tail_shared, - acc_s, - transpose_B=True, - policy=T.GemmWarpPolicy.FullCol, - ) + if has_tail: + T.gemm( + Q_tail_buf, + K_tail_shared, + acc_s, + transpose_B=True, + policy=T.GemmWarpPolicy.FullCol, + ) T.copy(m_i, m_i_prev) T.reduce_max(acc_s, m_i, dim=1, clear=False) @@ -1085,7 +1092,7 @@ def sparse_mla_fwd_decode_partial_fp8( inner_iter=1, threads=256, ): - assert d_v == 512, f"only support d_v=512" + assert d_v in (256, 512), f"only support d_v=256 or d_v=512, got {d_v}" assert topk % block_I == 0, ( "otherwise will load some index=0 thus causing wrong kv to be loaded" ) @@ -1100,6 +1107,10 @@ def sparse_mla_fwd_decode_partial_fp8( BI = block_I group_size = 128 + assert d_v % group_size == 0 + assert d_tail in (0, 64), f"only support d_tail=0 or d_tail=64, got {d_tail}" + num_main_tiles = d_v // group_size + has_tail = d_tail > 0 dim_quant_fp8 = d_v + d_tail rope_offset_fp8 = d_v n_groups = topk // (BI * inner_iter) @@ -1149,19 +1160,21 @@ def sparse_mla_fwd_decode_partial_fp8( H0 = (bx % head_blocks_per_seq) * h_per_block H1 = H0 + h_per_block - # We intentionally split the K=512 GEMM into 4x128 tiles. + # Split the main-dimension GEMM into 128-wide tiles. # Although this adds extra intermediate memory traffic, # it shortens the MFMA accumulation dependency chain and improves performance. q_tile0 = T.alloc_shared([h_per_block, group_size], fp8_dtype) q_tile1 = T.alloc_shared([h_per_block, group_size], fp8_dtype) - q_tile2 = T.alloc_shared([h_per_block, group_size], fp8_dtype) - q_tile3 = T.alloc_shared([h_per_block, group_size], fp8_dtype) kv_tile0 = T.alloc_shared([BI, group_size], fp8_dtype) kv_tile1 = T.alloc_shared([BI, group_size], fp8_dtype) - kv_tile2 = T.alloc_shared([BI, group_size], fp8_dtype) - kv_tile3 = T.alloc_shared([BI, group_size], fp8_dtype) - q_tail_buf = T.alloc_shared([h_per_block, d_tail], fp8_dtype) - k_tail_shared = T.alloc_shared([BI, d_tail], fp8_dtype) + if num_main_tiles == 4: + q_tile2 = T.alloc_shared([h_per_block, group_size], fp8_dtype) + q_tile3 = T.alloc_shared([h_per_block, group_size], fp8_dtype) + kv_tile2 = T.alloc_shared([BI, group_size], fp8_dtype) + kv_tile3 = T.alloc_shared([BI, group_size], fp8_dtype) + if has_tail: + q_tail_buf = T.alloc_shared([h_per_block, d_tail], fp8_dtype) + k_tail_shared = T.alloc_shared([BI, d_tail], fp8_dtype) s_fp8_shared = T.alloc_shared([h_per_block, BI], fp8_dtype) page_idx_shared = T.alloc_shared([BI], T.int32) @@ -1178,21 +1191,31 @@ def sparse_mla_fwd_decode_partial_fp8( acc_o_tile0 = T.alloc_fragment([h_per_block, group_size], accum_dtype) acc_o_tile1 = T.alloc_fragment([h_per_block, group_size], accum_dtype) - acc_o_tile2 = T.alloc_fragment([h_per_block, group_size], accum_dtype) - acc_o_tile3 = T.alloc_fragment([h_per_block, group_size], accum_dtype) + if num_main_tiles == 4: + acc_o_tile2 = T.alloc_fragment([h_per_block, group_size], accum_dtype) + acc_o_tile3 = T.alloc_fragment([h_per_block, group_size], accum_dtype) T.fill(acc_o_tile0, 0) T.fill(acc_o_tile1, 0) - T.fill(acc_o_tile2, 0) - T.fill(acc_o_tile3, 0) + if num_main_tiles == 4: + T.fill(acc_o_tile2, 0) + T.fill(acc_o_tile3, 0) T.fill(sumexp, 0) T.fill(m_i, -(2**30)) - T.copy(q_fp8[b_i, s_i, H0:H1, d_v:], q_tail_buf) - T.copy(q_fp8[b_i, s_i, H0:H1, 0 * group_size : 1 * group_size], q_tile0) - T.copy(q_fp8[b_i, s_i, H0:H1, 1 * group_size : 2 * group_size], q_tile1) - T.copy(q_fp8[b_i, s_i, H0:H1, 2 * group_size : 3 * group_size], q_tile2) - T.copy(q_fp8[b_i, s_i, H0:H1, 3 * group_size : 4 * group_size], q_tile3) + if has_tail: + T.copy(q_fp8[b_i, s_i, H0:H1, d_v:], q_tail_buf) + T.copy(q_fp8[b_i, s_i, H0:H1, 0:group_size], q_tile0) + T.copy(q_fp8[b_i, s_i, H0:H1, group_size : 2 * group_size], q_tile1) + if num_main_tiles == 4: + T.copy( + q_fp8[b_i, s_i, H0:H1, 2 * group_size : 3 * group_size], + q_tile2, + ) + T.copy( + q_fp8[b_i, s_i, H0:H1, 3 * group_size : 4 * group_size], + q_tile3, + ) for k_i in T.serial(inner_iter): topk_block_i = group_i * inner_iter + k_i @@ -1205,14 +1228,18 @@ def sparse_mla_fwd_decode_partial_fp8( for bi_i, j in T.Parallel(BI, group_size): page = page_idx_shared[bi_i] - kv_tile0[bi_i, j] = kv_fp8[b_i, page, g_i, 0 * group_size + j] - kv_tile1[bi_i, j] = kv_fp8[b_i, page, g_i, 1 * group_size + j] - kv_tile2[bi_i, j] = kv_fp8[b_i, page, g_i, 2 * group_size + j] - kv_tile3[bi_i, j] = kv_fp8[b_i, page, g_i, 3 * group_size + j] + kv_tile0[bi_i, j] = kv_fp8[b_i, page, g_i, j] + kv_tile1[bi_i, j] = kv_fp8[b_i, page, g_i, group_size + j] + if num_main_tiles == 4: + kv_tile2[bi_i, j] = kv_fp8[b_i, page, g_i, 2 * group_size + j] + kv_tile3[bi_i, j] = kv_fp8[b_i, page, g_i, 3 * group_size + j] - for bi_i, j in T.Parallel(BI, d_tail): - page = page_idx_shared[bi_i] - k_tail_shared[bi_i, j] = kv_fp8[b_i, page, g_i, rope_offset_fp8 + j] + if has_tail: + for bi_i, j in T.Parallel(BI, d_tail): + page = page_idx_shared[bi_i] + k_tail_shared[bi_i, j] = kv_fp8[ + b_i, page, g_i, rope_offset_fp8 + j + ] for h_i, bi_i in T.Parallel(h_per_block, BI): acc_s[h_i, bi_i] = T.if_then_else( @@ -1223,19 +1250,33 @@ def sparse_mla_fwd_decode_partial_fp8( T.gemm(q_tile1, kv_tile1, acc_tile, transpose_B=True, clear_accum=True) for h_i, bi_i in T.Parallel(h_per_block, BI): acc_s[h_i, bi_i] += acc_tile[h_i, bi_i] - T.gemm(q_tile2, kv_tile2, acc_tile, transpose_B=True, clear_accum=True) - for h_i, bi_i in T.Parallel(h_per_block, BI): - acc_s[h_i, bi_i] += acc_tile[h_i, bi_i] - T.gemm(q_tile3, kv_tile3, acc_tile, transpose_B=True, clear_accum=True) - for h_i, bi_i in T.Parallel(h_per_block, BI): - acc_s[h_i, bi_i] += acc_tile[h_i, bi_i] - T.gemm( - q_tail_buf, - k_tail_shared, - acc_s, - transpose_B=True, - policy=T.GemmWarpPolicy.FullCol, - ) + if num_main_tiles == 4: + T.gemm( + q_tile2, + kv_tile2, + acc_tile, + transpose_B=True, + clear_accum=True, + ) + for h_i, bi_i in T.Parallel(h_per_block, BI): + acc_s[h_i, bi_i] += acc_tile[h_i, bi_i] + T.gemm( + q_tile3, + kv_tile3, + acc_tile, + transpose_B=True, + clear_accum=True, + ) + for h_i, bi_i in T.Parallel(h_per_block, BI): + acc_s[h_i, bi_i] += acc_tile[h_i, bi_i] + if has_tail: + T.gemm( + q_tail_buf, + k_tail_shared, + acc_s, + transpose_B=True, + policy=T.GemmWarpPolicy.FullCol, + ) T.copy(m_i, m_i_prev) T.reduce_max(acc_s, m_i, dim=1, clear=False) @@ -1249,10 +1290,11 @@ def sparse_mla_fwd_decode_partial_fp8( for h_i in T.Parallel(h_per_block): sumexp[h_i] = sumexp[h_i] * alpha[h_i] + sumexp_i[h_i] for h_i, j in T.Parallel(h_per_block, group_size): - acc_o_tile0[h_i, j] = acc_o_tile0[h_i, j] * alpha[h_i] - acc_o_tile1[h_i, j] = acc_o_tile1[h_i, j] * alpha[h_i] - acc_o_tile2[h_i, j] = acc_o_tile2[h_i, j] * alpha[h_i] - acc_o_tile3[h_i, j] = acc_o_tile3[h_i, j] * alpha[h_i] + acc_o_tile0[h_i, j] *= alpha[h_i] + acc_o_tile1[h_i, j] *= alpha[h_i] + if num_main_tiles == 4: + acc_o_tile2[h_i, j] *= alpha[h_i] + acc_o_tile3[h_i, j] *= alpha[h_i] for h_i, bi_i in T.Parallel(h_per_block, BI): s_fp8_shared[h_i, bi_i] = T.clamp( @@ -1262,36 +1304,27 @@ def sparse_mla_fwd_decode_partial_fp8( ) T.gemm(s_fp8_shared, kv_tile0, sv_tile, clear_accum=True) for h_i, j in T.Parallel(h_per_block, group_size): - acc_o_tile0[h_i, j] = ( - acc_o_tile0[h_i, j] + sv_tile[h_i, j] * s_scale_const - ) - + acc_o_tile0[h_i, j] += sv_tile[h_i, j] * s_scale_const T.gemm(s_fp8_shared, kv_tile1, sv_tile, clear_accum=True) for h_i, j in T.Parallel(h_per_block, group_size): - acc_o_tile1[h_i, j] = ( - acc_o_tile1[h_i, j] + sv_tile[h_i, j] * s_scale_const - ) - - T.gemm(s_fp8_shared, kv_tile2, sv_tile, clear_accum=True) - for h_i, j in T.Parallel(h_per_block, group_size): - acc_o_tile2[h_i, j] = ( - acc_o_tile2[h_i, j] + sv_tile[h_i, j] * s_scale_const - ) - - T.gemm(s_fp8_shared, kv_tile3, sv_tile, clear_accum=True) - for h_i, j in T.Parallel(h_per_block, group_size): - acc_o_tile3[h_i, j] = ( - acc_o_tile3[h_i, j] + sv_tile[h_i, j] * s_scale_const - ) + acc_o_tile1[h_i, j] += sv_tile[h_i, j] * s_scale_const + if num_main_tiles == 4: + T.gemm(s_fp8_shared, kv_tile2, sv_tile, clear_accum=True) + for h_i, j in T.Parallel(h_per_block, group_size): + acc_o_tile2[h_i, j] += sv_tile[h_i, j] * s_scale_const + T.gemm(s_fp8_shared, kv_tile3, sv_tile, clear_accum=True) + for h_i, j in T.Parallel(h_per_block, group_size): + acc_o_tile3[h_i, j] += sv_tile[h_i, j] * s_scale_const for h_i in T.Parallel(h_per_block): denom = T.if_then_else(sumexp[h_i] == 0.0, 1.0, sumexp[h_i]) inv_denom[h_i] = 1.0 / denom for h_i, j in T.Parallel(h_per_block, group_size): - acc_o_tile0[h_i, j] = acc_o_tile0[h_i, j] * inv_denom[h_i] - acc_o_tile1[h_i, j] = acc_o_tile1[h_i, j] * inv_denom[h_i] - acc_o_tile2[h_i, j] = acc_o_tile2[h_i, j] * inv_denom[h_i] - acc_o_tile3[h_i, j] = acc_o_tile3[h_i, j] * inv_denom[h_i] + acc_o_tile0[h_i, j] *= inv_denom[h_i] + acc_o_tile1[h_i, j] *= inv_denom[h_i] + if num_main_tiles == 4: + acc_o_tile2[h_i, j] *= inv_denom[h_i] + acc_o_tile3[h_i, j] *= inv_denom[h_i] for h_i in T.Parallel(h_per_block): sumexp[h_i] = T.if_then_else( @@ -1300,22 +1333,24 @@ def sparse_mla_fwd_decode_partial_fp8( T.log2(sumexp[h_i]) + m_i[h_i] * sm_scale, ) - T.copy( - acc_o_tile0, - partial_o[b_i, s_i, group_i, H0:H1, 0 * group_size : 1 * group_size], - ) + T.copy(acc_o_tile0, partial_o[b_i, s_i, group_i, H0:H1, 0:group_size]) T.copy( acc_o_tile1, - partial_o[b_i, s_i, group_i, H0:H1, 1 * group_size : 2 * group_size], - ) - T.copy( - acc_o_tile2, - partial_o[b_i, s_i, group_i, H0:H1, 2 * group_size : 3 * group_size], - ) - T.copy( - acc_o_tile3, - partial_o[b_i, s_i, group_i, H0:H1, 3 * group_size : 4 * group_size], + partial_o[b_i, s_i, group_i, H0:H1, group_size : 2 * group_size], ) + if num_main_tiles == 4: + T.copy( + acc_o_tile2, + partial_o[ + b_i, s_i, group_i, H0:H1, 2 * group_size : 3 * group_size + ], + ) + T.copy( + acc_o_tile3, + partial_o[ + b_i, s_i, group_i, H0:H1, 3 * group_size : 4 * group_size + ], + ) T.copy(sumexp, partial_lse[b_i, s_i, group_i, H0:H1]) diff --git a/python/sglang/srt/mem_cache/memory_pool.py b/python/sglang/srt/mem_cache/memory_pool.py index c2f9c052c2..1f1c762bac 100644 --- a/python/sglang/srt/mem_cache/memory_pool.py +++ b/python/sglang/srt/mem_cache/memory_pool.py @@ -4601,7 +4601,8 @@ class MLATokenToKVPool(KVCache): and (self.use_dsa or self.dsa_kv_cache_store_fp8) ), "the DSA write paths have no resolved-loc variant" if _is_hip and self.use_dsa and self.dtype == fp8_dtype: - # HIP FP8 path uses raw MLA KV layout (nope + rope) without per-block scales. + # HIP TileLang FP8 consumes the model's raw MLA layout + # (NoPE + optional RoPE) without per-block scales. # Fuse BF16/FP16 -> FP8 cast with paged KV write. set_mla_kv_buffer_triton_fp8_quant( dst_buffer, @@ -4619,8 +4620,8 @@ class MLATokenToKVPool(KVCache): ) # Reuse existing two-tensor write kernel (works with FP8 byte layout) - # cache_k_nope_fp8: (num_tokens, 1, 528) uint8 [nope_fp8(512) | scales(16)] - # cache_k_rope_fp8: (num_tokens, 1, 128) uint8 [rope_bf16_bytes(128)] + # NoPE bytes are [FP8 values | one FP32 scale per 128 values]. + # RoPE bytes contain the raw BF16 tail and may have zero width. self._scatter_mla_rows(dst_buffer, loc, cache_k_nope_fp8, cache_k_rope_fp8) else: if cache_k_nope.dtype != self.dtype: diff --git a/test/registered/kernels/ops/attention/test_tilelang_dsa_zero_rope.py b/test/registered/kernels/ops/attention/test_tilelang_dsa_zero_rope.py new file mode 100644 index 0000000000..1458bd73bd --- /dev/null +++ b/test/registered/kernels/ops/attention/test_tilelang_dsa_zero_rope.py @@ -0,0 +1,120 @@ +"""DSA packed-cache and HIP TileLang kernels must accept GLM's 256+0 geometry without changing 512+64.""" + +import math +import unittest + +import torch + +from sglang.kernels.ops.attention.dsa.dequant_k_cache import ( + _infer_dsa_dims, + dequantize_k_cache, + dequantize_k_cache_paged, +) +from sglang.kernels.ops.attention.dsa.quant_k_cache import quantize_k_cache +from sglang.srt.utils import is_gfx95_supported, is_hip +from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci +from sglang.test.test_utils import CustomTestCase + +# backend-specific: the zero-tail specialization only exists in the HIP TileLang kernels +register_cuda_ci(est_time=30, stage="base-b-kernel-unit", runner_config="1-gpu-large") +register_amd_ci(est_time=180, suite="stage-b-test-1-gpu-small-amd-mi35x") + + +class TestPackedRowInference(CustomTestCase): + def test_packed_width_maps_to_one_layout(self): + """A 64-wide RoPE tail adds 128 bytes and a NoPE tile adds 132, so no width is ambiguous.""" + for packed_width, layout in { + 264: (256, 0), + 392: (256, 64), + 528: (512, 0), + 656: (512, 64), + }.items(): + self.assertEqual(_infer_dsa_dims(packed_width), layout) + with self.assertRaises(ValueError): + _infer_dsa_dims(265) + + +@unittest.skipUnless(torch.cuda.is_available(), "GPU required") +class TestScaledCacheLayouts(CustomTestCase): + def test_quant_dequant_round_trip_and_paged_gather(self): + """A 264-byte row must dequantize to 256 columns and gather by page like the 656-byte row.""" + torch.manual_seed(7) + for dim_nope, dim_rope in ((256, 0), (512, 64)): + with self.subTest(dim_nope=dim_nope, dim_rope=dim_rope): + source = torch.randn( + 8, 1, 1, dim_nope + dim_rope, device="cuda", dtype=torch.bfloat16 + ) + packed = quantize_k_cache(source, dv=dim_nope) + restored = dequantize_k_cache(packed, dv=dim_nope) + torch.testing.assert_close(restored, source, atol=0.08, rtol=0.08) + + pages = torch.tensor([7, 1, 1, 4], device="cuda", dtype=torch.int32) + gathered = dequantize_k_cache_paged(packed, pages) + expected = restored.view(8, 1, -1)[pages] + torch.testing.assert_close(gathered, expected, atol=0, rtol=0) + + +def _torch_sparse_attention(q, kv, indices, scale, d_v): + rows = indices[:, 0] + valid = rows >= 0 + selected = kv[rows.clamp_min(0), 0] + scores = torch.einsum("thd,tkd->thk", q.float(), selected.float()) * scale + scores.masked_fill_(~valid[:, None, :], float("-inf")) + probs = torch.softmax(scores, dim=-1) + probs = torch.where(valid[:, None, :], probs, 0) + return torch.einsum("thk,tkd->thd", probs, selected[..., :d_v].float()).to( + torch.bfloat16 + ) + + +@unittest.skipUnless( + torch.cuda.is_available() and is_hip() and is_gfx95_supported(), + "the zero-tail TileLang specialization is compiled for gfx950", +) +class TestTileLangDSAZeroRope(CustomTestCase): + def _assert_matches_torch(self, use_fp8, d_v, d_tail): + from sglang.kernels.ops.attention.dsa.tilelang_kernel import ( + FP8_DTYPE, + tilelang_sparse_fwd, + ) + + torch.manual_seed(7) + tokens, heads, topk = 17, 64, 2112 + dim = d_v + d_tail + q = torch.randn(tokens, heads, dim, device="cuda", dtype=torch.bfloat16) + kv = torch.randn(topk, 1, dim, device="cuda", dtype=torch.bfloat16) + indices = torch.arange(topk, device="cuda", dtype=torch.int32) + indices = indices.view(1, 1, topk).expand(tokens, -1, -1).clone() + indices[..., 2051:] = -1 # padded rows must be masked, not gathered from slot 0 + if use_fp8: + q = q.to(FP8_DTYPE) + kv = kv.to(FP8_DTYPE) + + scale = 1.0 / math.sqrt(dim) + expected = _torch_sparse_attention(q, kv, indices, scale, d_v) + # the HIP combine kernel returns [batch=1, tokens, heads, d_v] + actual = tilelang_sparse_fwd(q, kv, indices, scale, d_v=d_v).squeeze(0) + torch.testing.assert_close( + actual, + expected, + atol=0.20 if use_fp8 else 0.04, + rtol=0.12 if use_fp8 else 0.04, + ) + + def test_bf16_zero_rope_matches_torch(self): + """Before the fix the BF16 partial kernel emitted zero-extent tail copies and failed to compile.""" + self._assert_matches_torch(use_fp8=False, d_v=256, d_tail=0) + + def test_fp8_zero_rope_matches_torch(self): + """Before the fix the FP8 partial kernel asserted d_v == 512 and read four NoPE tiles.""" + self._assert_matches_torch(use_fp8=True, d_v=256, d_tail=0) + + def test_tail64_layout_unchanged_by_zero_tail_specialization(self): + """The 512+64 path was rewritten into has_tail/num_main_tiles branches and must still match.""" + for use_fp8 in (False, True): + with self.subTest(use_fp8=use_fp8): + self._assert_matches_torch(use_fp8=use_fp8, d_v=512, d_tail=64) + + +if __name__ == "__main__": + unittest.main()