[AMD] [GLM-5.3-Flash Day 0] Enable zero-RoPE TileLang DSA on gfx950 (#38547)

Co-authored-by: Raiden-Makoto <Raiden-Makoto@users.noreply.github.com>
Co-authored-by: Thomas Wang <thomawan@amd.com>
Co-authored-by: Kevin Mi <mikevin920@yahoo.com>
Co-authored-by: Kevin Mi <kevin.mi@radixark.ai>
Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Raiden Makoto
2026-09-22 13:08:25 -07:00
committed by GitHub
co-authored by Raiden-Makoto Thomas Wang Kevin Mi Kevin Mi Claude Fable 5.1
parent 4cbf290fb9
commit d6cc283da7
5 changed files with 338 additions and 159 deletions
@@ -5,13 +5,25 @@ import triton
import triton.language as tl
def dequantize_k_cache(quant_k_cache):
return _dequantize_k_cache_fast_wrapped(quant_k_cache)
def _infer_dsa_dims(dim_quant: int, tile_size: int = 128) -> tuple[int, int]:
"""Recover the NoPE and RoPE widths from a packed DSA cache row."""
for dim_rope in (0, 64):
scaled_width = (dim_quant - 2 * dim_rope) * tile_size
if scaled_width % (tile_size + 4):
continue
dim_nope = scaled_width // (tile_size + 4)
if dim_nope > 0 and dim_nope % tile_size == 0:
return dim_nope, dim_rope
raise ValueError(f"Unsupported packed DSA KV row width: {dim_quant}")
def dequantize_k_cache(quant_k_cache, dv: int | None = None):
return _dequantize_k_cache_fast_wrapped(quant_k_cache, dv=dv)
def _dequantize_k_cache_fast_wrapped(
quant_k_cache: torch.Tensor,
dv: int = 512,
dv: int | None = None,
tile_size: int = 128,
) -> torch.Tensor:
original_ndim = quant_k_cache.ndim
@@ -19,12 +31,15 @@ def _dequantize_k_cache_fast_wrapped(
# set block_size = 1
quant_k_cache = quant_k_cache.unsqueeze(1)
num_blocks, block_size, _, dim_quant = quant_k_cache.shape
assert dv == 512
assert dim_quant == 656
assert tile_size == 128
inferred_dv, dim_rope = _infer_dsa_dims(dim_quant)
if dv is None:
dv = inferred_dv
if dv != inferred_dv:
raise ValueError(f"dv={dv} does not match packed row width {dim_quant}")
quant_k_cache = quant_k_cache.view((-1, dim_quant))
output = _dequantize_k_cache_fast(quant_k_cache)
output = _dequantize_k_cache_fast(quant_k_cache, dim_nope=dv, dim_rope=dim_rope)
if original_ndim == 3:
return output.view(num_blocks, 1, -1)
@@ -32,14 +47,17 @@ def _dequantize_k_cache_fast_wrapped(
return output.view(num_blocks, block_size, 1, -1)
def _dequantize_k_cache_fast(quant_k_cache, group_size: int = 128):
def _dequantize_k_cache_fast(
quant_k_cache,
dim_nope: int,
dim_rope: int,
group_size: int = 128,
):
num_tokens, dim_quant = quant_k_cache.shape
assert quant_k_cache.dtype == torch.float8_e4m3fn
dim_nope = 512
dim_rope = 64
num_tiles = dim_nope // group_size
assert dim_quant == 656
assert dim_quant == dim_nope + num_tiles * 4 + dim_rope * 2
output = torch.empty(
(num_tokens, dim_nope + dim_rope),
@@ -47,8 +65,7 @@ def _dequantize_k_cache_fast(quant_k_cache, group_size: int = 128):
device=quant_k_cache.device,
)
num_blocks_per_token = triton.cdiv(dim_nope + dim_rope, group_size)
assert num_blocks_per_token == 5
num_blocks_per_token = num_tiles + triton.cdiv(dim_rope, group_size)
assert dim_nope % group_size == 0
@@ -138,18 +155,14 @@ def dequantize_k_cache_paged(
output: [num_tokens, 1, dim_nope + dim_rope], the de-quantized k-cache
"""
dim_quant = quant_k_cache.shape[-1]
assert dim_quant == 656, (
f"dim_quant: {dim_quant} != 656 detected in dequantize_k_cache_paged"
)
dim_nope, dim_rope = _infer_dsa_dims(dim_quant)
quant_k_cache = quant_k_cache.view((-1, dim_quant))
# num_tokens can exceed kv_cache_size due to prefix sharing (multiple seqs share same KV slots)
# Index bounds validated in dsa_backend.init_forward_metadata
num_tokens = page_table_1_flattened.shape[0]
assert quant_k_cache.dtype == torch.float8_e4m3fn
dim_nope = 512
dim_rope = 64
num_tiles = dim_nope // group_size # 512 // 128 = 4
num_tiles = dim_nope // group_size
output = torch.empty(
(num_tokens, 1, dim_nope + dim_rope),
@@ -157,9 +170,7 @@ def dequantize_k_cache_paged(
device=quant_k_cache.device,
)
# cdiv(512 + 64, 128) = 5
num_blocks_per_token = triton.cdiv(dim_nope + dim_rope, group_size)
assert num_blocks_per_token == 5
num_blocks_per_token = num_tiles + triton.cdiv(dim_rope, group_size)
assert dim_nope % group_size == 0
@@ -258,10 +269,11 @@ def gather_dequant_requant_fp8_paged(
extra_rows: int = 0,
out: Optional[torch.Tensor] = None,
) -> torch.Tensor:
"""Gather paged fp8 KV tokens and re-pack into flat [576] fp8 layout.
"""Gather paged FP8 KV tokens and re-pack into a flat raw FP8 layout.
The paged KV cache stores 656 bytes per token:
[512 nope_fp8 | 16 scales_f32 (4 groups) | 128 rope_bf16_bytes]
Each packed cache row stores:
[dim_nope FP8 bytes | 4 scale bytes per group | 2 bytes per RoPE element]
For example, 256+0 uses 264 bytes and 512+64 uses 656 bytes.
This kernel gathers the requested tokens, de-quantises nope with the
per-group scales, and re-quantises to per-tensor fp8 (scale=1.0).
Rope is cast bf16->fp8. The whole operation is fused into a single
@@ -275,7 +287,7 @@ def gather_dequant_requant_fp8_paged(
persistent (dirty) buffer via ``out``.
Args:
quant_k_cache: [total_num_tokens, 1, 656] fp8_e4m3fn
quant_k_cache: [total_num_tokens, 1, dim_quant] fp8_e4m3fn
page_table_1_flattened: [num_tokens] int32
group_size: per-group dequant tile size (default 128)
extra_rows: number of zero-filled landing-pad rows to append at
@@ -283,21 +295,20 @@ def gather_dequant_requant_fp8_paged(
kernel which over-reads past end-of-buffer for masked
indices)
out: optional pre-allocated destination of shape
[num_tokens + extra_rows, 1, 576] (or [.., 576]) fp8_e4m3fn,
contiguous. Contents may be arbitrary (fully overwritten).
[num_tokens + extra_rows, 1, dim_nope + dim_rope] (or the
equivalent 2D shape) fp8_e4m3fn, contiguous. Contents may be
arbitrary (fully overwritten).
Returns:
output: [num_tokens + extra_rows, 1, 576] fp8_e4m3fn
output: [num_tokens + extra_rows, 1, dim_nope + dim_rope] fp8_e4m3fn
"""
dim_quant = quant_k_cache.shape[-1]
assert dim_quant == 656
dim_nope, dim_rope = _infer_dsa_dims(dim_quant)
quant_k_cache = quant_k_cache.view((-1, dim_quant))
num_tokens = page_table_1_flattened.shape[0]
assert quant_k_cache.dtype == torch.float8_e4m3fn
dim_nope = 512
dim_rope = 64
num_tiles = dim_nope // group_size # 4
out_dim = dim_nope + dim_rope # 576
num_tiles = dim_nope // group_size
out_dim = dim_nope + dim_rope
assert num_tiles * group_size == dim_nope
total_rows = num_tokens + extra_rows
@@ -417,13 +428,17 @@ def _gather_dequant_requant_fp8_paged_vec_kernel(
tl.store(dst_q, y, mask=row_in_range[:, None, None])
# b. rope: [T, R] bf16 -> fp8; pad rows: 0.0 -> byte 0x00
offs_r = tl.arange(0, DIM_ROPE)
src_r = input_rope_ptr + paged[:, None] * input_rope_stride_0 + offs_r[None, :]
data = tl.load(src_r, mask=is_real[:, None], other=0.0).to(tl.float8e4nv)
dst_r = (
output_ptr + offs_t64[:, None] * output_stride_0 + DIM_NOPE + offs_r[None, :]
)
tl.store(dst_r, data, mask=row_in_range[:, None])
if DIM_ROPE > 0:
offs_r = tl.arange(0, DIM_ROPE)
src_r = input_rope_ptr + paged[:, None] * input_rope_stride_0 + offs_r[None, :]
data = tl.load(src_r, mask=is_real[:, None], other=0.0).to(tl.float8e4nv)
dst_r = (
output_ptr
+ offs_t64[:, None] * output_stride_0
+ DIM_NOPE
+ offs_r[None, :]
)
tl.store(dst_r, data, mask=row_in_range[:, None])
def gather_dequant_requant_fp8_paged_legacy(
@@ -441,15 +456,13 @@ def gather_dequant_requant_fp8_paged_legacy(
(token, 128-elem slice).
"""
dim_quant = quant_k_cache.shape[-1]
assert dim_quant == 656
dim_nope, dim_rope = _infer_dsa_dims(dim_quant)
quant_k_cache = quant_k_cache.view((-1, dim_quant))
num_tokens = page_table_1_flattened.shape[0]
assert quant_k_cache.dtype == torch.float8_e4m3fn
dim_nope = 512
dim_rope = 64
num_tiles = dim_nope // group_size # 4
out_dim = dim_nope + dim_rope # 576
num_tiles = dim_nope // group_size
out_dim = dim_nope + dim_rope
assert num_tiles * group_size == dim_nope
total_rows = num_tokens + extra_rows
@@ -462,8 +475,7 @@ def gather_dequant_requant_fp8_paged_legacy(
device=quant_k_cache.device,
)
num_blocks_per_token = triton.cdiv(dim_nope + dim_rope, group_size) # 5
assert num_blocks_per_token == 5
num_blocks_per_token = num_tiles + triton.cdiv(dim_rope, group_size)
input_nope_q = quant_k_cache[:, :dim_nope]
input_nope_s = quant_k_cache[:, dim_nope : dim_nope + num_tiles * 4].view(
@@ -54,8 +54,8 @@ def _gather_dsa_kv_scales(
block_start += num_programs * BLOCK
def quantize_k_cache(cache_k):
return _quantize_k_cache_fast_wrapped(cache_k)
def quantize_k_cache(cache_k, dv: int | None = None):
return _quantize_k_cache_fast_wrapped(cache_k, dv=dv)
def quantize_k_cache_separate(
@@ -71,7 +71,7 @@ def quantize_k_cache_separate(
Args:
k_nope: (num_tokens, dim_nope) or (num_tokens, 1, dim_nope)
Must have dim_nope=512 for FP8 MLA quantization
dim_nope must be divisible by 128
k_rope: (num_tokens, dim_rope) or (num_tokens, 1, dim_rope)
Must have dim_rope=64 for FP8 MLA quantization, or dim_rope=0
for no-PE MLA (empty rope); None is treated
@@ -80,8 +80,8 @@ def quantize_k_cache_separate(
Returns:
Tuple of (nope_part, rope_part) where:
- nope_part: (num_tokens, 1, 528) as uint8 view, contains [nope_fp8(512) | scales(16)]
- rope_part: (num_tokens, 1, 128) as uint8 view, contains [rope_bf16_bytes(128)]
- nope_part: uint8 view containing [nope_fp8 | one FP32 scale per tile]
- rope_part: uint8 view containing the raw BF16 RoPE bytes
(empty, (num_tokens, 1, 0), when dim_rope=0)
These two tensors can be directly passed to set_mla_kv_buffer_triton(kv_buffer, loc, nope_part, rope_part)
@@ -100,8 +100,10 @@ def quantize_k_cache_separate(
dim_rope = k_rope_2d.shape[1]
# Validate dimensions for FP8 MLA
if dim_nope != 512:
raise ValueError(f"Expected dim_nope=512 for FP8 MLA, got {dim_nope}")
if dim_nope % tile_size != 0:
raise ValueError(
f"Expected dim_nope divisible by {tile_size} for FP8 MLA, got {dim_nope}"
)
if dim_rope not in (0, 64):
raise ValueError(f"Expected dim_rope=64 (or 0 for no-PE MLA), got {dim_rope}")
if k_rope_2d.shape[0] != num_tokens:
@@ -170,14 +172,24 @@ def _quantize_k_cache_ref(
def _quantize_k_cache_fast_wrapped(
input_k_cache: torch.Tensor,
dv: int = 512,
dv: int | None = None,
tile_size: int = 128,
) -> torch.Tensor:
# TODO the final API may be 2D instead of 4D, thus we convert them here
num_blocks, block_size, _, dim_nope_and_rope = input_k_cache.shape
assert dv == 512
assert dim_nope_and_rope == 512 + 64
assert tile_size == 128
if dv is None:
if dim_nope_and_rope == 256:
dv = 256
elif dim_nope_and_rope == 512 + 64:
dv = 512
else:
raise ValueError(
"Cannot infer FP8 MLA NoPE width from total width "
f"{dim_nope_and_rope}; pass dv explicitly"
)
if dv % tile_size != 0 or dv > dim_nope_and_rope:
raise ValueError(f"Invalid dv={dv} for input width {dim_nope_and_rope}")
input_k_cache = input_k_cache.view((-1, dim_nope_and_rope))
# TODO deliberately split into two tensors, then upstream can provide the two tensors instead of concat into one
@@ -201,8 +213,8 @@ def _quantize_k_cache_fast(k_nope, k_rope, group_size: int = 128):
num_tokens, dim_nope = k_nope.shape
num_tokens_, dim_rope = k_rope.shape
assert num_tokens == num_tokens_
assert dim_nope == 512
assert dim_rope == 64
assert dim_nope % group_size == 0
assert dim_rope in (0, 64)
assert k_nope.dtype == k_rope.dtype
num_tiles = dim_nope // group_size
@@ -218,8 +230,7 @@ def _quantize_k_cache_fast(k_nope, k_rope, group_size: int = 128):
output_nope_s = output[..., dim_nope : dim_nope + num_tiles * 4].view(torch.float32)
output_rope = output[..., dim_nope + num_tiles * 4 :].view(torch.bfloat16)
num_blocks_per_token = triton.cdiv(dim_nope + dim_rope, group_size)
assert num_blocks_per_token == 5
num_blocks_per_token = num_tiles + triton.cdiv(dim_rope, group_size)
assert dim_nope % group_size == 0
NUM_NOPE_BLOCKS = dim_nope // group_size
@@ -252,8 +263,8 @@ def _quantize_k_cache_fast_separate(k_nope, k_rope, group_size: int = 128):
This avoids packing/unpacking and enables direct use with set_mla_kv_buffer_triton.
:param k_nope: (num_tokens, dim_nope 512) bfloat16
:param k_rope: (num_tokens, dim_rope 64) bfloat16
:param k_nope: (num_tokens, dim_nope) bfloat16; width divisible by 128
:param k_rope: (num_tokens, dim_rope) bfloat16; width 0 or 64
:param group_size: quantization tile size (default 128, kernel is tuned for this value)
:return: Tuple of (nope_part_u8, rope_part_u8)
- nope_part_u8: (num_tokens, 1, nope_part_bytes) uint8, layout [nope_fp8(dim_nope) | scales(num_tiles*4)]
@@ -861,6 +861,7 @@ def sparse_mla_fwd_decode_partial(
BI = block_I
D = dim
D_tail = tail_dim
has_tail = D_tail > 0
q_shape = [batch, seq_len, heads, dim + tail_dim]
kv_shape = [batch, seq_len_kv, kv_group, dim + tail_dim]
@@ -884,13 +885,16 @@ def sparse_mla_fwd_decode_partial(
with T.Kernel(seq_len * REPLICATE_H, N_GROUPS, threads=threads) as (bx, by):
if _q_in_shared:
Q_buf = T.alloc_shared([H_per_block, D], dtype)
Q_tail_buf = T.alloc_shared([H_per_block, D_tail], dtype)
if has_tail:
Q_tail_buf = T.alloc_shared([H_per_block, D_tail], dtype)
else:
Q_buf = T.alloc_fragment([H_per_block, D], dtype)
Q_tail_buf = T.alloc_fragment([H_per_block, D_tail], dtype)
if has_tail:
Q_tail_buf = T.alloc_fragment([H_per_block, D_tail], dtype)
KV_shared = T.alloc_shared([BI, D], dtype)
K_tail_shared = T.alloc_shared([BI, D_tail], dtype)
if has_tail:
K_tail_shared = T.alloc_shared([BI, D_tail], dtype)
S_shared = T.alloc_shared([H_per_block, BI], dtype)
mask = T.alloc_fragment([BI], T.bool)
@@ -913,7 +917,8 @@ def sparse_mla_fwd_decode_partial(
H1 = H0 + H_per_block
T.copy(Q[b_i, s_i, H0:H1, :D], Q_buf)
T.copy(Q[b_i, s_i, H0:H1, D:], Q_tail_buf)
if has_tail:
T.copy(Q[b_i, s_i, H0:H1, D:], Q_tail_buf)
for k_i in T.Pipelined(inner_iter, num_stages=num_stages):
topk_block_i = group_i * inner_iter + k_i
@@ -925,11 +930,12 @@ def sparse_mla_fwd_decode_partial(
KV_shared[bi_i, d_i] = KV[
b_i, T.if_then_else(idx >= 0, idx, 0), g_i, d_i
]
for bi_i, d_i in T.Parallel(BI, D_tail):
idx = Indices[b_i, s_i, g_i, topk_block_i * BI + bi_i]
K_tail_shared[bi_i, d_i] = KV[
b_i, T.if_then_else(idx >= 0, idx, 0), g_i, D + d_i
]
if has_tail:
for bi_i, d_i in T.Parallel(BI, D_tail):
idx = Indices[b_i, s_i, g_i, topk_block_i * BI + bi_i]
K_tail_shared[bi_i, d_i] = KV[
b_i, T.if_then_else(idx >= 0, idx, 0), g_i, D + d_i
]
for h_i, bi_i in T.Parallel(H_per_block, BI):
acc_s[h_i, bi_i] = T.if_then_else(
@@ -943,13 +949,14 @@ def sparse_mla_fwd_decode_partial(
transpose_B=True,
policy=T.GemmWarpPolicy.FullCol,
)
T.gemm(
Q_tail_buf,
K_tail_shared,
acc_s,
transpose_B=True,
policy=T.GemmWarpPolicy.FullCol,
)
if has_tail:
T.gemm(
Q_tail_buf,
K_tail_shared,
acc_s,
transpose_B=True,
policy=T.GemmWarpPolicy.FullCol,
)
T.copy(m_i, m_i_prev)
T.reduce_max(acc_s, m_i, dim=1, clear=False)
@@ -1085,7 +1092,7 @@ def sparse_mla_fwd_decode_partial_fp8(
inner_iter=1,
threads=256,
):
assert d_v == 512, f"only support d_v=512"
assert d_v in (256, 512), f"only support d_v=256 or d_v=512, got {d_v}"
assert topk % block_I == 0, (
"otherwise will load some index=0 thus causing wrong kv to be loaded"
)
@@ -1100,6 +1107,10 @@ def sparse_mla_fwd_decode_partial_fp8(
BI = block_I
group_size = 128
assert d_v % group_size == 0
assert d_tail in (0, 64), f"only support d_tail=0 or d_tail=64, got {d_tail}"
num_main_tiles = d_v // group_size
has_tail = d_tail > 0
dim_quant_fp8 = d_v + d_tail
rope_offset_fp8 = d_v
n_groups = topk // (BI * inner_iter)
@@ -1149,19 +1160,21 @@ def sparse_mla_fwd_decode_partial_fp8(
H0 = (bx % head_blocks_per_seq) * h_per_block
H1 = H0 + h_per_block
# We intentionally split the K=512 GEMM into 4x128 tiles.
# Split the main-dimension GEMM into 128-wide tiles.
# Although this adds extra intermediate memory traffic,
# it shortens the MFMA accumulation dependency chain and improves performance.
q_tile0 = T.alloc_shared([h_per_block, group_size], fp8_dtype)
q_tile1 = T.alloc_shared([h_per_block, group_size], fp8_dtype)
q_tile2 = T.alloc_shared([h_per_block, group_size], fp8_dtype)
q_tile3 = T.alloc_shared([h_per_block, group_size], fp8_dtype)
kv_tile0 = T.alloc_shared([BI, group_size], fp8_dtype)
kv_tile1 = T.alloc_shared([BI, group_size], fp8_dtype)
kv_tile2 = T.alloc_shared([BI, group_size], fp8_dtype)
kv_tile3 = T.alloc_shared([BI, group_size], fp8_dtype)
q_tail_buf = T.alloc_shared([h_per_block, d_tail], fp8_dtype)
k_tail_shared = T.alloc_shared([BI, d_tail], fp8_dtype)
if num_main_tiles == 4:
q_tile2 = T.alloc_shared([h_per_block, group_size], fp8_dtype)
q_tile3 = T.alloc_shared([h_per_block, group_size], fp8_dtype)
kv_tile2 = T.alloc_shared([BI, group_size], fp8_dtype)
kv_tile3 = T.alloc_shared([BI, group_size], fp8_dtype)
if has_tail:
q_tail_buf = T.alloc_shared([h_per_block, d_tail], fp8_dtype)
k_tail_shared = T.alloc_shared([BI, d_tail], fp8_dtype)
s_fp8_shared = T.alloc_shared([h_per_block, BI], fp8_dtype)
page_idx_shared = T.alloc_shared([BI], T.int32)
@@ -1178,21 +1191,31 @@ def sparse_mla_fwd_decode_partial_fp8(
acc_o_tile0 = T.alloc_fragment([h_per_block, group_size], accum_dtype)
acc_o_tile1 = T.alloc_fragment([h_per_block, group_size], accum_dtype)
acc_o_tile2 = T.alloc_fragment([h_per_block, group_size], accum_dtype)
acc_o_tile3 = T.alloc_fragment([h_per_block, group_size], accum_dtype)
if num_main_tiles == 4:
acc_o_tile2 = T.alloc_fragment([h_per_block, group_size], accum_dtype)
acc_o_tile3 = T.alloc_fragment([h_per_block, group_size], accum_dtype)
T.fill(acc_o_tile0, 0)
T.fill(acc_o_tile1, 0)
T.fill(acc_o_tile2, 0)
T.fill(acc_o_tile3, 0)
if num_main_tiles == 4:
T.fill(acc_o_tile2, 0)
T.fill(acc_o_tile3, 0)
T.fill(sumexp, 0)
T.fill(m_i, -(2**30))
T.copy(q_fp8[b_i, s_i, H0:H1, d_v:], q_tail_buf)
T.copy(q_fp8[b_i, s_i, H0:H1, 0 * group_size : 1 * group_size], q_tile0)
T.copy(q_fp8[b_i, s_i, H0:H1, 1 * group_size : 2 * group_size], q_tile1)
T.copy(q_fp8[b_i, s_i, H0:H1, 2 * group_size : 3 * group_size], q_tile2)
T.copy(q_fp8[b_i, s_i, H0:H1, 3 * group_size : 4 * group_size], q_tile3)
if has_tail:
T.copy(q_fp8[b_i, s_i, H0:H1, d_v:], q_tail_buf)
T.copy(q_fp8[b_i, s_i, H0:H1, 0:group_size], q_tile0)
T.copy(q_fp8[b_i, s_i, H0:H1, group_size : 2 * group_size], q_tile1)
if num_main_tiles == 4:
T.copy(
q_fp8[b_i, s_i, H0:H1, 2 * group_size : 3 * group_size],
q_tile2,
)
T.copy(
q_fp8[b_i, s_i, H0:H1, 3 * group_size : 4 * group_size],
q_tile3,
)
for k_i in T.serial(inner_iter):
topk_block_i = group_i * inner_iter + k_i
@@ -1205,14 +1228,18 @@ def sparse_mla_fwd_decode_partial_fp8(
for bi_i, j in T.Parallel(BI, group_size):
page = page_idx_shared[bi_i]
kv_tile0[bi_i, j] = kv_fp8[b_i, page, g_i, 0 * group_size + j]
kv_tile1[bi_i, j] = kv_fp8[b_i, page, g_i, 1 * group_size + j]
kv_tile2[bi_i, j] = kv_fp8[b_i, page, g_i, 2 * group_size + j]
kv_tile3[bi_i, j] = kv_fp8[b_i, page, g_i, 3 * group_size + j]
kv_tile0[bi_i, j] = kv_fp8[b_i, page, g_i, j]
kv_tile1[bi_i, j] = kv_fp8[b_i, page, g_i, group_size + j]
if num_main_tiles == 4:
kv_tile2[bi_i, j] = kv_fp8[b_i, page, g_i, 2 * group_size + j]
kv_tile3[bi_i, j] = kv_fp8[b_i, page, g_i, 3 * group_size + j]
for bi_i, j in T.Parallel(BI, d_tail):
page = page_idx_shared[bi_i]
k_tail_shared[bi_i, j] = kv_fp8[b_i, page, g_i, rope_offset_fp8 + j]
if has_tail:
for bi_i, j in T.Parallel(BI, d_tail):
page = page_idx_shared[bi_i]
k_tail_shared[bi_i, j] = kv_fp8[
b_i, page, g_i, rope_offset_fp8 + j
]
for h_i, bi_i in T.Parallel(h_per_block, BI):
acc_s[h_i, bi_i] = T.if_then_else(
@@ -1223,19 +1250,33 @@ def sparse_mla_fwd_decode_partial_fp8(
T.gemm(q_tile1, kv_tile1, acc_tile, transpose_B=True, clear_accum=True)
for h_i, bi_i in T.Parallel(h_per_block, BI):
acc_s[h_i, bi_i] += acc_tile[h_i, bi_i]
T.gemm(q_tile2, kv_tile2, acc_tile, transpose_B=True, clear_accum=True)
for h_i, bi_i in T.Parallel(h_per_block, BI):
acc_s[h_i, bi_i] += acc_tile[h_i, bi_i]
T.gemm(q_tile3, kv_tile3, acc_tile, transpose_B=True, clear_accum=True)
for h_i, bi_i in T.Parallel(h_per_block, BI):
acc_s[h_i, bi_i] += acc_tile[h_i, bi_i]
T.gemm(
q_tail_buf,
k_tail_shared,
acc_s,
transpose_B=True,
policy=T.GemmWarpPolicy.FullCol,
)
if num_main_tiles == 4:
T.gemm(
q_tile2,
kv_tile2,
acc_tile,
transpose_B=True,
clear_accum=True,
)
for h_i, bi_i in T.Parallel(h_per_block, BI):
acc_s[h_i, bi_i] += acc_tile[h_i, bi_i]
T.gemm(
q_tile3,
kv_tile3,
acc_tile,
transpose_B=True,
clear_accum=True,
)
for h_i, bi_i in T.Parallel(h_per_block, BI):
acc_s[h_i, bi_i] += acc_tile[h_i, bi_i]
if has_tail:
T.gemm(
q_tail_buf,
k_tail_shared,
acc_s,
transpose_B=True,
policy=T.GemmWarpPolicy.FullCol,
)
T.copy(m_i, m_i_prev)
T.reduce_max(acc_s, m_i, dim=1, clear=False)
@@ -1249,10 +1290,11 @@ def sparse_mla_fwd_decode_partial_fp8(
for h_i in T.Parallel(h_per_block):
sumexp[h_i] = sumexp[h_i] * alpha[h_i] + sumexp_i[h_i]
for h_i, j in T.Parallel(h_per_block, group_size):
acc_o_tile0[h_i, j] = acc_o_tile0[h_i, j] * alpha[h_i]
acc_o_tile1[h_i, j] = acc_o_tile1[h_i, j] * alpha[h_i]
acc_o_tile2[h_i, j] = acc_o_tile2[h_i, j] * alpha[h_i]
acc_o_tile3[h_i, j] = acc_o_tile3[h_i, j] * alpha[h_i]
acc_o_tile0[h_i, j] *= alpha[h_i]
acc_o_tile1[h_i, j] *= alpha[h_i]
if num_main_tiles == 4:
acc_o_tile2[h_i, j] *= alpha[h_i]
acc_o_tile3[h_i, j] *= alpha[h_i]
for h_i, bi_i in T.Parallel(h_per_block, BI):
s_fp8_shared[h_i, bi_i] = T.clamp(
@@ -1262,36 +1304,27 @@ def sparse_mla_fwd_decode_partial_fp8(
)
T.gemm(s_fp8_shared, kv_tile0, sv_tile, clear_accum=True)
for h_i, j in T.Parallel(h_per_block, group_size):
acc_o_tile0[h_i, j] = (
acc_o_tile0[h_i, j] + sv_tile[h_i, j] * s_scale_const
)
acc_o_tile0[h_i, j] += sv_tile[h_i, j] * s_scale_const
T.gemm(s_fp8_shared, kv_tile1, sv_tile, clear_accum=True)
for h_i, j in T.Parallel(h_per_block, group_size):
acc_o_tile1[h_i, j] = (
acc_o_tile1[h_i, j] + sv_tile[h_i, j] * s_scale_const
)
T.gemm(s_fp8_shared, kv_tile2, sv_tile, clear_accum=True)
for h_i, j in T.Parallel(h_per_block, group_size):
acc_o_tile2[h_i, j] = (
acc_o_tile2[h_i, j] + sv_tile[h_i, j] * s_scale_const
)
T.gemm(s_fp8_shared, kv_tile3, sv_tile, clear_accum=True)
for h_i, j in T.Parallel(h_per_block, group_size):
acc_o_tile3[h_i, j] = (
acc_o_tile3[h_i, j] + sv_tile[h_i, j] * s_scale_const
)
acc_o_tile1[h_i, j] += sv_tile[h_i, j] * s_scale_const
if num_main_tiles == 4:
T.gemm(s_fp8_shared, kv_tile2, sv_tile, clear_accum=True)
for h_i, j in T.Parallel(h_per_block, group_size):
acc_o_tile2[h_i, j] += sv_tile[h_i, j] * s_scale_const
T.gemm(s_fp8_shared, kv_tile3, sv_tile, clear_accum=True)
for h_i, j in T.Parallel(h_per_block, group_size):
acc_o_tile3[h_i, j] += sv_tile[h_i, j] * s_scale_const
for h_i in T.Parallel(h_per_block):
denom = T.if_then_else(sumexp[h_i] == 0.0, 1.0, sumexp[h_i])
inv_denom[h_i] = 1.0 / denom
for h_i, j in T.Parallel(h_per_block, group_size):
acc_o_tile0[h_i, j] = acc_o_tile0[h_i, j] * inv_denom[h_i]
acc_o_tile1[h_i, j] = acc_o_tile1[h_i, j] * inv_denom[h_i]
acc_o_tile2[h_i, j] = acc_o_tile2[h_i, j] * inv_denom[h_i]
acc_o_tile3[h_i, j] = acc_o_tile3[h_i, j] * inv_denom[h_i]
acc_o_tile0[h_i, j] *= inv_denom[h_i]
acc_o_tile1[h_i, j] *= inv_denom[h_i]
if num_main_tiles == 4:
acc_o_tile2[h_i, j] *= inv_denom[h_i]
acc_o_tile3[h_i, j] *= inv_denom[h_i]
for h_i in T.Parallel(h_per_block):
sumexp[h_i] = T.if_then_else(
@@ -1300,22 +1333,24 @@ def sparse_mla_fwd_decode_partial_fp8(
T.log2(sumexp[h_i]) + m_i[h_i] * sm_scale,
)
T.copy(
acc_o_tile0,
partial_o[b_i, s_i, group_i, H0:H1, 0 * group_size : 1 * group_size],
)
T.copy(acc_o_tile0, partial_o[b_i, s_i, group_i, H0:H1, 0:group_size])
T.copy(
acc_o_tile1,
partial_o[b_i, s_i, group_i, H0:H1, 1 * group_size : 2 * group_size],
)
T.copy(
acc_o_tile2,
partial_o[b_i, s_i, group_i, H0:H1, 2 * group_size : 3 * group_size],
)
T.copy(
acc_o_tile3,
partial_o[b_i, s_i, group_i, H0:H1, 3 * group_size : 4 * group_size],
partial_o[b_i, s_i, group_i, H0:H1, group_size : 2 * group_size],
)
if num_main_tiles == 4:
T.copy(
acc_o_tile2,
partial_o[
b_i, s_i, group_i, H0:H1, 2 * group_size : 3 * group_size
],
)
T.copy(
acc_o_tile3,
partial_o[
b_i, s_i, group_i, H0:H1, 3 * group_size : 4 * group_size
],
)
T.copy(sumexp, partial_lse[b_i, s_i, group_i, H0:H1])
+4 -3
View File
@@ -4601,7 +4601,8 @@ class MLATokenToKVPool(KVCache):
and (self.use_dsa or self.dsa_kv_cache_store_fp8)
), "the DSA write paths have no resolved-loc variant"
if _is_hip and self.use_dsa and self.dtype == fp8_dtype:
# HIP FP8 path uses raw MLA KV layout (nope + rope) without per-block scales.
# HIP TileLang FP8 consumes the model's raw MLA layout
# (NoPE + optional RoPE) without per-block scales.
# Fuse BF16/FP16 -> FP8 cast with paged KV write.
set_mla_kv_buffer_triton_fp8_quant(
dst_buffer,
@@ -4619,8 +4620,8 @@ class MLATokenToKVPool(KVCache):
)
# Reuse existing two-tensor write kernel (works with FP8 byte layout)
# cache_k_nope_fp8: (num_tokens, 1, 528) uint8 [nope_fp8(512) | scales(16)]
# cache_k_rope_fp8: (num_tokens, 1, 128) uint8 [rope_bf16_bytes(128)]
# NoPE bytes are [FP8 values | one FP32 scale per 128 values].
# RoPE bytes contain the raw BF16 tail and may have zero width.
self._scatter_mla_rows(dst_buffer, loc, cache_k_nope_fp8, cache_k_rope_fp8)
else:
if cache_k_nope.dtype != self.dtype: