[AMD] [GLM-5.3-Flash Day 0] Enable zero-RoPE TileLang DSA on gfx950 (#38547)
Co-authored-by: Raiden-Makoto <Raiden-Makoto@users.noreply.github.com> Co-authored-by: Thomas Wang <thomawan@amd.com> Co-authored-by: Kevin Mi <mikevin920@yahoo.com> Co-authored-by: Kevin Mi <kevin.mi@radixark.ai> Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Raiden-Makoto
Thomas Wang
Kevin Mi
Kevin Mi
Claude Fable 5.1
parent
4cbf290fb9
commit
d6cc283da7
@@ -5,13 +5,25 @@ import triton
|
||||
import triton.language as tl
|
||||
|
||||
|
||||
def dequantize_k_cache(quant_k_cache):
|
||||
return _dequantize_k_cache_fast_wrapped(quant_k_cache)
|
||||
def _infer_dsa_dims(dim_quant: int, tile_size: int = 128) -> tuple[int, int]:
|
||||
"""Recover the NoPE and RoPE widths from a packed DSA cache row."""
|
||||
for dim_rope in (0, 64):
|
||||
scaled_width = (dim_quant - 2 * dim_rope) * tile_size
|
||||
if scaled_width % (tile_size + 4):
|
||||
continue
|
||||
dim_nope = scaled_width // (tile_size + 4)
|
||||
if dim_nope > 0 and dim_nope % tile_size == 0:
|
||||
return dim_nope, dim_rope
|
||||
raise ValueError(f"Unsupported packed DSA KV row width: {dim_quant}")
|
||||
|
||||
|
||||
def dequantize_k_cache(quant_k_cache, dv: int | None = None):
|
||||
return _dequantize_k_cache_fast_wrapped(quant_k_cache, dv=dv)
|
||||
|
||||
|
||||
def _dequantize_k_cache_fast_wrapped(
|
||||
quant_k_cache: torch.Tensor,
|
||||
dv: int = 512,
|
||||
dv: int | None = None,
|
||||
tile_size: int = 128,
|
||||
) -> torch.Tensor:
|
||||
original_ndim = quant_k_cache.ndim
|
||||
@@ -19,12 +31,15 @@ def _dequantize_k_cache_fast_wrapped(
|
||||
# set block_size = 1
|
||||
quant_k_cache = quant_k_cache.unsqueeze(1)
|
||||
num_blocks, block_size, _, dim_quant = quant_k_cache.shape
|
||||
assert dv == 512
|
||||
assert dim_quant == 656
|
||||
assert tile_size == 128
|
||||
inferred_dv, dim_rope = _infer_dsa_dims(dim_quant)
|
||||
if dv is None:
|
||||
dv = inferred_dv
|
||||
if dv != inferred_dv:
|
||||
raise ValueError(f"dv={dv} does not match packed row width {dim_quant}")
|
||||
quant_k_cache = quant_k_cache.view((-1, dim_quant))
|
||||
|
||||
output = _dequantize_k_cache_fast(quant_k_cache)
|
||||
output = _dequantize_k_cache_fast(quant_k_cache, dim_nope=dv, dim_rope=dim_rope)
|
||||
|
||||
if original_ndim == 3:
|
||||
return output.view(num_blocks, 1, -1)
|
||||
@@ -32,14 +47,17 @@ def _dequantize_k_cache_fast_wrapped(
|
||||
return output.view(num_blocks, block_size, 1, -1)
|
||||
|
||||
|
||||
def _dequantize_k_cache_fast(quant_k_cache, group_size: int = 128):
|
||||
def _dequantize_k_cache_fast(
|
||||
quant_k_cache,
|
||||
dim_nope: int,
|
||||
dim_rope: int,
|
||||
group_size: int = 128,
|
||||
):
|
||||
num_tokens, dim_quant = quant_k_cache.shape
|
||||
|
||||
assert quant_k_cache.dtype == torch.float8_e4m3fn
|
||||
dim_nope = 512
|
||||
dim_rope = 64
|
||||
num_tiles = dim_nope // group_size
|
||||
assert dim_quant == 656
|
||||
assert dim_quant == dim_nope + num_tiles * 4 + dim_rope * 2
|
||||
|
||||
output = torch.empty(
|
||||
(num_tokens, dim_nope + dim_rope),
|
||||
@@ -47,8 +65,7 @@ def _dequantize_k_cache_fast(quant_k_cache, group_size: int = 128):
|
||||
device=quant_k_cache.device,
|
||||
)
|
||||
|
||||
num_blocks_per_token = triton.cdiv(dim_nope + dim_rope, group_size)
|
||||
assert num_blocks_per_token == 5
|
||||
num_blocks_per_token = num_tiles + triton.cdiv(dim_rope, group_size)
|
||||
|
||||
assert dim_nope % group_size == 0
|
||||
|
||||
@@ -138,18 +155,14 @@ def dequantize_k_cache_paged(
|
||||
output: [num_tokens, 1, dim_nope + dim_rope], the de-quantized k-cache
|
||||
"""
|
||||
dim_quant = quant_k_cache.shape[-1]
|
||||
assert dim_quant == 656, (
|
||||
f"dim_quant: {dim_quant} != 656 detected in dequantize_k_cache_paged"
|
||||
)
|
||||
dim_nope, dim_rope = _infer_dsa_dims(dim_quant)
|
||||
quant_k_cache = quant_k_cache.view((-1, dim_quant))
|
||||
|
||||
# num_tokens can exceed kv_cache_size due to prefix sharing (multiple seqs share same KV slots)
|
||||
# Index bounds validated in dsa_backend.init_forward_metadata
|
||||
num_tokens = page_table_1_flattened.shape[0]
|
||||
assert quant_k_cache.dtype == torch.float8_e4m3fn
|
||||
dim_nope = 512
|
||||
dim_rope = 64
|
||||
num_tiles = dim_nope // group_size # 512 // 128 = 4
|
||||
num_tiles = dim_nope // group_size
|
||||
|
||||
output = torch.empty(
|
||||
(num_tokens, 1, dim_nope + dim_rope),
|
||||
@@ -157,9 +170,7 @@ def dequantize_k_cache_paged(
|
||||
device=quant_k_cache.device,
|
||||
)
|
||||
|
||||
# cdiv(512 + 64, 128) = 5
|
||||
num_blocks_per_token = triton.cdiv(dim_nope + dim_rope, group_size)
|
||||
assert num_blocks_per_token == 5
|
||||
num_blocks_per_token = num_tiles + triton.cdiv(dim_rope, group_size)
|
||||
|
||||
assert dim_nope % group_size == 0
|
||||
|
||||
@@ -258,10 +269,11 @@ def gather_dequant_requant_fp8_paged(
|
||||
extra_rows: int = 0,
|
||||
out: Optional[torch.Tensor] = None,
|
||||
) -> torch.Tensor:
|
||||
"""Gather paged fp8 KV tokens and re-pack into flat [576] fp8 layout.
|
||||
"""Gather paged FP8 KV tokens and re-pack into a flat raw FP8 layout.
|
||||
|
||||
The paged KV cache stores 656 bytes per token:
|
||||
[512 nope_fp8 | 16 scales_f32 (4 groups) | 128 rope_bf16_bytes]
|
||||
Each packed cache row stores:
|
||||
[dim_nope FP8 bytes | 4 scale bytes per group | 2 bytes per RoPE element]
|
||||
For example, 256+0 uses 264 bytes and 512+64 uses 656 bytes.
|
||||
This kernel gathers the requested tokens, de-quantises nope with the
|
||||
per-group scales, and re-quantises to per-tensor fp8 (scale=1.0).
|
||||
Rope is cast bf16->fp8. The whole operation is fused into a single
|
||||
@@ -275,7 +287,7 @@ def gather_dequant_requant_fp8_paged(
|
||||
persistent (dirty) buffer via ``out``.
|
||||
|
||||
Args:
|
||||
quant_k_cache: [total_num_tokens, 1, 656] fp8_e4m3fn
|
||||
quant_k_cache: [total_num_tokens, 1, dim_quant] fp8_e4m3fn
|
||||
page_table_1_flattened: [num_tokens] int32
|
||||
group_size: per-group dequant tile size (default 128)
|
||||
extra_rows: number of zero-filled landing-pad rows to append at
|
||||
@@ -283,21 +295,20 @@ def gather_dequant_requant_fp8_paged(
|
||||
kernel which over-reads past end-of-buffer for masked
|
||||
indices)
|
||||
out: optional pre-allocated destination of shape
|
||||
[num_tokens + extra_rows, 1, 576] (or [.., 576]) fp8_e4m3fn,
|
||||
contiguous. Contents may be arbitrary (fully overwritten).
|
||||
[num_tokens + extra_rows, 1, dim_nope + dim_rope] (or the
|
||||
equivalent 2D shape) fp8_e4m3fn, contiguous. Contents may be
|
||||
arbitrary (fully overwritten).
|
||||
Returns:
|
||||
output: [num_tokens + extra_rows, 1, 576] fp8_e4m3fn
|
||||
output: [num_tokens + extra_rows, 1, dim_nope + dim_rope] fp8_e4m3fn
|
||||
"""
|
||||
dim_quant = quant_k_cache.shape[-1]
|
||||
assert dim_quant == 656
|
||||
dim_nope, dim_rope = _infer_dsa_dims(dim_quant)
|
||||
quant_k_cache = quant_k_cache.view((-1, dim_quant))
|
||||
|
||||
num_tokens = page_table_1_flattened.shape[0]
|
||||
assert quant_k_cache.dtype == torch.float8_e4m3fn
|
||||
dim_nope = 512
|
||||
dim_rope = 64
|
||||
num_tiles = dim_nope // group_size # 4
|
||||
out_dim = dim_nope + dim_rope # 576
|
||||
num_tiles = dim_nope // group_size
|
||||
out_dim = dim_nope + dim_rope
|
||||
assert num_tiles * group_size == dim_nope
|
||||
|
||||
total_rows = num_tokens + extra_rows
|
||||
@@ -417,13 +428,17 @@ def _gather_dequant_requant_fp8_paged_vec_kernel(
|
||||
tl.store(dst_q, y, mask=row_in_range[:, None, None])
|
||||
|
||||
# b. rope: [T, R] bf16 -> fp8; pad rows: 0.0 -> byte 0x00
|
||||
offs_r = tl.arange(0, DIM_ROPE)
|
||||
src_r = input_rope_ptr + paged[:, None] * input_rope_stride_0 + offs_r[None, :]
|
||||
data = tl.load(src_r, mask=is_real[:, None], other=0.0).to(tl.float8e4nv)
|
||||
dst_r = (
|
||||
output_ptr + offs_t64[:, None] * output_stride_0 + DIM_NOPE + offs_r[None, :]
|
||||
)
|
||||
tl.store(dst_r, data, mask=row_in_range[:, None])
|
||||
if DIM_ROPE > 0:
|
||||
offs_r = tl.arange(0, DIM_ROPE)
|
||||
src_r = input_rope_ptr + paged[:, None] * input_rope_stride_0 + offs_r[None, :]
|
||||
data = tl.load(src_r, mask=is_real[:, None], other=0.0).to(tl.float8e4nv)
|
||||
dst_r = (
|
||||
output_ptr
|
||||
+ offs_t64[:, None] * output_stride_0
|
||||
+ DIM_NOPE
|
||||
+ offs_r[None, :]
|
||||
)
|
||||
tl.store(dst_r, data, mask=row_in_range[:, None])
|
||||
|
||||
|
||||
def gather_dequant_requant_fp8_paged_legacy(
|
||||
@@ -441,15 +456,13 @@ def gather_dequant_requant_fp8_paged_legacy(
|
||||
(token, 128-elem slice).
|
||||
"""
|
||||
dim_quant = quant_k_cache.shape[-1]
|
||||
assert dim_quant == 656
|
||||
dim_nope, dim_rope = _infer_dsa_dims(dim_quant)
|
||||
quant_k_cache = quant_k_cache.view((-1, dim_quant))
|
||||
|
||||
num_tokens = page_table_1_flattened.shape[0]
|
||||
assert quant_k_cache.dtype == torch.float8_e4m3fn
|
||||
dim_nope = 512
|
||||
dim_rope = 64
|
||||
num_tiles = dim_nope // group_size # 4
|
||||
out_dim = dim_nope + dim_rope # 576
|
||||
num_tiles = dim_nope // group_size
|
||||
out_dim = dim_nope + dim_rope
|
||||
assert num_tiles * group_size == dim_nope
|
||||
|
||||
total_rows = num_tokens + extra_rows
|
||||
@@ -462,8 +475,7 @@ def gather_dequant_requant_fp8_paged_legacy(
|
||||
device=quant_k_cache.device,
|
||||
)
|
||||
|
||||
num_blocks_per_token = triton.cdiv(dim_nope + dim_rope, group_size) # 5
|
||||
assert num_blocks_per_token == 5
|
||||
num_blocks_per_token = num_tiles + triton.cdiv(dim_rope, group_size)
|
||||
|
||||
input_nope_q = quant_k_cache[:, :dim_nope]
|
||||
input_nope_s = quant_k_cache[:, dim_nope : dim_nope + num_tiles * 4].view(
|
||||
|
||||
@@ -54,8 +54,8 @@ def _gather_dsa_kv_scales(
|
||||
block_start += num_programs * BLOCK
|
||||
|
||||
|
||||
def quantize_k_cache(cache_k):
|
||||
return _quantize_k_cache_fast_wrapped(cache_k)
|
||||
def quantize_k_cache(cache_k, dv: int | None = None):
|
||||
return _quantize_k_cache_fast_wrapped(cache_k, dv=dv)
|
||||
|
||||
|
||||
def quantize_k_cache_separate(
|
||||
@@ -71,7 +71,7 @@ def quantize_k_cache_separate(
|
||||
|
||||
Args:
|
||||
k_nope: (num_tokens, dim_nope) or (num_tokens, 1, dim_nope)
|
||||
Must have dim_nope=512 for FP8 MLA quantization
|
||||
dim_nope must be divisible by 128
|
||||
k_rope: (num_tokens, dim_rope) or (num_tokens, 1, dim_rope)
|
||||
Must have dim_rope=64 for FP8 MLA quantization, or dim_rope=0
|
||||
for no-PE MLA (empty rope); None is treated
|
||||
@@ -80,8 +80,8 @@ def quantize_k_cache_separate(
|
||||
|
||||
Returns:
|
||||
Tuple of (nope_part, rope_part) where:
|
||||
- nope_part: (num_tokens, 1, 528) as uint8 view, contains [nope_fp8(512) | scales(16)]
|
||||
- rope_part: (num_tokens, 1, 128) as uint8 view, contains [rope_bf16_bytes(128)]
|
||||
- nope_part: uint8 view containing [nope_fp8 | one FP32 scale per tile]
|
||||
- rope_part: uint8 view containing the raw BF16 RoPE bytes
|
||||
(empty, (num_tokens, 1, 0), when dim_rope=0)
|
||||
|
||||
These two tensors can be directly passed to set_mla_kv_buffer_triton(kv_buffer, loc, nope_part, rope_part)
|
||||
@@ -100,8 +100,10 @@ def quantize_k_cache_separate(
|
||||
dim_rope = k_rope_2d.shape[1]
|
||||
|
||||
# Validate dimensions for FP8 MLA
|
||||
if dim_nope != 512:
|
||||
raise ValueError(f"Expected dim_nope=512 for FP8 MLA, got {dim_nope}")
|
||||
if dim_nope % tile_size != 0:
|
||||
raise ValueError(
|
||||
f"Expected dim_nope divisible by {tile_size} for FP8 MLA, got {dim_nope}"
|
||||
)
|
||||
if dim_rope not in (0, 64):
|
||||
raise ValueError(f"Expected dim_rope=64 (or 0 for no-PE MLA), got {dim_rope}")
|
||||
if k_rope_2d.shape[0] != num_tokens:
|
||||
@@ -170,14 +172,24 @@ def _quantize_k_cache_ref(
|
||||
|
||||
def _quantize_k_cache_fast_wrapped(
|
||||
input_k_cache: torch.Tensor,
|
||||
dv: int = 512,
|
||||
dv: int | None = None,
|
||||
tile_size: int = 128,
|
||||
) -> torch.Tensor:
|
||||
# TODO the final API may be 2D instead of 4D, thus we convert them here
|
||||
num_blocks, block_size, _, dim_nope_and_rope = input_k_cache.shape
|
||||
assert dv == 512
|
||||
assert dim_nope_and_rope == 512 + 64
|
||||
assert tile_size == 128
|
||||
if dv is None:
|
||||
if dim_nope_and_rope == 256:
|
||||
dv = 256
|
||||
elif dim_nope_and_rope == 512 + 64:
|
||||
dv = 512
|
||||
else:
|
||||
raise ValueError(
|
||||
"Cannot infer FP8 MLA NoPE width from total width "
|
||||
f"{dim_nope_and_rope}; pass dv explicitly"
|
||||
)
|
||||
if dv % tile_size != 0 or dv > dim_nope_and_rope:
|
||||
raise ValueError(f"Invalid dv={dv} for input width {dim_nope_and_rope}")
|
||||
input_k_cache = input_k_cache.view((-1, dim_nope_and_rope))
|
||||
|
||||
# TODO deliberately split into two tensors, then upstream can provide the two tensors instead of concat into one
|
||||
@@ -201,8 +213,8 @@ def _quantize_k_cache_fast(k_nope, k_rope, group_size: int = 128):
|
||||
num_tokens, dim_nope = k_nope.shape
|
||||
num_tokens_, dim_rope = k_rope.shape
|
||||
assert num_tokens == num_tokens_
|
||||
assert dim_nope == 512
|
||||
assert dim_rope == 64
|
||||
assert dim_nope % group_size == 0
|
||||
assert dim_rope in (0, 64)
|
||||
assert k_nope.dtype == k_rope.dtype
|
||||
num_tiles = dim_nope // group_size
|
||||
|
||||
@@ -218,8 +230,7 @@ def _quantize_k_cache_fast(k_nope, k_rope, group_size: int = 128):
|
||||
output_nope_s = output[..., dim_nope : dim_nope + num_tiles * 4].view(torch.float32)
|
||||
output_rope = output[..., dim_nope + num_tiles * 4 :].view(torch.bfloat16)
|
||||
|
||||
num_blocks_per_token = triton.cdiv(dim_nope + dim_rope, group_size)
|
||||
assert num_blocks_per_token == 5
|
||||
num_blocks_per_token = num_tiles + triton.cdiv(dim_rope, group_size)
|
||||
|
||||
assert dim_nope % group_size == 0
|
||||
NUM_NOPE_BLOCKS = dim_nope // group_size
|
||||
@@ -252,8 +263,8 @@ def _quantize_k_cache_fast_separate(k_nope, k_rope, group_size: int = 128):
|
||||
|
||||
This avoids packing/unpacking and enables direct use with set_mla_kv_buffer_triton.
|
||||
|
||||
:param k_nope: (num_tokens, dim_nope 512) bfloat16
|
||||
:param k_rope: (num_tokens, dim_rope 64) bfloat16
|
||||
:param k_nope: (num_tokens, dim_nope) bfloat16; width divisible by 128
|
||||
:param k_rope: (num_tokens, dim_rope) bfloat16; width 0 or 64
|
||||
:param group_size: quantization tile size (default 128, kernel is tuned for this value)
|
||||
:return: Tuple of (nope_part_u8, rope_part_u8)
|
||||
- nope_part_u8: (num_tokens, 1, nope_part_bytes) uint8, layout [nope_fp8(dim_nope) | scales(num_tiles*4)]
|
||||
|
||||
@@ -861,6 +861,7 @@ def sparse_mla_fwd_decode_partial(
|
||||
BI = block_I
|
||||
D = dim
|
||||
D_tail = tail_dim
|
||||
has_tail = D_tail > 0
|
||||
|
||||
q_shape = [batch, seq_len, heads, dim + tail_dim]
|
||||
kv_shape = [batch, seq_len_kv, kv_group, dim + tail_dim]
|
||||
@@ -884,13 +885,16 @@ def sparse_mla_fwd_decode_partial(
|
||||
with T.Kernel(seq_len * REPLICATE_H, N_GROUPS, threads=threads) as (bx, by):
|
||||
if _q_in_shared:
|
||||
Q_buf = T.alloc_shared([H_per_block, D], dtype)
|
||||
Q_tail_buf = T.alloc_shared([H_per_block, D_tail], dtype)
|
||||
if has_tail:
|
||||
Q_tail_buf = T.alloc_shared([H_per_block, D_tail], dtype)
|
||||
else:
|
||||
Q_buf = T.alloc_fragment([H_per_block, D], dtype)
|
||||
Q_tail_buf = T.alloc_fragment([H_per_block, D_tail], dtype)
|
||||
if has_tail:
|
||||
Q_tail_buf = T.alloc_fragment([H_per_block, D_tail], dtype)
|
||||
|
||||
KV_shared = T.alloc_shared([BI, D], dtype)
|
||||
K_tail_shared = T.alloc_shared([BI, D_tail], dtype)
|
||||
if has_tail:
|
||||
K_tail_shared = T.alloc_shared([BI, D_tail], dtype)
|
||||
S_shared = T.alloc_shared([H_per_block, BI], dtype)
|
||||
mask = T.alloc_fragment([BI], T.bool)
|
||||
|
||||
@@ -913,7 +917,8 @@ def sparse_mla_fwd_decode_partial(
|
||||
H1 = H0 + H_per_block
|
||||
|
||||
T.copy(Q[b_i, s_i, H0:H1, :D], Q_buf)
|
||||
T.copy(Q[b_i, s_i, H0:H1, D:], Q_tail_buf)
|
||||
if has_tail:
|
||||
T.copy(Q[b_i, s_i, H0:H1, D:], Q_tail_buf)
|
||||
|
||||
for k_i in T.Pipelined(inner_iter, num_stages=num_stages):
|
||||
topk_block_i = group_i * inner_iter + k_i
|
||||
@@ -925,11 +930,12 @@ def sparse_mla_fwd_decode_partial(
|
||||
KV_shared[bi_i, d_i] = KV[
|
||||
b_i, T.if_then_else(idx >= 0, idx, 0), g_i, d_i
|
||||
]
|
||||
for bi_i, d_i in T.Parallel(BI, D_tail):
|
||||
idx = Indices[b_i, s_i, g_i, topk_block_i * BI + bi_i]
|
||||
K_tail_shared[bi_i, d_i] = KV[
|
||||
b_i, T.if_then_else(idx >= 0, idx, 0), g_i, D + d_i
|
||||
]
|
||||
if has_tail:
|
||||
for bi_i, d_i in T.Parallel(BI, D_tail):
|
||||
idx = Indices[b_i, s_i, g_i, topk_block_i * BI + bi_i]
|
||||
K_tail_shared[bi_i, d_i] = KV[
|
||||
b_i, T.if_then_else(idx >= 0, idx, 0), g_i, D + d_i
|
||||
]
|
||||
|
||||
for h_i, bi_i in T.Parallel(H_per_block, BI):
|
||||
acc_s[h_i, bi_i] = T.if_then_else(
|
||||
@@ -943,13 +949,14 @@ def sparse_mla_fwd_decode_partial(
|
||||
transpose_B=True,
|
||||
policy=T.GemmWarpPolicy.FullCol,
|
||||
)
|
||||
T.gemm(
|
||||
Q_tail_buf,
|
||||
K_tail_shared,
|
||||
acc_s,
|
||||
transpose_B=True,
|
||||
policy=T.GemmWarpPolicy.FullCol,
|
||||
)
|
||||
if has_tail:
|
||||
T.gemm(
|
||||
Q_tail_buf,
|
||||
K_tail_shared,
|
||||
acc_s,
|
||||
transpose_B=True,
|
||||
policy=T.GemmWarpPolicy.FullCol,
|
||||
)
|
||||
|
||||
T.copy(m_i, m_i_prev)
|
||||
T.reduce_max(acc_s, m_i, dim=1, clear=False)
|
||||
@@ -1085,7 +1092,7 @@ def sparse_mla_fwd_decode_partial_fp8(
|
||||
inner_iter=1,
|
||||
threads=256,
|
||||
):
|
||||
assert d_v == 512, f"only support d_v=512"
|
||||
assert d_v in (256, 512), f"only support d_v=256 or d_v=512, got {d_v}"
|
||||
assert topk % block_I == 0, (
|
||||
"otherwise will load some index=0 thus causing wrong kv to be loaded"
|
||||
)
|
||||
@@ -1100,6 +1107,10 @@ def sparse_mla_fwd_decode_partial_fp8(
|
||||
|
||||
BI = block_I
|
||||
group_size = 128
|
||||
assert d_v % group_size == 0
|
||||
assert d_tail in (0, 64), f"only support d_tail=0 or d_tail=64, got {d_tail}"
|
||||
num_main_tiles = d_v // group_size
|
||||
has_tail = d_tail > 0
|
||||
dim_quant_fp8 = d_v + d_tail
|
||||
rope_offset_fp8 = d_v
|
||||
n_groups = topk // (BI * inner_iter)
|
||||
@@ -1149,19 +1160,21 @@ def sparse_mla_fwd_decode_partial_fp8(
|
||||
H0 = (bx % head_blocks_per_seq) * h_per_block
|
||||
H1 = H0 + h_per_block
|
||||
|
||||
# We intentionally split the K=512 GEMM into 4x128 tiles.
|
||||
# Split the main-dimension GEMM into 128-wide tiles.
|
||||
# Although this adds extra intermediate memory traffic,
|
||||
# it shortens the MFMA accumulation dependency chain and improves performance.
|
||||
q_tile0 = T.alloc_shared([h_per_block, group_size], fp8_dtype)
|
||||
q_tile1 = T.alloc_shared([h_per_block, group_size], fp8_dtype)
|
||||
q_tile2 = T.alloc_shared([h_per_block, group_size], fp8_dtype)
|
||||
q_tile3 = T.alloc_shared([h_per_block, group_size], fp8_dtype)
|
||||
kv_tile0 = T.alloc_shared([BI, group_size], fp8_dtype)
|
||||
kv_tile1 = T.alloc_shared([BI, group_size], fp8_dtype)
|
||||
kv_tile2 = T.alloc_shared([BI, group_size], fp8_dtype)
|
||||
kv_tile3 = T.alloc_shared([BI, group_size], fp8_dtype)
|
||||
q_tail_buf = T.alloc_shared([h_per_block, d_tail], fp8_dtype)
|
||||
k_tail_shared = T.alloc_shared([BI, d_tail], fp8_dtype)
|
||||
if num_main_tiles == 4:
|
||||
q_tile2 = T.alloc_shared([h_per_block, group_size], fp8_dtype)
|
||||
q_tile3 = T.alloc_shared([h_per_block, group_size], fp8_dtype)
|
||||
kv_tile2 = T.alloc_shared([BI, group_size], fp8_dtype)
|
||||
kv_tile3 = T.alloc_shared([BI, group_size], fp8_dtype)
|
||||
if has_tail:
|
||||
q_tail_buf = T.alloc_shared([h_per_block, d_tail], fp8_dtype)
|
||||
k_tail_shared = T.alloc_shared([BI, d_tail], fp8_dtype)
|
||||
s_fp8_shared = T.alloc_shared([h_per_block, BI], fp8_dtype)
|
||||
page_idx_shared = T.alloc_shared([BI], T.int32)
|
||||
|
||||
@@ -1178,21 +1191,31 @@ def sparse_mla_fwd_decode_partial_fp8(
|
||||
|
||||
acc_o_tile0 = T.alloc_fragment([h_per_block, group_size], accum_dtype)
|
||||
acc_o_tile1 = T.alloc_fragment([h_per_block, group_size], accum_dtype)
|
||||
acc_o_tile2 = T.alloc_fragment([h_per_block, group_size], accum_dtype)
|
||||
acc_o_tile3 = T.alloc_fragment([h_per_block, group_size], accum_dtype)
|
||||
if num_main_tiles == 4:
|
||||
acc_o_tile2 = T.alloc_fragment([h_per_block, group_size], accum_dtype)
|
||||
acc_o_tile3 = T.alloc_fragment([h_per_block, group_size], accum_dtype)
|
||||
|
||||
T.fill(acc_o_tile0, 0)
|
||||
T.fill(acc_o_tile1, 0)
|
||||
T.fill(acc_o_tile2, 0)
|
||||
T.fill(acc_o_tile3, 0)
|
||||
if num_main_tiles == 4:
|
||||
T.fill(acc_o_tile2, 0)
|
||||
T.fill(acc_o_tile3, 0)
|
||||
T.fill(sumexp, 0)
|
||||
T.fill(m_i, -(2**30))
|
||||
|
||||
T.copy(q_fp8[b_i, s_i, H0:H1, d_v:], q_tail_buf)
|
||||
T.copy(q_fp8[b_i, s_i, H0:H1, 0 * group_size : 1 * group_size], q_tile0)
|
||||
T.copy(q_fp8[b_i, s_i, H0:H1, 1 * group_size : 2 * group_size], q_tile1)
|
||||
T.copy(q_fp8[b_i, s_i, H0:H1, 2 * group_size : 3 * group_size], q_tile2)
|
||||
T.copy(q_fp8[b_i, s_i, H0:H1, 3 * group_size : 4 * group_size], q_tile3)
|
||||
if has_tail:
|
||||
T.copy(q_fp8[b_i, s_i, H0:H1, d_v:], q_tail_buf)
|
||||
T.copy(q_fp8[b_i, s_i, H0:H1, 0:group_size], q_tile0)
|
||||
T.copy(q_fp8[b_i, s_i, H0:H1, group_size : 2 * group_size], q_tile1)
|
||||
if num_main_tiles == 4:
|
||||
T.copy(
|
||||
q_fp8[b_i, s_i, H0:H1, 2 * group_size : 3 * group_size],
|
||||
q_tile2,
|
||||
)
|
||||
T.copy(
|
||||
q_fp8[b_i, s_i, H0:H1, 3 * group_size : 4 * group_size],
|
||||
q_tile3,
|
||||
)
|
||||
|
||||
for k_i in T.serial(inner_iter):
|
||||
topk_block_i = group_i * inner_iter + k_i
|
||||
@@ -1205,14 +1228,18 @@ def sparse_mla_fwd_decode_partial_fp8(
|
||||
|
||||
for bi_i, j in T.Parallel(BI, group_size):
|
||||
page = page_idx_shared[bi_i]
|
||||
kv_tile0[bi_i, j] = kv_fp8[b_i, page, g_i, 0 * group_size + j]
|
||||
kv_tile1[bi_i, j] = kv_fp8[b_i, page, g_i, 1 * group_size + j]
|
||||
kv_tile2[bi_i, j] = kv_fp8[b_i, page, g_i, 2 * group_size + j]
|
||||
kv_tile3[bi_i, j] = kv_fp8[b_i, page, g_i, 3 * group_size + j]
|
||||
kv_tile0[bi_i, j] = kv_fp8[b_i, page, g_i, j]
|
||||
kv_tile1[bi_i, j] = kv_fp8[b_i, page, g_i, group_size + j]
|
||||
if num_main_tiles == 4:
|
||||
kv_tile2[bi_i, j] = kv_fp8[b_i, page, g_i, 2 * group_size + j]
|
||||
kv_tile3[bi_i, j] = kv_fp8[b_i, page, g_i, 3 * group_size + j]
|
||||
|
||||
for bi_i, j in T.Parallel(BI, d_tail):
|
||||
page = page_idx_shared[bi_i]
|
||||
k_tail_shared[bi_i, j] = kv_fp8[b_i, page, g_i, rope_offset_fp8 + j]
|
||||
if has_tail:
|
||||
for bi_i, j in T.Parallel(BI, d_tail):
|
||||
page = page_idx_shared[bi_i]
|
||||
k_tail_shared[bi_i, j] = kv_fp8[
|
||||
b_i, page, g_i, rope_offset_fp8 + j
|
||||
]
|
||||
|
||||
for h_i, bi_i in T.Parallel(h_per_block, BI):
|
||||
acc_s[h_i, bi_i] = T.if_then_else(
|
||||
@@ -1223,19 +1250,33 @@ def sparse_mla_fwd_decode_partial_fp8(
|
||||
T.gemm(q_tile1, kv_tile1, acc_tile, transpose_B=True, clear_accum=True)
|
||||
for h_i, bi_i in T.Parallel(h_per_block, BI):
|
||||
acc_s[h_i, bi_i] += acc_tile[h_i, bi_i]
|
||||
T.gemm(q_tile2, kv_tile2, acc_tile, transpose_B=True, clear_accum=True)
|
||||
for h_i, bi_i in T.Parallel(h_per_block, BI):
|
||||
acc_s[h_i, bi_i] += acc_tile[h_i, bi_i]
|
||||
T.gemm(q_tile3, kv_tile3, acc_tile, transpose_B=True, clear_accum=True)
|
||||
for h_i, bi_i in T.Parallel(h_per_block, BI):
|
||||
acc_s[h_i, bi_i] += acc_tile[h_i, bi_i]
|
||||
T.gemm(
|
||||
q_tail_buf,
|
||||
k_tail_shared,
|
||||
acc_s,
|
||||
transpose_B=True,
|
||||
policy=T.GemmWarpPolicy.FullCol,
|
||||
)
|
||||
if num_main_tiles == 4:
|
||||
T.gemm(
|
||||
q_tile2,
|
||||
kv_tile2,
|
||||
acc_tile,
|
||||
transpose_B=True,
|
||||
clear_accum=True,
|
||||
)
|
||||
for h_i, bi_i in T.Parallel(h_per_block, BI):
|
||||
acc_s[h_i, bi_i] += acc_tile[h_i, bi_i]
|
||||
T.gemm(
|
||||
q_tile3,
|
||||
kv_tile3,
|
||||
acc_tile,
|
||||
transpose_B=True,
|
||||
clear_accum=True,
|
||||
)
|
||||
for h_i, bi_i in T.Parallel(h_per_block, BI):
|
||||
acc_s[h_i, bi_i] += acc_tile[h_i, bi_i]
|
||||
if has_tail:
|
||||
T.gemm(
|
||||
q_tail_buf,
|
||||
k_tail_shared,
|
||||
acc_s,
|
||||
transpose_B=True,
|
||||
policy=T.GemmWarpPolicy.FullCol,
|
||||
)
|
||||
|
||||
T.copy(m_i, m_i_prev)
|
||||
T.reduce_max(acc_s, m_i, dim=1, clear=False)
|
||||
@@ -1249,10 +1290,11 @@ def sparse_mla_fwd_decode_partial_fp8(
|
||||
for h_i in T.Parallel(h_per_block):
|
||||
sumexp[h_i] = sumexp[h_i] * alpha[h_i] + sumexp_i[h_i]
|
||||
for h_i, j in T.Parallel(h_per_block, group_size):
|
||||
acc_o_tile0[h_i, j] = acc_o_tile0[h_i, j] * alpha[h_i]
|
||||
acc_o_tile1[h_i, j] = acc_o_tile1[h_i, j] * alpha[h_i]
|
||||
acc_o_tile2[h_i, j] = acc_o_tile2[h_i, j] * alpha[h_i]
|
||||
acc_o_tile3[h_i, j] = acc_o_tile3[h_i, j] * alpha[h_i]
|
||||
acc_o_tile0[h_i, j] *= alpha[h_i]
|
||||
acc_o_tile1[h_i, j] *= alpha[h_i]
|
||||
if num_main_tiles == 4:
|
||||
acc_o_tile2[h_i, j] *= alpha[h_i]
|
||||
acc_o_tile3[h_i, j] *= alpha[h_i]
|
||||
|
||||
for h_i, bi_i in T.Parallel(h_per_block, BI):
|
||||
s_fp8_shared[h_i, bi_i] = T.clamp(
|
||||
@@ -1262,36 +1304,27 @@ def sparse_mla_fwd_decode_partial_fp8(
|
||||
)
|
||||
T.gemm(s_fp8_shared, kv_tile0, sv_tile, clear_accum=True)
|
||||
for h_i, j in T.Parallel(h_per_block, group_size):
|
||||
acc_o_tile0[h_i, j] = (
|
||||
acc_o_tile0[h_i, j] + sv_tile[h_i, j] * s_scale_const
|
||||
)
|
||||
|
||||
acc_o_tile0[h_i, j] += sv_tile[h_i, j] * s_scale_const
|
||||
T.gemm(s_fp8_shared, kv_tile1, sv_tile, clear_accum=True)
|
||||
for h_i, j in T.Parallel(h_per_block, group_size):
|
||||
acc_o_tile1[h_i, j] = (
|
||||
acc_o_tile1[h_i, j] + sv_tile[h_i, j] * s_scale_const
|
||||
)
|
||||
|
||||
T.gemm(s_fp8_shared, kv_tile2, sv_tile, clear_accum=True)
|
||||
for h_i, j in T.Parallel(h_per_block, group_size):
|
||||
acc_o_tile2[h_i, j] = (
|
||||
acc_o_tile2[h_i, j] + sv_tile[h_i, j] * s_scale_const
|
||||
)
|
||||
|
||||
T.gemm(s_fp8_shared, kv_tile3, sv_tile, clear_accum=True)
|
||||
for h_i, j in T.Parallel(h_per_block, group_size):
|
||||
acc_o_tile3[h_i, j] = (
|
||||
acc_o_tile3[h_i, j] + sv_tile[h_i, j] * s_scale_const
|
||||
)
|
||||
acc_o_tile1[h_i, j] += sv_tile[h_i, j] * s_scale_const
|
||||
if num_main_tiles == 4:
|
||||
T.gemm(s_fp8_shared, kv_tile2, sv_tile, clear_accum=True)
|
||||
for h_i, j in T.Parallel(h_per_block, group_size):
|
||||
acc_o_tile2[h_i, j] += sv_tile[h_i, j] * s_scale_const
|
||||
T.gemm(s_fp8_shared, kv_tile3, sv_tile, clear_accum=True)
|
||||
for h_i, j in T.Parallel(h_per_block, group_size):
|
||||
acc_o_tile3[h_i, j] += sv_tile[h_i, j] * s_scale_const
|
||||
|
||||
for h_i in T.Parallel(h_per_block):
|
||||
denom = T.if_then_else(sumexp[h_i] == 0.0, 1.0, sumexp[h_i])
|
||||
inv_denom[h_i] = 1.0 / denom
|
||||
for h_i, j in T.Parallel(h_per_block, group_size):
|
||||
acc_o_tile0[h_i, j] = acc_o_tile0[h_i, j] * inv_denom[h_i]
|
||||
acc_o_tile1[h_i, j] = acc_o_tile1[h_i, j] * inv_denom[h_i]
|
||||
acc_o_tile2[h_i, j] = acc_o_tile2[h_i, j] * inv_denom[h_i]
|
||||
acc_o_tile3[h_i, j] = acc_o_tile3[h_i, j] * inv_denom[h_i]
|
||||
acc_o_tile0[h_i, j] *= inv_denom[h_i]
|
||||
acc_o_tile1[h_i, j] *= inv_denom[h_i]
|
||||
if num_main_tiles == 4:
|
||||
acc_o_tile2[h_i, j] *= inv_denom[h_i]
|
||||
acc_o_tile3[h_i, j] *= inv_denom[h_i]
|
||||
|
||||
for h_i in T.Parallel(h_per_block):
|
||||
sumexp[h_i] = T.if_then_else(
|
||||
@@ -1300,22 +1333,24 @@ def sparse_mla_fwd_decode_partial_fp8(
|
||||
T.log2(sumexp[h_i]) + m_i[h_i] * sm_scale,
|
||||
)
|
||||
|
||||
T.copy(
|
||||
acc_o_tile0,
|
||||
partial_o[b_i, s_i, group_i, H0:H1, 0 * group_size : 1 * group_size],
|
||||
)
|
||||
T.copy(acc_o_tile0, partial_o[b_i, s_i, group_i, H0:H1, 0:group_size])
|
||||
T.copy(
|
||||
acc_o_tile1,
|
||||
partial_o[b_i, s_i, group_i, H0:H1, 1 * group_size : 2 * group_size],
|
||||
)
|
||||
T.copy(
|
||||
acc_o_tile2,
|
||||
partial_o[b_i, s_i, group_i, H0:H1, 2 * group_size : 3 * group_size],
|
||||
)
|
||||
T.copy(
|
||||
acc_o_tile3,
|
||||
partial_o[b_i, s_i, group_i, H0:H1, 3 * group_size : 4 * group_size],
|
||||
partial_o[b_i, s_i, group_i, H0:H1, group_size : 2 * group_size],
|
||||
)
|
||||
if num_main_tiles == 4:
|
||||
T.copy(
|
||||
acc_o_tile2,
|
||||
partial_o[
|
||||
b_i, s_i, group_i, H0:H1, 2 * group_size : 3 * group_size
|
||||
],
|
||||
)
|
||||
T.copy(
|
||||
acc_o_tile3,
|
||||
partial_o[
|
||||
b_i, s_i, group_i, H0:H1, 3 * group_size : 4 * group_size
|
||||
],
|
||||
)
|
||||
|
||||
T.copy(sumexp, partial_lse[b_i, s_i, group_i, H0:H1])
|
||||
|
||||
|
||||
@@ -4601,7 +4601,8 @@ class MLATokenToKVPool(KVCache):
|
||||
and (self.use_dsa or self.dsa_kv_cache_store_fp8)
|
||||
), "the DSA write paths have no resolved-loc variant"
|
||||
if _is_hip and self.use_dsa and self.dtype == fp8_dtype:
|
||||
# HIP FP8 path uses raw MLA KV layout (nope + rope) without per-block scales.
|
||||
# HIP TileLang FP8 consumes the model's raw MLA layout
|
||||
# (NoPE + optional RoPE) without per-block scales.
|
||||
# Fuse BF16/FP16 -> FP8 cast with paged KV write.
|
||||
set_mla_kv_buffer_triton_fp8_quant(
|
||||
dst_buffer,
|
||||
@@ -4619,8 +4620,8 @@ class MLATokenToKVPool(KVCache):
|
||||
)
|
||||
|
||||
# Reuse existing two-tensor write kernel (works with FP8 byte layout)
|
||||
# cache_k_nope_fp8: (num_tokens, 1, 528) uint8 [nope_fp8(512) | scales(16)]
|
||||
# cache_k_rope_fp8: (num_tokens, 1, 128) uint8 [rope_bf16_bytes(128)]
|
||||
# NoPE bytes are [FP8 values | one FP32 scale per 128 values].
|
||||
# RoPE bytes contain the raw BF16 tail and may have zero width.
|
||||
self._scatter_mla_rows(dst_buffer, loc, cache_k_nope_fp8, cache_k_rope_fp8)
|
||||
else:
|
||||
if cache_k_nope.dtype != self.dtype:
|
||||
|
||||
Reference in New Issue
Block a user