[AMD] Add .co for deepseek v4 fp8 decode kernel and add group decode opt (#41120)

This commit is contained in:
Thomas Wang
2026-09-24 12:21:34 -07:00
committed by GitHub
parent 5bb850d536
commit 1446e24d13
5 changed files with 251 additions and 17 deletions
@@ -0,0 +1,139 @@
"""
Index streams for the grouped target-verify decode.
The per-token HCA stream is [swa][compressed tail], one per draft, so a
request's kv is read once per draft. The asm kernel can take four drafts
in one tile, but only if the stream is laid out the way its masking
already works:
[compressed tail][swa]
"""
from typing import Tuple
import torch
import triton
import triton.language as tl
# asm v4 only support specified max_seqlen_q
_ALLOWED_BLOCK_Q = (1, 2, 4)
@triton.jit
def _grouped_stream_kernel(
out_ptr, # [*] int32
indptr_ptr, # [G+1] int32
slot_ptr, # [G] int32, state slot of the group's request
pos_ptr, # [G] int32, position of the group's LAST draft
tail_len_ptr, # [G] int32, compressed entries, all drafts see them
tail_page_ptr, # [G, TAIL_W] int32
ring_stride,
swa_pages,
TAIL_W: tl.constexpr,
TAIL_B: tl.constexpr,
WIN_B: tl.constexpr,
):
g = tl.program_id(0)
base = tl.load(indptr_ptr + g)
n_tail = tl.load(tail_len_ptr + g).to(tl.int32)
# The tail goes first. Every draft in the group sees all of it.
for off in tl.range(0, TAIL_W, TAIL_B):
j = off + tl.arange(0, TAIL_B)
m = j < n_tail
jc = tl.minimum(j, TAIL_W - 1)
pi = tl.load(tail_page_ptr + g * TAIL_W + jc, mask=m, other=-1).to(tl.int32)
tl.store(out_ptr + base + j, tl.where(pi >= 0, pi + swa_pages, -1), mask=m)
# Then the swa from this group's first draft to its last.
slot = tl.load(slot_ptr + g)
pos = tl.load(pos_ptr + g)
n = tl.load(indptr_ptr + g + 1) - base - n_tail
i = tl.arange(0, WIN_B)
m = i < n
abs_pos = pos - n + 1 + i
val = slot * ring_stride + abs_pos % ring_stride
tl.store(out_ptr + base + n_tail + i, val, mask=m)
def _group_index(R, num_draft, block_q, device):
"""
Per-group gather index and group size, built on the device.
With R=3, num_draft=7, block_q=4 the six groups come out as
last_token = [3, 6, 10, 13, 17, 20]
group_size = [4, 3, 4, 3, 4, 3]
"""
n_full = num_draft // block_q
remainder = num_draft % block_q
n_groups = n_full + (1 if remainder else 0)
ends = (
torch.arange(1, n_groups + 1, dtype=torch.int64, device=device) * block_q
).clamp(max=num_draft)
base = torch.arange(R, dtype=torch.int64, device=device) * num_draft
last_token = (base[:, None] + (ends - 1)).reshape(-1)
group_size = torch.diff(ends, prepend=ends.new_zeros(1))
return last_token, group_size.to(torch.int32).repeat(R)
def build_grouped_verify_streams(
*,
state_slot: torch.Tensor, # [N] int, per query token
positions: torch.Tensor, # [N] int, per query token
hca_len: torch.Tensor, # [N] int, compressed entries per token
hca_page_indices: torch.Tensor, # [N, Wc] int32
win: int,
ring_stride: int,
swa_pages: int,
num_draft: int,
block_q: int = 4,
) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
"""
Returns (indices, kv_indptr, qo_indptr) for the grouped call.
"""
assert block_q in _ALLOWED_BLOCK_Q, (
f"block_q={block_q} is not one of {_ALLOWED_BLOCK_Q}; aiter's v4 "
"dispatcher would not route it to the tile this path needs"
)
dev = state_slot.device
N = state_slot.shape[0]
assert N % num_draft == 0, f"{N} tokens is not a whole number of drafts"
R = N // num_draft
last_token, group_size = _group_index(R, num_draft, block_q, dev)
G = last_token.numel()
slot_g = state_slot.to(torch.int32)[last_token].contiguous()
pos_g = positions.to(torch.int32)[last_token].contiguous()
tail_g = hca_len.to(torch.int32)[last_token].contiguous()
page_g = hca_page_indices[last_token].contiguous()
lens = tail_g + torch.minimum(pos_g + 1, win + group_size - 1)
kv_indptr = torch.zeros((G + 1,), dtype=torch.int32, device=dev)
kv_indptr[1:] = lens.cumsum(0).to(torch.int32)
Wc = page_g.shape[1]
# Sized to the worst case rather than to kv_indptr[-1], which would need a
# device read and so could not be captured in a graph. The streams are
# packed by kv_indptr, so the slack past the last one is never touched.
indices = torch.empty(
(G * (Wc + win + block_q - 1),), dtype=torch.int32, device=dev
)
_grouped_stream_kernel[(G,)](
indices,
kv_indptr,
slot_g,
pos_g,
tail_g,
page_g,
ring_stride,
swa_pages,
TAIL_W=max(Wc, 1),
TAIL_B=min(triton.next_power_of_2(max(Wc, 1)), 1024),
WIN_B=triton.next_power_of_2(win + block_q - 1),
num_warps=4,
)
qo_indptr = torch.zeros((G + 1,), dtype=torch.int32, device=dev)
qo_indptr[1:] = group_size.cumsum(0).to(torch.int32)
return indices, kv_indptr, qo_indptr
@@ -293,6 +293,7 @@ def decode_fp8_2buff(
attn_sink: torch.Tensor, # [H] fp32
v_head_dim: int,
qo_indptr: Optional[torch.Tensor] = None,
max_seqlen_q: int = 1,
num_kv_splits: Optional[int] = None,
compress_ratio: Optional[int] = None,
) -> torch.Tensor:
@@ -310,6 +311,9 @@ def decode_fp8_2buff(
``compress_ratio`` names the stream (0 SWA, 4 CSA, 128 HCA) so the split
count past 40 tokens can fit the stream; None keeps the fixed 4.
``max_seqlen_q`` above 1 puts several q rows in one tile, which the v4
dispatcher serves for gqa=16 at 1, 2 and 4 off the same 64-row tile.
"""
from aiter.mla import mla_decode_fwd_v4_nm
@@ -339,24 +343,21 @@ def decode_fp8_2buff(
if qo_indptr is None:
qo_indptr = decode_qo_indptr(T, q.device)
# num_seqs comes from qo_indptr.numel()-1 and the kernel writes
# num_seqs * max_seqlen_q rows into `out`, so both have to be sized off q's
# own T. A qo_indptr built from a padded token count writes past `out`.
assert qo_indptr.shape[0] >= T + 1, (
f"qo_indptr holds {qo_indptr.shape[0]} entries, kernel reads {T + 1}"
n_seq = qo_indptr.shape[0] - 1
assert kv_indptr.shape[0] >= n_seq + 1, (
f"kv_indptr holds {kv_indptr.shape[0]} entries, kernel reads {n_seq + 1}"
)
assert kv_indptr.shape[0] >= T + 1, (
f"kv_indptr holds {kv_indptr.shape[0]} entries, kernel reads {T + 1}"
)
qo_indptr = qo_indptr[: T + 1]
rows = unified_kv.shape[0]
out = q_rope.new_empty((T, H, v_head_dim))
out = q_rope.new_empty((n_seq * max_seqlen_q, H, v_head_dim))
# Left None, the wrapper's occupancy heuristic picks it, folds the cross-split
# merge back into `out`, and leaves the final bf16 there whether or not it
# split. Pinning it to 1 costs 6.9x at bs=1 kv=2048.
if num_kv_splits is None and T > _DECODE_SPLIT_TAIL_MIN_TOKENS:
num_kv_splits = _decode_fp8_tail_splits(T, compress_ratio)
# Sequences, not tokens: the wave-fitting rule counts the workgroups the
# kernel launches, which is n_seq x splits. They are the same number on the
# per-token path and differ once q rows share a tile.
if num_kv_splits is None and n_seq > _DECODE_SPLIT_TAIL_MIN_TOKENS:
num_kv_splits = _decode_fp8_tail_splits(n_seq, compress_ratio)
mla_decode_fwd_v4_nm(
q,
q_rope,
@@ -366,7 +367,7 @@ def decode_fp8_2buff(
qo_indptr,
kv_indptr,
kv_indices,
1, # max_seqlen_q; qo_indptr is per-token so every sequence is one token
max_seqlen_q,
sink=attn_sink,
num_kv_splits=num_kv_splits,
)
@@ -374,7 +375,7 @@ def decode_fp8_2buff(
# ReqToTokenPool reserves, so the builders can't emit a zero-length one, and
# the compare + masked_fill_ was costing a launch per layer for it. One would
# come back NaN now (all-sink denominator); the guard UT pins that.
return out
return out[:T]
@triton.jit
@@ -3,6 +3,7 @@ from __future__ import annotations
import enum
import functools
import logging
import os
from dataclasses import dataclass, field
from typing import (
TYPE_CHECKING,
@@ -104,6 +105,14 @@ class UnifiedKvMetadata:
csa_indices: Optional[torch.Tensor] = None
csa_indptr: Optional[torch.Tensor] = None
# Grouped target-verify streams for the asm decode: one per (request,
# draft group), laid out [compressed tail][that group's window slice] so
# the kernel's own causal bound and band mask land on each draft's real
# cutoff.
gasm_indices: Optional[torch.Tensor] = None
gasm_kv_indptr: Optional[torch.Tensor] = None
gasm_qo_indptr: Optional[torch.Tensor] = None
# prefill/extend per-token mapping
pf_state_slot: Optional[torch.Tensor] = None
pf_chunk_start: Optional[torch.Tensor] = None
@@ -131,6 +140,9 @@ class UnifiedKvMetadata:
"swa_indptr",
"hca_indices",
"hca_indptr",
"gasm_indices",
"gasm_kv_indptr",
"gasm_qo_indptr",
"csa_indices",
"csa_indptr",
"pf_state_slot",
@@ -160,6 +172,9 @@ class UnifiedKvMetadata:
"swa_indptr",
"hca_indices",
"hca_indptr",
"gasm_indices",
"gasm_kv_indptr",
"gasm_qo_indptr",
"csa_indices",
"csa_indptr",
"pf_state_slot",
@@ -511,6 +526,24 @@ class DSV4RawDecodeMetadata:
self.out_cache_loc.copy_(other.out_cache_loc)
_GROUPED_ASM_BLOCK_Q = 4
# aiter serves gqa*msq off one 64-row q tile and only whitelists msq above 1
# at gqa=16 -- the TP-only shape. DP attention gives a rank all 128 heads, so
# the tile is full of heads with no room to group, and none is needed either:
# one kv read already feeds 128 heads rather than 16. An unsupported pair does
# not downgrade, it fails aiter's kernel lookup outright.
_GROUPED_ASM_GQA = 16
_GROUPED_ASM_MIN_REQS = 16
def _grouped_asm_enabled() -> bool:
"""
Grouped target-verify decode through the asm kernel, off by default.
"""
return os.environ.get("SGLANG_DSV4_GROUPED_ASM", "0") == "1"
class _GraphBucket(enum.Enum):
DECODE_OR_IDLE = "decode_or_idle"
TARGET_VERIFY = "target_verify"
@@ -714,7 +747,9 @@ class DeepseekV4HipRadixBackend(
# per-token (num_draft*bs -> bs) req-slot map produced by the prefill
# expansion above.
self._attach_unified_kv_decode_streams(
core_attn_metadata, req_pool_indices_repeated
core_attn_metadata,
req_pool_indices_repeated,
num_draft=self.target_verify_num_draft_tokens,
)
indexer_metadata = (
self.init_forward_metadata_indexer(core_attn_metadata)
@@ -923,7 +958,9 @@ class DeepseekV4HipRadixBackend(
num_draft_tokens * bs,
)
self._attach_unified_kv_decode_streams(
core_attn_metadata, req_pool_indices_repeated
core_attn_metadata,
req_pool_indices_repeated,
num_draft=self.target_verify_num_draft_tokens,
)
indexer_metadata = (
self.init_forward_metadata_indexer(core_attn_metadata)
@@ -1469,7 +1506,10 @@ class DeepseekV4HipRadixBackend(
self.forward_metadata = current_raw
def _attach_unified_kv_decode_streams(
self, core: DSV4AttnMetadata, state_slot: torch.Tensor
self,
core: DSV4AttnMetadata,
state_slot: torch.Tensor,
num_draft: int = 0,
) -> None:
# state_slot maps each query token to its request slot;
# target-verify repeats request slots for the draft tokens.
@@ -1521,6 +1561,35 @@ class DeepseekV4HipRadixBackend(
ring_stride=pool.unified_swa_ring_size,
swa_pages=pool.unified_swa_pages,
)
core.unified.gasm_indices = None
core.unified.gasm_kv_indptr = None
core.unified.gasm_qo_indptr = None
if (
_grouped_asm_enabled()
and num_draft > 1
and N % num_draft == 0
and N // num_draft >= _GROUPED_ASM_MIN_REQS
):
from sglang.kernels.ops.attention.dsv4.unified_kv_kernels import (
grouped_verify_streams,
)
(
core.unified.gasm_indices,
core.unified.gasm_kv_indptr,
core.unified.gasm_qo_indptr,
) = grouped_verify_streams.build_grouped_verify_streams(
state_slot=state_slot,
positions=core.positions_casual,
hca_len=hca_len,
hca_page_indices=hca_page_indices,
win=pool.unified_swa_window,
ring_stride=pool.unified_swa_ring_size,
swa_pages=pool.unified_swa_pages,
num_draft=num_draft,
block_q=_GROUPED_ASM_BLOCK_Q,
)
# SWA ring write target, same value for every layer this forward.
req_slot = state_slot.to(torch.int64)
core.unified.swa_loc = (
@@ -1691,6 +1760,25 @@ class DeepseekV4HipRadixBackend(
"the v4 nm asm kernel hardcodes 1/sqrt(512), this backend is "
f"at {self.softmax_scale}"
)
gasm = (
unified_metadata.gasm_indices
if compress_ratio == 128 and q.shape[1] == _GROUPED_ASM_GQA
else None
)
if gasm is not None:
return runtime.decode_fp8_2buff(
q=q,
q_rope=q_rope,
unified_kv=unified,
unified_kv_rope=pool.get_unified_kv_rope(layer_id),
kv_indices=gasm,
kv_indptr=unified_metadata.gasm_kv_indptr,
qo_indptr=unified_metadata.gasm_qo_indptr,
attn_sink=attn_sink,
v_head_dim=layer.v_head_dim,
max_seqlen_q=_GROUPED_ASM_BLOCK_Q,
compress_ratio=compress_ratio,
)
return runtime.decode_fp8_2buff(
q=q,
q_rope=q_rope,