[AMD] Add .co for deepseek v4 fp8 decode kernel and add group decode opt (#41120)
This commit is contained in:
Executable
BIN
Binary file not shown.
+139
@@ -0,0 +1,139 @@
|
||||
"""
|
||||
Index streams for the grouped target-verify decode.
|
||||
|
||||
The per-token HCA stream is [swa][compressed tail], one per draft, so a
|
||||
request's kv is read once per draft. The asm kernel can take four drafts
|
||||
in one tile, but only if the stream is laid out the way its masking
|
||||
already works:
|
||||
|
||||
[compressed tail][swa]
|
||||
"""
|
||||
|
||||
from typing import Tuple
|
||||
|
||||
import torch
|
||||
import triton
|
||||
import triton.language as tl
|
||||
|
||||
# asm v4 only support specified max_seqlen_q
|
||||
_ALLOWED_BLOCK_Q = (1, 2, 4)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _grouped_stream_kernel(
|
||||
out_ptr, # [*] int32
|
||||
indptr_ptr, # [G+1] int32
|
||||
slot_ptr, # [G] int32, state slot of the group's request
|
||||
pos_ptr, # [G] int32, position of the group's LAST draft
|
||||
tail_len_ptr, # [G] int32, compressed entries, all drafts see them
|
||||
tail_page_ptr, # [G, TAIL_W] int32
|
||||
ring_stride,
|
||||
swa_pages,
|
||||
TAIL_W: tl.constexpr,
|
||||
TAIL_B: tl.constexpr,
|
||||
WIN_B: tl.constexpr,
|
||||
):
|
||||
g = tl.program_id(0)
|
||||
base = tl.load(indptr_ptr + g)
|
||||
n_tail = tl.load(tail_len_ptr + g).to(tl.int32)
|
||||
|
||||
# The tail goes first. Every draft in the group sees all of it.
|
||||
for off in tl.range(0, TAIL_W, TAIL_B):
|
||||
j = off + tl.arange(0, TAIL_B)
|
||||
m = j < n_tail
|
||||
jc = tl.minimum(j, TAIL_W - 1)
|
||||
pi = tl.load(tail_page_ptr + g * TAIL_W + jc, mask=m, other=-1).to(tl.int32)
|
||||
tl.store(out_ptr + base + j, tl.where(pi >= 0, pi + swa_pages, -1), mask=m)
|
||||
|
||||
# Then the swa from this group's first draft to its last.
|
||||
slot = tl.load(slot_ptr + g)
|
||||
pos = tl.load(pos_ptr + g)
|
||||
n = tl.load(indptr_ptr + g + 1) - base - n_tail
|
||||
i = tl.arange(0, WIN_B)
|
||||
m = i < n
|
||||
abs_pos = pos - n + 1 + i
|
||||
val = slot * ring_stride + abs_pos % ring_stride
|
||||
tl.store(out_ptr + base + n_tail + i, val, mask=m)
|
||||
|
||||
|
||||
def _group_index(R, num_draft, block_q, device):
|
||||
"""
|
||||
Per-group gather index and group size, built on the device.
|
||||
|
||||
With R=3, num_draft=7, block_q=4 the six groups come out as
|
||||
|
||||
last_token = [3, 6, 10, 13, 17, 20]
|
||||
group_size = [4, 3, 4, 3, 4, 3]
|
||||
"""
|
||||
n_full = num_draft // block_q
|
||||
remainder = num_draft % block_q
|
||||
n_groups = n_full + (1 if remainder else 0)
|
||||
ends = (
|
||||
torch.arange(1, n_groups + 1, dtype=torch.int64, device=device) * block_q
|
||||
).clamp(max=num_draft)
|
||||
base = torch.arange(R, dtype=torch.int64, device=device) * num_draft
|
||||
last_token = (base[:, None] + (ends - 1)).reshape(-1)
|
||||
group_size = torch.diff(ends, prepend=ends.new_zeros(1))
|
||||
return last_token, group_size.to(torch.int32).repeat(R)
|
||||
|
||||
|
||||
def build_grouped_verify_streams(
|
||||
*,
|
||||
state_slot: torch.Tensor, # [N] int, per query token
|
||||
positions: torch.Tensor, # [N] int, per query token
|
||||
hca_len: torch.Tensor, # [N] int, compressed entries per token
|
||||
hca_page_indices: torch.Tensor, # [N, Wc] int32
|
||||
win: int,
|
||||
ring_stride: int,
|
||||
swa_pages: int,
|
||||
num_draft: int,
|
||||
block_q: int = 4,
|
||||
) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
Returns (indices, kv_indptr, qo_indptr) for the grouped call.
|
||||
"""
|
||||
assert block_q in _ALLOWED_BLOCK_Q, (
|
||||
f"block_q={block_q} is not one of {_ALLOWED_BLOCK_Q}; aiter's v4 "
|
||||
"dispatcher would not route it to the tile this path needs"
|
||||
)
|
||||
dev = state_slot.device
|
||||
N = state_slot.shape[0]
|
||||
assert N % num_draft == 0, f"{N} tokens is not a whole number of drafts"
|
||||
R = N // num_draft
|
||||
last_token, group_size = _group_index(R, num_draft, block_q, dev)
|
||||
G = last_token.numel()
|
||||
|
||||
slot_g = state_slot.to(torch.int32)[last_token].contiguous()
|
||||
pos_g = positions.to(torch.int32)[last_token].contiguous()
|
||||
tail_g = hca_len.to(torch.int32)[last_token].contiguous()
|
||||
page_g = hca_page_indices[last_token].contiguous()
|
||||
|
||||
lens = tail_g + torch.minimum(pos_g + 1, win + group_size - 1)
|
||||
kv_indptr = torch.zeros((G + 1,), dtype=torch.int32, device=dev)
|
||||
kv_indptr[1:] = lens.cumsum(0).to(torch.int32)
|
||||
|
||||
Wc = page_g.shape[1]
|
||||
# Sized to the worst case rather than to kv_indptr[-1], which would need a
|
||||
# device read and so could not be captured in a graph. The streams are
|
||||
# packed by kv_indptr, so the slack past the last one is never touched.
|
||||
indices = torch.empty(
|
||||
(G * (Wc + win + block_q - 1),), dtype=torch.int32, device=dev
|
||||
)
|
||||
_grouped_stream_kernel[(G,)](
|
||||
indices,
|
||||
kv_indptr,
|
||||
slot_g,
|
||||
pos_g,
|
||||
tail_g,
|
||||
page_g,
|
||||
ring_stride,
|
||||
swa_pages,
|
||||
TAIL_W=max(Wc, 1),
|
||||
TAIL_B=min(triton.next_power_of_2(max(Wc, 1)), 1024),
|
||||
WIN_B=triton.next_power_of_2(win + block_q - 1),
|
||||
num_warps=4,
|
||||
)
|
||||
|
||||
qo_indptr = torch.zeros((G + 1,), dtype=torch.int32, device=dev)
|
||||
qo_indptr[1:] = group_size.cumsum(0).to(torch.int32)
|
||||
return indices, kv_indptr, qo_indptr
|
||||
@@ -293,6 +293,7 @@ def decode_fp8_2buff(
|
||||
attn_sink: torch.Tensor, # [H] fp32
|
||||
v_head_dim: int,
|
||||
qo_indptr: Optional[torch.Tensor] = None,
|
||||
max_seqlen_q: int = 1,
|
||||
num_kv_splits: Optional[int] = None,
|
||||
compress_ratio: Optional[int] = None,
|
||||
) -> torch.Tensor:
|
||||
@@ -310,6 +311,9 @@ def decode_fp8_2buff(
|
||||
|
||||
``compress_ratio`` names the stream (0 SWA, 4 CSA, 128 HCA) so the split
|
||||
count past 40 tokens can fit the stream; None keeps the fixed 4.
|
||||
|
||||
``max_seqlen_q`` above 1 puts several q rows in one tile, which the v4
|
||||
dispatcher serves for gqa=16 at 1, 2 and 4 off the same 64-row tile.
|
||||
"""
|
||||
from aiter.mla import mla_decode_fwd_v4_nm
|
||||
|
||||
@@ -339,24 +343,21 @@ def decode_fp8_2buff(
|
||||
|
||||
if qo_indptr is None:
|
||||
qo_indptr = decode_qo_indptr(T, q.device)
|
||||
# num_seqs comes from qo_indptr.numel()-1 and the kernel writes
|
||||
# num_seqs * max_seqlen_q rows into `out`, so both have to be sized off q's
|
||||
# own T. A qo_indptr built from a padded token count writes past `out`.
|
||||
assert qo_indptr.shape[0] >= T + 1, (
|
||||
f"qo_indptr holds {qo_indptr.shape[0]} entries, kernel reads {T + 1}"
|
||||
n_seq = qo_indptr.shape[0] - 1
|
||||
assert kv_indptr.shape[0] >= n_seq + 1, (
|
||||
f"kv_indptr holds {kv_indptr.shape[0]} entries, kernel reads {n_seq + 1}"
|
||||
)
|
||||
assert kv_indptr.shape[0] >= T + 1, (
|
||||
f"kv_indptr holds {kv_indptr.shape[0]} entries, kernel reads {T + 1}"
|
||||
)
|
||||
qo_indptr = qo_indptr[: T + 1]
|
||||
|
||||
rows = unified_kv.shape[0]
|
||||
out = q_rope.new_empty((T, H, v_head_dim))
|
||||
out = q_rope.new_empty((n_seq * max_seqlen_q, H, v_head_dim))
|
||||
# Left None, the wrapper's occupancy heuristic picks it, folds the cross-split
|
||||
# merge back into `out`, and leaves the final bf16 there whether or not it
|
||||
# split. Pinning it to 1 costs 6.9x at bs=1 kv=2048.
|
||||
if num_kv_splits is None and T > _DECODE_SPLIT_TAIL_MIN_TOKENS:
|
||||
num_kv_splits = _decode_fp8_tail_splits(T, compress_ratio)
|
||||
# Sequences, not tokens: the wave-fitting rule counts the workgroups the
|
||||
# kernel launches, which is n_seq x splits. They are the same number on the
|
||||
# per-token path and differ once q rows share a tile.
|
||||
if num_kv_splits is None and n_seq > _DECODE_SPLIT_TAIL_MIN_TOKENS:
|
||||
num_kv_splits = _decode_fp8_tail_splits(n_seq, compress_ratio)
|
||||
mla_decode_fwd_v4_nm(
|
||||
q,
|
||||
q_rope,
|
||||
@@ -366,7 +367,7 @@ def decode_fp8_2buff(
|
||||
qo_indptr,
|
||||
kv_indptr,
|
||||
kv_indices,
|
||||
1, # max_seqlen_q; qo_indptr is per-token so every sequence is one token
|
||||
max_seqlen_q,
|
||||
sink=attn_sink,
|
||||
num_kv_splits=num_kv_splits,
|
||||
)
|
||||
@@ -374,7 +375,7 @@ def decode_fp8_2buff(
|
||||
# ReqToTokenPool reserves, so the builders can't emit a zero-length one, and
|
||||
# the compare + masked_fill_ was costing a launch per layer for it. One would
|
||||
# come back NaN now (all-sink denominator); the guard UT pins that.
|
||||
return out
|
||||
return out[:T]
|
||||
|
||||
|
||||
@triton.jit
|
||||
|
||||
@@ -3,6 +3,7 @@ from __future__ import annotations
|
||||
import enum
|
||||
import functools
|
||||
import logging
|
||||
import os
|
||||
from dataclasses import dataclass, field
|
||||
from typing import (
|
||||
TYPE_CHECKING,
|
||||
@@ -104,6 +105,14 @@ class UnifiedKvMetadata:
|
||||
csa_indices: Optional[torch.Tensor] = None
|
||||
csa_indptr: Optional[torch.Tensor] = None
|
||||
|
||||
# Grouped target-verify streams for the asm decode: one per (request,
|
||||
# draft group), laid out [compressed tail][that group's window slice] so
|
||||
# the kernel's own causal bound and band mask land on each draft's real
|
||||
# cutoff.
|
||||
gasm_indices: Optional[torch.Tensor] = None
|
||||
gasm_kv_indptr: Optional[torch.Tensor] = None
|
||||
gasm_qo_indptr: Optional[torch.Tensor] = None
|
||||
|
||||
# prefill/extend per-token mapping
|
||||
pf_state_slot: Optional[torch.Tensor] = None
|
||||
pf_chunk_start: Optional[torch.Tensor] = None
|
||||
@@ -131,6 +140,9 @@ class UnifiedKvMetadata:
|
||||
"swa_indptr",
|
||||
"hca_indices",
|
||||
"hca_indptr",
|
||||
"gasm_indices",
|
||||
"gasm_kv_indptr",
|
||||
"gasm_qo_indptr",
|
||||
"csa_indices",
|
||||
"csa_indptr",
|
||||
"pf_state_slot",
|
||||
@@ -160,6 +172,9 @@ class UnifiedKvMetadata:
|
||||
"swa_indptr",
|
||||
"hca_indices",
|
||||
"hca_indptr",
|
||||
"gasm_indices",
|
||||
"gasm_kv_indptr",
|
||||
"gasm_qo_indptr",
|
||||
"csa_indices",
|
||||
"csa_indptr",
|
||||
"pf_state_slot",
|
||||
@@ -511,6 +526,24 @@ class DSV4RawDecodeMetadata:
|
||||
self.out_cache_loc.copy_(other.out_cache_loc)
|
||||
|
||||
|
||||
_GROUPED_ASM_BLOCK_Q = 4
|
||||
|
||||
# aiter serves gqa*msq off one 64-row q tile and only whitelists msq above 1
|
||||
# at gqa=16 -- the TP-only shape. DP attention gives a rank all 128 heads, so
|
||||
# the tile is full of heads with no room to group, and none is needed either:
|
||||
# one kv read already feeds 128 heads rather than 16. An unsupported pair does
|
||||
# not downgrade, it fails aiter's kernel lookup outright.
|
||||
_GROUPED_ASM_GQA = 16
|
||||
_GROUPED_ASM_MIN_REQS = 16
|
||||
|
||||
|
||||
def _grouped_asm_enabled() -> bool:
|
||||
"""
|
||||
Grouped target-verify decode through the asm kernel, off by default.
|
||||
"""
|
||||
return os.environ.get("SGLANG_DSV4_GROUPED_ASM", "0") == "1"
|
||||
|
||||
|
||||
class _GraphBucket(enum.Enum):
|
||||
DECODE_OR_IDLE = "decode_or_idle"
|
||||
TARGET_VERIFY = "target_verify"
|
||||
@@ -714,7 +747,9 @@ class DeepseekV4HipRadixBackend(
|
||||
# per-token (num_draft*bs -> bs) req-slot map produced by the prefill
|
||||
# expansion above.
|
||||
self._attach_unified_kv_decode_streams(
|
||||
core_attn_metadata, req_pool_indices_repeated
|
||||
core_attn_metadata,
|
||||
req_pool_indices_repeated,
|
||||
num_draft=self.target_verify_num_draft_tokens,
|
||||
)
|
||||
indexer_metadata = (
|
||||
self.init_forward_metadata_indexer(core_attn_metadata)
|
||||
@@ -923,7 +958,9 @@ class DeepseekV4HipRadixBackend(
|
||||
num_draft_tokens * bs,
|
||||
)
|
||||
self._attach_unified_kv_decode_streams(
|
||||
core_attn_metadata, req_pool_indices_repeated
|
||||
core_attn_metadata,
|
||||
req_pool_indices_repeated,
|
||||
num_draft=self.target_verify_num_draft_tokens,
|
||||
)
|
||||
indexer_metadata = (
|
||||
self.init_forward_metadata_indexer(core_attn_metadata)
|
||||
@@ -1469,7 +1506,10 @@ class DeepseekV4HipRadixBackend(
|
||||
self.forward_metadata = current_raw
|
||||
|
||||
def _attach_unified_kv_decode_streams(
|
||||
self, core: DSV4AttnMetadata, state_slot: torch.Tensor
|
||||
self,
|
||||
core: DSV4AttnMetadata,
|
||||
state_slot: torch.Tensor,
|
||||
num_draft: int = 0,
|
||||
) -> None:
|
||||
# state_slot maps each query token to its request slot;
|
||||
# target-verify repeats request slots for the draft tokens.
|
||||
@@ -1521,6 +1561,35 @@ class DeepseekV4HipRadixBackend(
|
||||
ring_stride=pool.unified_swa_ring_size,
|
||||
swa_pages=pool.unified_swa_pages,
|
||||
)
|
||||
core.unified.gasm_indices = None
|
||||
core.unified.gasm_kv_indptr = None
|
||||
core.unified.gasm_qo_indptr = None
|
||||
if (
|
||||
_grouped_asm_enabled()
|
||||
and num_draft > 1
|
||||
and N % num_draft == 0
|
||||
and N // num_draft >= _GROUPED_ASM_MIN_REQS
|
||||
):
|
||||
from sglang.kernels.ops.attention.dsv4.unified_kv_kernels import (
|
||||
grouped_verify_streams,
|
||||
)
|
||||
|
||||
(
|
||||
core.unified.gasm_indices,
|
||||
core.unified.gasm_kv_indptr,
|
||||
core.unified.gasm_qo_indptr,
|
||||
) = grouped_verify_streams.build_grouped_verify_streams(
|
||||
state_slot=state_slot,
|
||||
positions=core.positions_casual,
|
||||
hca_len=hca_len,
|
||||
hca_page_indices=hca_page_indices,
|
||||
win=pool.unified_swa_window,
|
||||
ring_stride=pool.unified_swa_ring_size,
|
||||
swa_pages=pool.unified_swa_pages,
|
||||
num_draft=num_draft,
|
||||
block_q=_GROUPED_ASM_BLOCK_Q,
|
||||
)
|
||||
|
||||
# SWA ring write target, same value for every layer this forward.
|
||||
req_slot = state_slot.to(torch.int64)
|
||||
core.unified.swa_loc = (
|
||||
@@ -1691,6 +1760,25 @@ class DeepseekV4HipRadixBackend(
|
||||
"the v4 nm asm kernel hardcodes 1/sqrt(512), this backend is "
|
||||
f"at {self.softmax_scale}"
|
||||
)
|
||||
gasm = (
|
||||
unified_metadata.gasm_indices
|
||||
if compress_ratio == 128 and q.shape[1] == _GROUPED_ASM_GQA
|
||||
else None
|
||||
)
|
||||
if gasm is not None:
|
||||
return runtime.decode_fp8_2buff(
|
||||
q=q,
|
||||
q_rope=q_rope,
|
||||
unified_kv=unified,
|
||||
unified_kv_rope=pool.get_unified_kv_rope(layer_id),
|
||||
kv_indices=gasm,
|
||||
kv_indptr=unified_metadata.gasm_kv_indptr,
|
||||
qo_indptr=unified_metadata.gasm_qo_indptr,
|
||||
attn_sink=attn_sink,
|
||||
v_head_dim=layer.v_head_dim,
|
||||
max_seqlen_q=_GROUPED_ASM_BLOCK_Q,
|
||||
compress_ratio=compress_ratio,
|
||||
)
|
||||
return runtime.decode_fp8_2buff(
|
||||
q=q,
|
||||
q_rope=q_rope,
|
||||
|
||||
Reference in New Issue
Block a user