[AMD] [GLM-5.3-Flash Day 0] Enable the k-pool DSA indexer on gfx950 (#39341)

Co-authored-by: Yikai Zhang <ykzhang12@gmail.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
Co-authored-by: Raiden-Makoto <Raiden-Makoto@users.noreply.github.com>
Co-authored-by: Thomas Wang <thomawan@amd.com>
Co-authored-by: Kevin Mi <45493463+kevin-mii@users.noreply.github.com>
Co-authored-by: Kevin Mi <mikevin920@yahoo.com>
This commit is contained in:
Jacob0226
2026-09-22 13:14:30 -07:00
committed by GitHub
co-authored by Yikai Zhang Claude Opus 5 Raiden-Makoto Thomas Wang Kevin Mi Kevin Mi
parent 077c319914
commit 3afdde5f05
4 changed files with 322 additions and 32 deletions
@@ -14,7 +14,18 @@ from sglang.srt.layers.attention.dsa.dsa_indexer import (
rotate_activation,
)
from sglang.srt.layers.attention.dsa.dsa_topk_backend import TopkTransformMethod
from sglang.srt.layers.attention.dsa.utils import dsa_use_prefill_cp
from sglang.srt.layers.attention.dsa.paged_mqa_logits_backend import (
DSAPagedMQALogitsBackend,
)
from sglang.srt.layers.attention.dsa.utils import (
aiter_can_use_preshuffle_paged_mqa,
dsa_use_prefill_cp,
)
from sglang.srt.layers.attention.mqa_logits_utils import (
MQA_LOGITS_BYTES_PER_ELEM,
MQA_LOGITS_MAX_BYTES_ROCM,
mqa_logits_rows_per_chunk,
)
from sglang.srt.layers.layernorm import LayerNorm
from sglang.srt.layers.utils import MultiPlatformOp
from sglang.srt.utils import add_prefix, ceil_align, is_cuda, is_hip, is_npu
@@ -46,7 +57,7 @@ from sglang.srt.model_executor.runner import get_is_capture_mode
from sglang.srt.model_executor.runner_backend_utils.breakable_cuda_graph import (
is_in_breakable_cuda_graph,
)
from sglang.srt.runtime_context import get_device
from sglang.srt.runtime_context import get_device, get_exec
if TYPE_CHECKING:
from sglang.srt.mem_cache.memory_pool import DSATokenToKVPool
@@ -110,6 +121,11 @@ class IndexerKPool(MultiPlatformOp):
f"index_kpool ({self.index_kpool}) must divide page_size (64)"
)
# resolve() pins ROCm to AITER and lets --dsa-paged-mqa-logits-backend reach this path
self.paged_mqa_logits_backend = DSAPagedMQALogitsBackend.resolve(
get_exec().kernel.dsa_paged_mqa_logits_backend
)
self.index_kpool_compress_ape = nn.Parameter(
torch.zeros(self.index_kpool, self.head_dim, dtype=torch.float32)
)
@@ -808,12 +824,66 @@ class IndexerKPool(MultiPlatformOp):
return None, attn_metadata.topk_indices_offset, None
return None, None, None
@staticmethod
def _fp8_mqa_logits(
q_fp8: torch.Tensor,
k_fp8: torch.Tensor,
k_scale: torch.Tensor,
weights: torch.Tensor,
starts: torch.Tensor,
ends: torch.Tensor,
*,
clean_logits: bool,
) -> torch.Tensor:
"""Ragged MQA logits: DeepGEMM on CUDA, AITER's Triton kernel on ROCm."""
if is_hip():
from aiter.ops.triton.fp8_mqa_logits import fp8_mqa_logits
# above 2 GiB of fp32 logits the kernel miscompiles and abort()s the
# process (ROCm/aiter#5114); rows are independent, so block the queries
num_q, num_k = q_fp8.shape[0], k_fp8.shape[0]
rows_per_call = mqa_logits_rows_per_chunk(
num_rows=num_q,
row_bytes=num_k * MQA_LOGITS_BYTES_PER_ELEM,
budget_bytes=MQA_LOGITS_MAX_BYTES_ROCM,
)
if rows_per_call is None:
return fp8_mqa_logits(
q_fp8,
k_fp8,
k_scale,
weights,
starts,
ends,
clean_logits=clean_logits,
)
logits = torch.empty(
(num_q, num_k), dtype=torch.float32, device=q_fp8.device
)
for i in range(0, num_q, rows_per_call):
rows = slice(i, i + rows_per_call)
logits[rows] = fp8_mqa_logits(
q_fp8[rows],
k_fp8,
k_scale,
weights[rows],
starts[rows],
ends[rows],
clean_logits=clean_logits,
)
return logits
return deep_gemm.fp8_mqa_logits(
q_fp8, (k_fp8, k_scale), weights, starts, ends, clean_logits=clean_logits
)
@staticmethod
def _should_use_tilelang_paged_mqa_logits(q_fp8: torch.Tensor) -> bool:
# q_fp8 is [tokens, heads, dim], without the next_n dim
if not is_cuda():
return False
arch_major, _ = torch.cuda.get_device_capability(q_fp8.device)
num_heads = q_fp8.shape[2]
num_heads = q_fp8.shape[1]
return arch_major == 9 and num_heads not in (32, 64)
def _get_topk_paged(
@@ -850,7 +920,7 @@ class IndexerKPool(MultiPlatformOp):
if n_real < num_q_padded:
q_fp8 = q_fp8[:n_real]
weights = weights[:n_real]
q_fp8 = q_fp8.unsqueeze(1) # the next_n dim is 1 now
# aiter_paged_mqa_logits adds the next_n dim itself; the other two want it already there
assert len(kv_cache_fp8.shape) == 2
block_kv = 64
num_heads_kv = 1
@@ -860,7 +930,11 @@ class IndexerKPool(MultiPlatformOp):
)
assert len(weights.shape) == 3
weights = weights.squeeze(2)
use_tilelang_paged_mqa = self._should_use_tilelang_paged_mqa_logits(q_fp8)
use_aiter_paged_mqa = self.paged_mqa_logits_backend.is_aiter()
use_tilelang_paged_mqa = (
not use_aiter_paged_mqa
and self._should_use_tilelang_paged_mqa_logits(q_fp8)
)
pool_seqlens, pool_context_lens, pool_block_tables, pool_schedule_metadata = (
self._get_kpool_decode_metadata(
@@ -868,17 +942,40 @@ class IndexerKPool(MultiPlatformOp):
block_tables,
seqlens_32,
blocksize,
build_schedule_metadata=not use_tilelang_paged_mqa,
build_schedule_metadata=not (
use_aiter_paged_mqa or use_tilelang_paged_mqa
),
)
)
pool_max_seq_len = pool_block_tables.shape[1] * blocksize
if use_tilelang_paged_mqa:
if use_aiter_paged_mqa:
# kpool_fp8_index writes in AITER's tile order, so a row-major reader would misread the cache
if not aiter_can_use_preshuffle_paged_mqa():
raise RuntimeError(
"The ROCm k-pool indexer needs AITER's preshuffle paged-MQA "
"kernel. It requires Triton >= 3.5.0, or "
"AITER_ENABLE_AOT_GLUON_PA_MQA_LOGITS=1 with the AOT gluon "
"artifacts present in the image."
)
from sglang.kernels.ops.attention.dsa import aiter_paged_mqa_logits
logits = aiter_paged_mqa_logits(
q_fp8,
kv_cache_fp8,
weights,
pool_seqlens,
pool_block_tables,
pool_max_seq_len,
preshuffle=True,
kv_block_size=block_kv,
)
elif use_tilelang_paged_mqa:
from sglang.kernels.ops.attention.dsa.tilelang_kernel import (
tilelang_fp8_paged_mqa_logits,
)
logits = tilelang_fp8_paged_mqa_logits(
q_fp8,
q_fp8.unsqueeze(1),
kv_cache_fp8,
weights,
pool_seqlens,
@@ -889,7 +986,7 @@ class IndexerKPool(MultiPlatformOp):
)
else:
logits = deep_gemm.fp8_paged_mqa_logits(
q_fp8,
q_fp8.unsqueeze(1),
kv_cache_fp8,
weights,
pool_context_lens,
@@ -967,9 +1064,10 @@ class IndexerKPool(MultiPlatformOp):
scale_out=k_scale,
)
k_fp8 = k_u8.view(torch.float8_e4m3fn)
logits = deep_gemm.fp8_mqa_logits(
logits = self._fp8_mqa_logits(
q_fp8[:n_real].contiguous(),
(k_fp8.contiguous(), k_scale.contiguous()),
k_fp8.contiguous(),
k_scale.contiguous(),
weights[:n_real].contiguous(),
ks_per_q,
ke_per_q,
@@ -1199,9 +1297,10 @@ class IndexerKPool(MultiPlatformOp):
and zero_starts_by_batch[i] is not None
else torch.zeros((q_len,), dtype=torch.int32, device=q_fp8.device)
)
local_logits = deep_gemm.fp8_mqa_logits(
local_logits = self._fp8_mqa_logits(
q_fp8[q_slice].contiguous(),
(k_fp8.contiguous(), k_scale.contiguous()),
k_fp8.contiguous(),
k_scale.contiguous(),
weights[q_slice].contiguous(),
row_starts,
local_pool_lens,
@@ -1575,7 +1674,7 @@ class IndexerKPool(MultiPlatformOp):
if not return_indices:
return None
if is_cuda():
if is_cuda() or is_hip():
if (
forward_batch.forward_mode.is_decode_or_idle()
or forward_batch.forward_mode.is_target_verify()
@@ -1603,5 +1702,7 @@ class IndexerKPool(MultiPlatformOp):
kpool_extend_cache=kpool_extend_cache,
)
else:
raise NotImplementedError("kpool indexer is only supported on CUDA")
raise NotImplementedError(
"kpool indexer is only supported on CUDA and ROCm"
)
return topk_result
@@ -4,11 +4,50 @@ import torch
import triton
import triton.language as tl
from sglang.srt.layers.attention.dsa.utils import (
INDEXER_K_CACHE_PRESHUFFLE_TILE,
aiter_can_use_preshuffle_paged_mqa,
)
BLOCK_SIZE_K = 64
INDEX_HEAD_DIM = 128
KPOOL_SCORE_DTYPES = (torch.float16, torch.bfloat16, torch.float32)
def _preshuffle_tile() -> int:
"""Tile width for the index-K cache, or 0 to keep the row-major layout."""
return (
INDEXER_K_CACHE_PRESHUFFLE_TILE if aiter_can_use_preshuffle_paged_mqa() else 0
)
@triton.jit
def _kpool_cache_k_offsets(
page,
slot,
cols,
BUF_NUMEL_PER_PAGE: tl.constexpr,
HEAD_DIM: tl.constexpr,
PRESHUFFLE_TILE: tl.constexpr,
):
"""Offsets of one pooled token's index-K row; must match `_set_k_and_s_triton_kernel`,
which lays out the non-pooled cache for the same aiter kernel."""
if PRESHUFFLE_TILE:
tile = PRESHUFFLE_TILE
token_tile_id = slot // tile
token_in_tile = slot % tile
col_tile_id = cols // tile
col_in_tile = cols % tile
return (
page * BUF_NUMEL_PER_PAGE
+ token_tile_id * (tile * HEAD_DIM)
+ col_tile_id * (tile * tile)
+ token_in_tile * tile
+ col_in_tile
)
return page * BUF_NUMEL_PER_PAGE + slot * HEAD_DIM + cols
def kpool_max_closed_pools(num_draft_tokens: int, pool_size: int) -> int:
return (num_draft_tokens + pool_size - 1) // pool_size
@@ -59,6 +98,7 @@ def gather_index_k_scale_prefix_into(
scale_out,
PAGE_SIZE=pool.page_size,
BUF_NUMEL_PER_PAGE=buf.shape[1],
PRESHUFFLE_TILE=_preshuffle_tile(),
HEAD_DIM=INDEX_HEAD_DIM,
S_OFFSET_NBYTES_IN_PAGE=pool.page_size * INDEX_HEAD_DIM,
BLOCK_D=triton.next_power_of_2(INDEX_HEAD_DIM),
@@ -76,6 +116,7 @@ def _gather_index_k_scale_prefix_into_kernel(
BUF_NUMEL_PER_PAGE: tl.constexpr,
HEAD_DIM: tl.constexpr,
S_OFFSET_NBYTES_IN_PAGE: tl.constexpr,
PRESHUFFLE_TILE: tl.constexpr,
BLOCK_D: tl.constexpr,
):
token_id = tl.program_id(0)
@@ -85,7 +126,14 @@ def _gather_index_k_scale_prefix_into_kernel(
offs = tl.arange(0, BLOCK_D)
mask = offs < HEAD_DIM
src_k_offsets = page * BUF_NUMEL_PER_PAGE + token_offset_in_page * HEAD_DIM + offs
src_k_offsets = _kpool_cache_k_offsets(
page,
token_offset_in_page,
offs,
BUF_NUMEL_PER_PAGE,
HEAD_DIM,
PRESHUFFLE_TILE,
)
dst_k_offsets = token_id * HEAD_DIM + offs
k = tl.load(buf_u8_ptr + src_k_offsets, mask=mask)
tl.store(k_out_ptr + dst_k_offsets, k, mask=mask)
@@ -725,6 +773,7 @@ def kpool_softmax_rotate_write_cache(
ape.stride(0),
PAGE_SIZE=pool.page_size,
BUF_NUMEL_PER_PAGE=buf.shape[1],
PRESHUFFLE_TILE=_preshuffle_tile(),
POOL_SIZE=slot_k.shape[1],
HEAD_DIM=slot_k.shape[2],
S_OFFSET_NBYTES_IN_PAGE=pool.page_size * pool.index_head_dim,
@@ -818,6 +867,7 @@ def kpool_decode_update_and_maybe_write_cache(
REQ_POOL_SIZE=tail_k.shape[0],
PAGE_SIZE=pool.page_size,
BUF_NUMEL_PER_PAGE=buf.shape[1],
PRESHUFFLE_TILE=_preshuffle_tile(),
POOL_SIZE=pool.index_kpool,
TAIL_SIZE=tail_k.shape[1],
HEAD_DIM=tail_k.shape[2],
@@ -872,6 +922,7 @@ def _kpool_softmax_rotate_write_cache_kernel(
POOL_SIZE: tl.constexpr,
HEAD_DIM: tl.constexpr,
S_OFFSET_NBYTES_IN_PAGE: tl.constexpr,
PRESHUFFLE_TILE: tl.constexpr,
ROUND_SCALE: tl.constexpr,
HAS_WRITE_MASK: tl.constexpr,
RETURN_COMPRESSED: tl.constexpr,
@@ -947,10 +998,13 @@ def _kpool_softmax_rotate_write_cache_kernel(
loc = tl.load(loc_ptr + row, mask=do_write, other=0)
loc_page_index = loc // PAGE_SIZE
loc_token_offset_in_page = loc % PAGE_SIZE
out_k_offsets = (
loc_page_index * BUF_NUMEL_PER_PAGE
+ loc_token_offset_in_page * HEAD_DIM
+ offs
out_k_offsets = _kpool_cache_k_offsets(
loc_page_index,
loc_token_offset_in_page,
offs,
BUF_NUMEL_PER_PAGE,
HEAD_DIM,
PRESHUFFLE_TILE,
)
out_s_offset = (
loc_page_index * BUF_NUMEL_PER_PAGE // 4
@@ -1000,6 +1054,7 @@ def _kpool_decode_update_and_maybe_write_cache_kernel(
HEAD_DIM: tl.constexpr,
BLOCK_TABLE_COLS: tl.constexpr,
S_OFFSET_NBYTES_IN_PAGE: tl.constexpr,
PRESHUFFLE_TILE: tl.constexpr,
ROUND_SCALE: tl.constexpr,
BLOCK_D: tl.constexpr,
SLOTS_PER_PAGE: tl.constexpr,
@@ -1112,10 +1167,13 @@ def _kpool_decode_update_and_maybe_write_cache_kernel(
)
loc_page_index = packed_page.to(tl.int64)
loc_token_offset_in_page = pool_id % SLOTS_PER_PAGE
out_k_offsets = (
loc_page_index * BUF_NUMEL_PER_PAGE
+ loc_token_offset_in_page * HEAD_DIM
+ offs
out_k_offsets = _kpool_cache_k_offsets(
loc_page_index,
loc_token_offset_in_page,
offs,
BUF_NUMEL_PER_PAGE,
HEAD_DIM,
PRESHUFFLE_TILE,
)
out_s_offset = (
loc_page_index * BUF_NUMEL_PER_PAGE // 4
@@ -1175,6 +1233,7 @@ def _kpool_assemble_softmax_rotate_write_cache_kernel(
TAIL_SIZE: tl.constexpr,
HEAD_DIM: tl.constexpr,
S_OFFSET_NBYTES_IN_PAGE: tl.constexpr,
PRESHUFFLE_TILE: tl.constexpr,
ROUND_SCALE: tl.constexpr,
HAS_WRITE_MASK: tl.constexpr,
BLOCK_D: tl.constexpr,
@@ -1222,8 +1281,13 @@ def _kpool_assemble_softmax_rotate_write_cache_kernel(
loc = tl.load(loc_ptr + row)
loc_page_index = loc // SLOTS_PER_PAGE
loc_token_offset_in_page = loc % SLOTS_PER_PAGE
out_k_offsets = (
loc_page_index * BUF_NUMEL_PER_PAGE + loc_token_offset_in_page * HEAD_DIM + offs
out_k_offsets = _kpool_cache_k_offsets(
loc_page_index,
loc_token_offset_in_page,
offs,
BUF_NUMEL_PER_PAGE,
HEAD_DIM,
PRESHUFFLE_TILE,
)
out_s_offset = (
loc_page_index * BUF_NUMEL_PER_PAGE // 4
@@ -1290,6 +1354,7 @@ def kpool_assemble_softmax_rotate_write_cache(
tail_k.stride(1),
ape.stride(0),
BUF_NUMEL_PER_PAGE=buf.shape[1],
PRESHUFFLE_TILE=_preshuffle_tile(),
POOL_SIZE=pool_size,
TAIL_SIZE=tail_k.shape[1],
HEAD_DIM=INDEX_HEAD_DIM,
@@ -1537,6 +1602,7 @@ def _kpool_write_tail_and_maybe_compress_kernel(
SLOTS_PER_PAGE: tl.constexpr,
BUF_NUMEL_PER_PAGE: tl.constexpr,
S_OFFSET_NBYTES_IN_PAGE: tl.constexpr,
PRESHUFFLE_TILE: tl.constexpr,
ROUND_SCALE: tl.constexpr,
HAS_EFFECTIVE_N: tl.constexpr,
MAX_CLOSED_POOLS: tl.constexpr,
@@ -1601,10 +1667,13 @@ def _kpool_write_tail_and_maybe_compress_kernel(
loc = tl.load(write_loc_ptr + b * write_loc_stride_0 + p)
loc_page_index = loc // SLOTS_PER_PAGE
loc_token_offset_in_page = loc % SLOTS_PER_PAGE
out_k_offsets = (
loc_page_index * BUF_NUMEL_PER_PAGE
+ loc_token_offset_in_page * HEAD_DIM
+ offs
out_k_offsets = _kpool_cache_k_offsets(
loc_page_index,
loc_token_offset_in_page,
offs,
BUF_NUMEL_PER_PAGE,
HEAD_DIM,
PRESHUFFLE_TILE,
)
out_s_offset = (
loc_page_index * BUF_NUMEL_PER_PAGE // 4
@@ -1694,6 +1763,7 @@ def kpool_write_tail_and_maybe_compress(
BLOCK_D=triton.next_power_of_2(INDEX_HEAD_DIM),
SLOTS_PER_PAGE=slots_per_page,
BUF_NUMEL_PER_PAGE=buf.shape[1],
PRESHUFFLE_TILE=_preshuffle_tile(),
S_OFFSET_NBYTES_IN_PAGE=slots_per_page * pool.index_head_dim,
ROUND_SCALE=round_scale,
HAS_EFFECTIVE_N=effective_n_per_batch is not None,