[AMD][V4.1][*/N] Route low-ratio indexer and candidate-block top-k through top-k v2 (#41947)

This commit is contained in:
Xinyi Song
2026-10-01 01:56:31 -07:00
committed by GitHub
parent 5fab1f0f51
commit 0224fcbd1c
2 changed files with 40 additions and 16 deletions
@@ -10,10 +10,31 @@ import torch.nn.functional as F
import triton
import triton.language as tl
from sglang.kernels.ops.attention.dsv4 import topk_transform_paged
from sglang.kernels.ops.attention.dsv4 import plan_topk_v2, topk_transform_paged_v2
# the k the AOT fast_topk op (topk_hip.hip) is instantiated for; any other k takes torch.topk
_AOT_FAST_TOPK_K = 2048
# the k the block top-k kernel path is taken for; any other k takes torch.topk
_BLOCK_TOPK_K = 2048
def topk_transform_paged_hip(
scores: torch.Tensor,
seq_lens: torch.Tensor,
page_table: Optional[torch.Tensor],
page_indices: torch.Tensor,
page_size: int,
raw_indices: Optional[torch.Tensor],
) -> None:
"""topk_transform_paged (rows unordered) on the top-k v2 kernel."""
assert scores.stride(0) % 4 == 0, "top-k v2 needs 16-byte aligned score rows"
topk_transform_paged_v2(
scores,
seq_lens,
page_table,
page_indices,
page_size,
plan_topk_v2(seq_lens),
raw_indices,
)
def topk_transform_paged_sorted(
@@ -117,7 +138,12 @@ def candidate_block_scores(
assert block_size & (block_size - 1) == 0, f"{block_size = } must be a power of 2"
rows, width = logits.shape
num_blocks = triton.cdiv(width, block_size)
scores = torch.empty((rows, num_blocks), dtype=torch.float32, device=logits.device)
# rows padded to 16 bytes: the top-k v2 kernel only takes 16-byte aligned rows
scores = torch.empty(
(rows, triton.cdiv(num_blocks, 4) * 4),
dtype=torch.float32,
device=logits.device,
)[:, :num_blocks]
grid = (rows, triton.cdiv(num_blocks, _LEVEL_ONE_BLOCKS_PER_PROGRAM))
_candidate_block_scores_kernel[grid](
logits,
@@ -280,9 +306,9 @@ def select_candidate_blocks_hip(
rows, width = logits.shape
device = logits.device
num_blocks = triton.cdiv(width, block_size)
use_aot = topk_blocks == _AOT_FAST_TOPK_K
use_kernel = topk_blocks == _BLOCK_TOPK_K
scores = candidate_block_scores(
logits, seq_lens, block_size=block_size, fill_tail=not use_aot
logits, seq_lens, block_size=block_size, fill_tail=not use_kernel
)
block_lens = torch.empty(rows, dtype=torch.int32, device=device)
compact_lens = torch.empty(rows, dtype=torch.int32, device=device)
@@ -295,10 +321,10 @@ def select_candidate_blocks_hip(
BLOCK_SIZE=block_size,
BLOCK=1024,
)
if use_aot:
if use_kernel:
# exact top-k over the first ceil(len / block_size) block scores of each row, -1 padded
ids = torch.empty((rows, topk_blocks), dtype=torch.int32, device=device)
torch.ops.sgl_kernel.fast_topk(scores, ids, block_lens, None)
topk_transform_paged_hip(scores, block_lens, None, ids, 1, None)
else:
picked = scores.topk(min(topk_blocks, num_blocks), dim=-1)
ids = picked.indices.to(torch.int32).masked_fill(
@@ -372,7 +398,7 @@ def topk_within_candidate_blocks_hip(
)
assert compact.shape[1] <= candidates.compact_page_size
compact_pos = torch.empty((rows, topk), dtype=torch.int32, device=logits.device)
topk_transform_paged(
topk_transform_paged_hip(
compact,
candidates.compact_lens,
candidates.compact_page_table,
@@ -1,5 +1,5 @@
"""DeepSeek V4.1 low-ratio (1 / 2) indexer on ROCm: FlyDSL fp4 paged MQA logits over the split
payload / scale index-K pools, then the AOT top-k transform -- the DeepGEMM path's contract."""
payload / scale index-K pools, then the top-k v2 transform -- the DeepGEMM path's contract."""
from __future__ import annotations
@@ -15,7 +15,7 @@ from sglang.kernels.ops.attention.dsv4.candidate_blocks_hip import (
cat_candidate_blocks,
select_candidate_blocks_hip,
slice_candidate_blocks,
topk_transform_paged_sorted,
topk_transform_paged_hip,
topk_within_candidate_blocks_hip,
)
from sglang.kernels.ops.attention.dsv4.fp4_indexer_hip import (
@@ -292,7 +292,6 @@ def low_ratio_index_topk_hip_decode(
page_size=indexer_metadata.compressed_page_size,
page_indices=core.sparse_page_indices(ratio),
raw_indices=core.sparse_raw_indices(ratio),
sort_output=True,
)
return
if two_level and indexer.is_candidate_source:
@@ -302,7 +301,7 @@ def low_ratio_index_topk_hip_decode(
topk_blocks=indexer.candidate_topk_blocks,
block_size=indexer.candidate_block_size,
)
topk_transform_paged_sorted(
topk_transform_paged_hip(
logits,
indexer_metadata.compressed_seq_lens,
indexer_metadata.page_table,
@@ -547,7 +546,7 @@ def _select_topk_extend_hip(
rows_page = page_indices[rows]
rows_raw = raw_indices[rows] if raw_indices is not None else None
if consume[b] is None:
topk_transform_paged_sorted(
topk_transform_paged_hip(
logits[rows],
compress_lens[rows].contiguous(),
page_table[rows],
@@ -564,10 +563,9 @@ def _select_topk_extend_hip(
page_size=page_size,
page_indices=rows_page,
raw_indices=rows_raw,
sort_output=True,
)
return
topk_transform_paged_sorted(
topk_transform_paged_hip(
logits,
compress_lens.contiguous(),
page_table,