[AMD][V4.1][*/N] Route low-ratio indexer and candidate-block top-k through top-k v2 (#41947)
This commit is contained in:
@@ -10,10 +10,31 @@ import torch.nn.functional as F
|
||||
import triton
|
||||
import triton.language as tl
|
||||
|
||||
from sglang.kernels.ops.attention.dsv4 import topk_transform_paged
|
||||
from sglang.kernels.ops.attention.dsv4 import plan_topk_v2, topk_transform_paged_v2
|
||||
|
||||
# the k the AOT fast_topk op (topk_hip.hip) is instantiated for; any other k takes torch.topk
|
||||
_AOT_FAST_TOPK_K = 2048
|
||||
# the k the block top-k kernel path is taken for; any other k takes torch.topk
|
||||
_BLOCK_TOPK_K = 2048
|
||||
|
||||
|
||||
def topk_transform_paged_hip(
|
||||
scores: torch.Tensor,
|
||||
seq_lens: torch.Tensor,
|
||||
page_table: Optional[torch.Tensor],
|
||||
page_indices: torch.Tensor,
|
||||
page_size: int,
|
||||
raw_indices: Optional[torch.Tensor],
|
||||
) -> None:
|
||||
"""topk_transform_paged (rows unordered) on the top-k v2 kernel."""
|
||||
assert scores.stride(0) % 4 == 0, "top-k v2 needs 16-byte aligned score rows"
|
||||
topk_transform_paged_v2(
|
||||
scores,
|
||||
seq_lens,
|
||||
page_table,
|
||||
page_indices,
|
||||
page_size,
|
||||
plan_topk_v2(seq_lens),
|
||||
raw_indices,
|
||||
)
|
||||
|
||||
|
||||
def topk_transform_paged_sorted(
|
||||
@@ -117,7 +138,12 @@ def candidate_block_scores(
|
||||
assert block_size & (block_size - 1) == 0, f"{block_size = } must be a power of 2"
|
||||
rows, width = logits.shape
|
||||
num_blocks = triton.cdiv(width, block_size)
|
||||
scores = torch.empty((rows, num_blocks), dtype=torch.float32, device=logits.device)
|
||||
# rows padded to 16 bytes: the top-k v2 kernel only takes 16-byte aligned rows
|
||||
scores = torch.empty(
|
||||
(rows, triton.cdiv(num_blocks, 4) * 4),
|
||||
dtype=torch.float32,
|
||||
device=logits.device,
|
||||
)[:, :num_blocks]
|
||||
grid = (rows, triton.cdiv(num_blocks, _LEVEL_ONE_BLOCKS_PER_PROGRAM))
|
||||
_candidate_block_scores_kernel[grid](
|
||||
logits,
|
||||
@@ -280,9 +306,9 @@ def select_candidate_blocks_hip(
|
||||
rows, width = logits.shape
|
||||
device = logits.device
|
||||
num_blocks = triton.cdiv(width, block_size)
|
||||
use_aot = topk_blocks == _AOT_FAST_TOPK_K
|
||||
use_kernel = topk_blocks == _BLOCK_TOPK_K
|
||||
scores = candidate_block_scores(
|
||||
logits, seq_lens, block_size=block_size, fill_tail=not use_aot
|
||||
logits, seq_lens, block_size=block_size, fill_tail=not use_kernel
|
||||
)
|
||||
block_lens = torch.empty(rows, dtype=torch.int32, device=device)
|
||||
compact_lens = torch.empty(rows, dtype=torch.int32, device=device)
|
||||
@@ -295,10 +321,10 @@ def select_candidate_blocks_hip(
|
||||
BLOCK_SIZE=block_size,
|
||||
BLOCK=1024,
|
||||
)
|
||||
if use_aot:
|
||||
if use_kernel:
|
||||
# exact top-k over the first ceil(len / block_size) block scores of each row, -1 padded
|
||||
ids = torch.empty((rows, topk_blocks), dtype=torch.int32, device=device)
|
||||
torch.ops.sgl_kernel.fast_topk(scores, ids, block_lens, None)
|
||||
topk_transform_paged_hip(scores, block_lens, None, ids, 1, None)
|
||||
else:
|
||||
picked = scores.topk(min(topk_blocks, num_blocks), dim=-1)
|
||||
ids = picked.indices.to(torch.int32).masked_fill(
|
||||
@@ -372,7 +398,7 @@ def topk_within_candidate_blocks_hip(
|
||||
)
|
||||
assert compact.shape[1] <= candidates.compact_page_size
|
||||
compact_pos = torch.empty((rows, topk), dtype=torch.int32, device=logits.device)
|
||||
topk_transform_paged(
|
||||
topk_transform_paged_hip(
|
||||
compact,
|
||||
candidates.compact_lens,
|
||||
candidates.compact_page_table,
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
"""DeepSeek V4.1 low-ratio (1 / 2) indexer on ROCm: FlyDSL fp4 paged MQA logits over the split
|
||||
payload / scale index-K pools, then the AOT top-k transform -- the DeepGEMM path's contract."""
|
||||
payload / scale index-K pools, then the top-k v2 transform -- the DeepGEMM path's contract."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -15,7 +15,7 @@ from sglang.kernels.ops.attention.dsv4.candidate_blocks_hip import (
|
||||
cat_candidate_blocks,
|
||||
select_candidate_blocks_hip,
|
||||
slice_candidate_blocks,
|
||||
topk_transform_paged_sorted,
|
||||
topk_transform_paged_hip,
|
||||
topk_within_candidate_blocks_hip,
|
||||
)
|
||||
from sglang.kernels.ops.attention.dsv4.fp4_indexer_hip import (
|
||||
@@ -292,7 +292,6 @@ def low_ratio_index_topk_hip_decode(
|
||||
page_size=indexer_metadata.compressed_page_size,
|
||||
page_indices=core.sparse_page_indices(ratio),
|
||||
raw_indices=core.sparse_raw_indices(ratio),
|
||||
sort_output=True,
|
||||
)
|
||||
return
|
||||
if two_level and indexer.is_candidate_source:
|
||||
@@ -302,7 +301,7 @@ def low_ratio_index_topk_hip_decode(
|
||||
topk_blocks=indexer.candidate_topk_blocks,
|
||||
block_size=indexer.candidate_block_size,
|
||||
)
|
||||
topk_transform_paged_sorted(
|
||||
topk_transform_paged_hip(
|
||||
logits,
|
||||
indexer_metadata.compressed_seq_lens,
|
||||
indexer_metadata.page_table,
|
||||
@@ -547,7 +546,7 @@ def _select_topk_extend_hip(
|
||||
rows_page = page_indices[rows]
|
||||
rows_raw = raw_indices[rows] if raw_indices is not None else None
|
||||
if consume[b] is None:
|
||||
topk_transform_paged_sorted(
|
||||
topk_transform_paged_hip(
|
||||
logits[rows],
|
||||
compress_lens[rows].contiguous(),
|
||||
page_table[rows],
|
||||
@@ -564,10 +563,9 @@ def _select_topk_extend_hip(
|
||||
page_size=page_size,
|
||||
page_indices=rows_page,
|
||||
raw_indices=rows_raw,
|
||||
sort_output=True,
|
||||
)
|
||||
return
|
||||
topk_transform_paged_sorted(
|
||||
topk_transform_paged_hip(
|
||||
logits,
|
||||
compress_lens.contiguous(),
|
||||
page_table,
|
||||
|
||||
Reference in New Issue
Block a user