[AMD] MiniMax-M3 indexer CP: packed scoring for EAGLE verify rows (#42614)

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Kevin Mi
2026-10-09 15:05:50 -07:00
committed by GitHub
co-authored by Claude Opus 5.5
parent fbf8d730fd
commit 6d8ddaf1d7
5 changed files with 210 additions and 6 deletions
@@ -9,6 +9,9 @@ import torch
import triton
import triton.language as tl
# Rows per scoring tile; unrelated to the selector's top-k, which is also 16.
TILE_ROWS = 16
@triton.jit
def _score_shard(
@@ -92,6 +95,102 @@ def _score_shard(
)
@triton.jit
def _score_shard_packed(
Q,
K,
ReqToToken,
Slots,
Lengths,
Scores,
BATCH: tl.constexpr,
WORLD: tl.constexpr,
RANK: tl.constexpr,
PACK: tl.constexpr,
TILE: tl.constexpr,
MAX_BLOCKS: tl.constexpr,
LOCAL_BLOCKS: tl.constexpr,
Q_HEAD_STRIDE: tl.constexpr,
Q_ROW_STRIDE: tl.constexpr,
K_SLOT_STRIDE: tl.constexpr,
K_DIM_STRIDE: tl.constexpr,
TABLE_STRIDE: tl.constexpr,
TABLE_ROWS: tl.constexpr,
CACHE_SLOTS: tl.constexpr,
BLOCKS_PER_CHUNK: tl.constexpr,
INIT_BLOCKS: tl.constexpr,
LOCAL_KEEP: tl.constexpr,
sm_scale,
k_scale,
):
# tile row i is (draft row i // WORLD, head i % WORLD): one K read per request
group, chunk = tl.program_id(0), tl.program_id(1)
tile = tl.arange(0, TILE)
head, sub = tile % WORLD, tile // WORLD
in_tile = tile < PACK * WORLD
rows = group * PACK + sub
dims = tl.arange(0, 128)
positions = tl.arange(0, 128)
lengths = tl.load(Lengths + rows, mask=in_tile, other=0)
group_len = tl.max(lengths, 0)
# caller contract: a group's PACK rows share one request, so one slot addresses K
request = tl.load(Slots + group * PACK).to(tl.int64)
# Match the native scorer's request/slot addressing, including padding.
request = (request + CACHE_SLOTS) % CACHE_SLOTS
active = (lengths > 0) & (request >= 0) & (request < TABLE_ROWS) & in_tile
num_blocks = tl.cdiv(lengths, 128)
local_start = tl.maximum(0, num_blocks - LOCAL_KEEP)
q = tl.load(
Q
+ head[:, None] * Q_HEAD_STRIDE
+ rows[:, None] * Q_ROW_STRIDE
+ dims[None, :],
mask=in_tile[:, None],
other=0,
)
start = chunk * BLOCKS_PER_CHUNK
end = tl.minimum(start + BLOCKS_PER_CHUNK, LOCAL_BLOCKS)
for local in range(start, end):
block = local * WORLD + RANK
valid = active & (block < MAX_BLOCKS) & (block < num_blocks)
# At <=top-k blocks the native selector emits every block without
# reading scores. In particular, graph padding need not read K.
score = tl.full((TILE,), 0.0, tl.float32)
# one branch per group: its longest row decides whether K is read;
# a padding slot past the table must not index ReqToToken
group_blocks = tl.cdiv(group_len, 128)
if (request < TABLE_ROWS) & (block < group_blocks) & (group_blocks > 16):
pos = block * 128 + positions
token_valid = pos < group_len
slots = tl.load(
ReqToToken + request * TABLE_STRIDE + pos,
mask=token_valid,
other=0,
).to(tl.int64)
slots = (slots + CACHE_SLOTS) % CACHE_SLOTS
k = tl.load(
K + dims[:, None] * K_DIM_STRIDE + slots[None, :] * K_SLOT_STRIDE,
mask=token_valid[None, :],
other=0.0,
).to(q.dtype)
# same dot orientation and scale order as _score_shard, so scores match exactly
dot = tl.dot(q, k) * (sm_scale * 1.4426950409 * k_scale)
dot = tl.where(pos[None, :] < lengths[:, None], dot, float("-inf"))
score = tl.max(dot, 1)
score = tl.where(
block >= local_start,
1e29,
tl.where(block < INIT_BLOCKS, 1e30, score),
)
score = tl.where(valid, score, float("-inf"))
# Every slot is written on every replay, including empty shards.
tl.store(
Scores + (head * BATCH + rows) * LOCAL_BLOCKS + local,
score,
mask=in_tile,
)
@triton.jit
def _pack(score, block, valid):
# Same fp32 total order as ROCm TopKTrait::pack_score_id. 16 ID bits
@@ -190,14 +289,56 @@ def score_local_blocks(
local_blocks,
sm_scale,
k_scale,
packed_queries=1,
):
"""Score [world,batch,128] queries through SGLang's token-slot mapping."""
"""Score [world,batch,128] queries through SGLang's token-slot mapping.
With packed_queries > 1, each group of that many consecutive rows belongs to one request
and reads each K block once for the whole group.
"""
world, batch, _ = gathered_q.shape
blocks = triton.cdiv(max_seqlen, 128)
local = triton.cdiv(blocks, world)
scores = torch.empty(
(world, batch, local), dtype=torch.float32, device=gathered_q.device
)
if (
packed_queries > 1
and batch % packed_queries == 0
and packed_queries * world <= TILE_ROWS
):
groups = batch // packed_queries
chunks = min(local, max(1, min(256, 4096 // max(groups, 1))))
_score_shard_packed[(groups, chunks)](
gathered_q,
k_cache,
req_to_token,
slot_ids,
seq_lens,
scores,
BATCH=batch,
WORLD=world,
RANK=rank,
PACK=packed_queries,
TILE=TILE_ROWS,
MAX_BLOCKS=blocks,
LOCAL_BLOCKS=local,
Q_HEAD_STRIDE=gathered_q.stride(0),
Q_ROW_STRIDE=gathered_q.stride(1),
K_SLOT_STRIDE=k_cache.stride(0),
K_DIM_STRIDE=k_cache.stride(2),
TABLE_STRIDE=req_to_token.stride(0),
TABLE_ROWS=req_to_token.shape[0],
CACHE_SLOTS=k_cache.shape[0],
BLOCKS_PER_CHUNK=triton.cdiv(local, chunks),
INIT_BLOCKS=init_blocks,
LOCAL_KEEP=local_blocks,
sm_scale=sm_scale,
k_scale=k_scale,
num_warps=4,
num_stages=1,
)
return scores
chunks = min(local, max(1, min(256, 4096 // max(batch, 1))))
if batch:
_score_shard[(batch, chunks)](
@@ -1843,6 +1843,12 @@ class MiniMaxSparseAttnBackend(AttentionBackend):
topk_out=_topk_buf if _want_topk else None,
hisparse_swap_in_fn=hisparse_swap_in_fn,
indexer_cp=self.indexer_cp,
# the verify funnel lays out each request's draft rows consecutively
packed_queries=(
self.speculative_num_draft_tokens
if forward_batch.forward_mode.is_target_verify()
else 1
),
)
return (
None if idx_o is None else idx_o.reshape(q.shape[0], -1).contiguous(),
@@ -162,6 +162,7 @@ class MiniMaxIndexerCP:
sm_scale=None,
q_scale=None,
k_scale=None,
packed_queries=1,
):
from sglang.kernels.ops.attention.minimax_sparse.decode.indexer_cp import (
merge_candidates,
@@ -189,6 +190,7 @@ class MiniMaxIndexerCP:
local_blocks,
sm_scale,
1.0 if k_scale is None else k_scale,
packed_queries=packed_queries,
)
keys = select_local_candidates(scores, seq_lens, self.rank, max_seqlen)
received = torch.empty((4, 4, batch, 16), dtype=torch.int64, device=q.device)
@@ -327,6 +327,7 @@ def minimax_sparse_decode(
topk_out: Optional[torch.Tensor] = None,
hisparse_swap_in_fn: Optional[Callable] = None,
indexer_cp=None,
packed_queries: int = 1, # consecutive rows sharing one request (EAGLE verify)
) -> Tuple[torch.Tensor, torch.Tensor]:
# Index top-k sharing for DECODE. A group's source layer passes ``topk_out``
# (a persistent buffer) and publishes its reduced top-k there; the group's
@@ -362,6 +363,7 @@ def minimax_sparse_decode(
idx_sm_scale,
idx_q_scale,
idx_k_scale,
packed_queries=packed_queries,
)
_skip_reduce = True
else: