[AMD] MiniMax-M3 indexer CP: packed scoring for EAGLE verify rows (#42614)
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
fbf8d730fd
commit
6d8ddaf1d7
@@ -9,6 +9,9 @@ import torch
|
||||
import triton
|
||||
import triton.language as tl
|
||||
|
||||
# Rows per scoring tile; unrelated to the selector's top-k, which is also 16.
|
||||
TILE_ROWS = 16
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _score_shard(
|
||||
@@ -92,6 +95,102 @@ def _score_shard(
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _score_shard_packed(
|
||||
Q,
|
||||
K,
|
||||
ReqToToken,
|
||||
Slots,
|
||||
Lengths,
|
||||
Scores,
|
||||
BATCH: tl.constexpr,
|
||||
WORLD: tl.constexpr,
|
||||
RANK: tl.constexpr,
|
||||
PACK: tl.constexpr,
|
||||
TILE: tl.constexpr,
|
||||
MAX_BLOCKS: tl.constexpr,
|
||||
LOCAL_BLOCKS: tl.constexpr,
|
||||
Q_HEAD_STRIDE: tl.constexpr,
|
||||
Q_ROW_STRIDE: tl.constexpr,
|
||||
K_SLOT_STRIDE: tl.constexpr,
|
||||
K_DIM_STRIDE: tl.constexpr,
|
||||
TABLE_STRIDE: tl.constexpr,
|
||||
TABLE_ROWS: tl.constexpr,
|
||||
CACHE_SLOTS: tl.constexpr,
|
||||
BLOCKS_PER_CHUNK: tl.constexpr,
|
||||
INIT_BLOCKS: tl.constexpr,
|
||||
LOCAL_KEEP: tl.constexpr,
|
||||
sm_scale,
|
||||
k_scale,
|
||||
):
|
||||
# tile row i is (draft row i // WORLD, head i % WORLD): one K read per request
|
||||
group, chunk = tl.program_id(0), tl.program_id(1)
|
||||
tile = tl.arange(0, TILE)
|
||||
head, sub = tile % WORLD, tile // WORLD
|
||||
in_tile = tile < PACK * WORLD
|
||||
rows = group * PACK + sub
|
||||
dims = tl.arange(0, 128)
|
||||
positions = tl.arange(0, 128)
|
||||
lengths = tl.load(Lengths + rows, mask=in_tile, other=0)
|
||||
group_len = tl.max(lengths, 0)
|
||||
# caller contract: a group's PACK rows share one request, so one slot addresses K
|
||||
request = tl.load(Slots + group * PACK).to(tl.int64)
|
||||
# Match the native scorer's request/slot addressing, including padding.
|
||||
request = (request + CACHE_SLOTS) % CACHE_SLOTS
|
||||
active = (lengths > 0) & (request >= 0) & (request < TABLE_ROWS) & in_tile
|
||||
num_blocks = tl.cdiv(lengths, 128)
|
||||
local_start = tl.maximum(0, num_blocks - LOCAL_KEEP)
|
||||
q = tl.load(
|
||||
Q
|
||||
+ head[:, None] * Q_HEAD_STRIDE
|
||||
+ rows[:, None] * Q_ROW_STRIDE
|
||||
+ dims[None, :],
|
||||
mask=in_tile[:, None],
|
||||
other=0,
|
||||
)
|
||||
start = chunk * BLOCKS_PER_CHUNK
|
||||
end = tl.minimum(start + BLOCKS_PER_CHUNK, LOCAL_BLOCKS)
|
||||
for local in range(start, end):
|
||||
block = local * WORLD + RANK
|
||||
valid = active & (block < MAX_BLOCKS) & (block < num_blocks)
|
||||
# At <=top-k blocks the native selector emits every block without
|
||||
# reading scores. In particular, graph padding need not read K.
|
||||
score = tl.full((TILE,), 0.0, tl.float32)
|
||||
# one branch per group: its longest row decides whether K is read;
|
||||
# a padding slot past the table must not index ReqToToken
|
||||
group_blocks = tl.cdiv(group_len, 128)
|
||||
if (request < TABLE_ROWS) & (block < group_blocks) & (group_blocks > 16):
|
||||
pos = block * 128 + positions
|
||||
token_valid = pos < group_len
|
||||
slots = tl.load(
|
||||
ReqToToken + request * TABLE_STRIDE + pos,
|
||||
mask=token_valid,
|
||||
other=0,
|
||||
).to(tl.int64)
|
||||
slots = (slots + CACHE_SLOTS) % CACHE_SLOTS
|
||||
k = tl.load(
|
||||
K + dims[:, None] * K_DIM_STRIDE + slots[None, :] * K_SLOT_STRIDE,
|
||||
mask=token_valid[None, :],
|
||||
other=0.0,
|
||||
).to(q.dtype)
|
||||
# same dot orientation and scale order as _score_shard, so scores match exactly
|
||||
dot = tl.dot(q, k) * (sm_scale * 1.4426950409 * k_scale)
|
||||
dot = tl.where(pos[None, :] < lengths[:, None], dot, float("-inf"))
|
||||
score = tl.max(dot, 1)
|
||||
score = tl.where(
|
||||
block >= local_start,
|
||||
1e29,
|
||||
tl.where(block < INIT_BLOCKS, 1e30, score),
|
||||
)
|
||||
score = tl.where(valid, score, float("-inf"))
|
||||
# Every slot is written on every replay, including empty shards.
|
||||
tl.store(
|
||||
Scores + (head * BATCH + rows) * LOCAL_BLOCKS + local,
|
||||
score,
|
||||
mask=in_tile,
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _pack(score, block, valid):
|
||||
# Same fp32 total order as ROCm TopKTrait::pack_score_id. 16 ID bits
|
||||
@@ -190,14 +289,56 @@ def score_local_blocks(
|
||||
local_blocks,
|
||||
sm_scale,
|
||||
k_scale,
|
||||
packed_queries=1,
|
||||
):
|
||||
"""Score [world,batch,128] queries through SGLang's token-slot mapping."""
|
||||
"""Score [world,batch,128] queries through SGLang's token-slot mapping.
|
||||
|
||||
With packed_queries > 1, each group of that many consecutive rows belongs to one request
|
||||
and reads each K block once for the whole group.
|
||||
"""
|
||||
world, batch, _ = gathered_q.shape
|
||||
blocks = triton.cdiv(max_seqlen, 128)
|
||||
local = triton.cdiv(blocks, world)
|
||||
scores = torch.empty(
|
||||
(world, batch, local), dtype=torch.float32, device=gathered_q.device
|
||||
)
|
||||
if (
|
||||
packed_queries > 1
|
||||
and batch % packed_queries == 0
|
||||
and packed_queries * world <= TILE_ROWS
|
||||
):
|
||||
groups = batch // packed_queries
|
||||
chunks = min(local, max(1, min(256, 4096 // max(groups, 1))))
|
||||
_score_shard_packed[(groups, chunks)](
|
||||
gathered_q,
|
||||
k_cache,
|
||||
req_to_token,
|
||||
slot_ids,
|
||||
seq_lens,
|
||||
scores,
|
||||
BATCH=batch,
|
||||
WORLD=world,
|
||||
RANK=rank,
|
||||
PACK=packed_queries,
|
||||
TILE=TILE_ROWS,
|
||||
MAX_BLOCKS=blocks,
|
||||
LOCAL_BLOCKS=local,
|
||||
Q_HEAD_STRIDE=gathered_q.stride(0),
|
||||
Q_ROW_STRIDE=gathered_q.stride(1),
|
||||
K_SLOT_STRIDE=k_cache.stride(0),
|
||||
K_DIM_STRIDE=k_cache.stride(2),
|
||||
TABLE_STRIDE=req_to_token.stride(0),
|
||||
TABLE_ROWS=req_to_token.shape[0],
|
||||
CACHE_SLOTS=k_cache.shape[0],
|
||||
BLOCKS_PER_CHUNK=triton.cdiv(local, chunks),
|
||||
INIT_BLOCKS=init_blocks,
|
||||
LOCAL_KEEP=local_blocks,
|
||||
sm_scale=sm_scale,
|
||||
k_scale=k_scale,
|
||||
num_warps=4,
|
||||
num_stages=1,
|
||||
)
|
||||
return scores
|
||||
chunks = min(local, max(1, min(256, 4096 // max(batch, 1))))
|
||||
if batch:
|
||||
_score_shard[(batch, chunks)](
|
||||
|
||||
@@ -1843,6 +1843,12 @@ class MiniMaxSparseAttnBackend(AttentionBackend):
|
||||
topk_out=_topk_buf if _want_topk else None,
|
||||
hisparse_swap_in_fn=hisparse_swap_in_fn,
|
||||
indexer_cp=self.indexer_cp,
|
||||
# the verify funnel lays out each request's draft rows consecutively
|
||||
packed_queries=(
|
||||
self.speculative_num_draft_tokens
|
||||
if forward_batch.forward_mode.is_target_verify()
|
||||
else 1
|
||||
),
|
||||
)
|
||||
return (
|
||||
None if idx_o is None else idx_o.reshape(q.shape[0], -1).contiguous(),
|
||||
|
||||
@@ -162,6 +162,7 @@ class MiniMaxIndexerCP:
|
||||
sm_scale=None,
|
||||
q_scale=None,
|
||||
k_scale=None,
|
||||
packed_queries=1,
|
||||
):
|
||||
from sglang.kernels.ops.attention.minimax_sparse.decode.indexer_cp import (
|
||||
merge_candidates,
|
||||
@@ -189,6 +190,7 @@ class MiniMaxIndexerCP:
|
||||
local_blocks,
|
||||
sm_scale,
|
||||
1.0 if k_scale is None else k_scale,
|
||||
packed_queries=packed_queries,
|
||||
)
|
||||
keys = select_local_candidates(scores, seq_lens, self.rank, max_seqlen)
|
||||
received = torch.empty((4, 4, batch, 16), dtype=torch.int64, device=q.device)
|
||||
|
||||
@@ -327,6 +327,7 @@ def minimax_sparse_decode(
|
||||
topk_out: Optional[torch.Tensor] = None,
|
||||
hisparse_swap_in_fn: Optional[Callable] = None,
|
||||
indexer_cp=None,
|
||||
packed_queries: int = 1, # consecutive rows sharing one request (EAGLE verify)
|
||||
) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
# Index top-k sharing for DECODE. A group's source layer passes ``topk_out``
|
||||
# (a persistent buffer) and publishes its reduced top-k there; the group's
|
||||
@@ -362,6 +363,7 @@ def minimax_sparse_decode(
|
||||
idx_sm_scale,
|
||||
idx_q_scale,
|
||||
idx_k_scale,
|
||||
packed_queries=packed_queries,
|
||||
)
|
||||
_skip_reduce = True
|
||||
else:
|
||||
|
||||
Reference in New Issue
Block a user